Auto merge of #130540 - veera-sivarajan:fix-87525, r=estebank

Add a Lint for Pointer to Integer Transmutes in Consts

Fixes #87525

This PR adds a MirLint for pointer to integer transmutes in const functions and associated consts. The implementation closely follows this comment: https://github.com/rust-lang/rust/pull/85769#issuecomment-880969112. More details about the implementation can be found in the comments.

Note: This could break some sound code as mentioned by RalfJung in https://github.com/rust-lang/rust/pull/85769#issuecomment-886491680:

> ... technically const-code could transmute/cast an int to a ptr and then transmute it back and that would be correct -- so the lint will deny some sound code. Does not seem terribly likely though.

References:
1. https://doc.rust-lang.org/std/mem/fn.transmute.html
2. https://doc.rust-lang.org/reference/items/associated-items.html#associated-constants

10 files changed, 262 insertions, 6 deletions

diff --git a/compiler/rustc_lint_defs/src/builtin.rs b/compiler/rustc_lint_defs/src/builtin.rs
index 549dc64a562..bd87019508b 100644
--- a/compiler/rustc_lint_defs/src/builtin.rs
+++ b/compiler/rustc_lint_defs/src/builtin.rs
@@ -81,6 +81,7 @@ declare_lint_pass! {
         PRIVATE_INTERFACES,
         PROC_MACRO_DERIVE_RESOLUTION_FALLBACK,
         PTR_CAST_ADD_AUTO_TO_OBJECT,
+        PTR_TO_INTEGER_TRANSMUTE_IN_CONSTS,
         PUB_USE_OF_PRIVATE_EXTERN_CRATE,
         REDUNDANT_IMPORTS,
         REDUNDANT_LIFETIMES,
@@ -4998,3 +4999,37 @@ declare_lint! {
         reference: "issue #124535 <https://github.com/rust-lang/rust/issues/124535>",
     };
 }
+
+declare_lint! {
+    /// The `ptr_to_integer_transmute_in_consts` lint detects pointer to integer
+    /// transmute in const functions and associated constants.
+    ///
+    /// ### Example
+    ///
+    /// ```rust
+    /// const fn foo(ptr: *const u8) -> usize {
+    ///    unsafe {
+    ///        std::mem::transmute::<*const u8, usize>(ptr)
+    ///    }
+    /// }
+    /// ```
+    ///
+    /// {{produces}}
+    ///
+    /// ### Explanation
+    ///
+    /// Transmuting pointers to integers in a `const` context is undefined behavior.
+    /// Any attempt to use the resulting integer will abort const-evaluation.
+    ///
+    /// But sometimes the compiler might not emit an error for pointer to integer transmutes
+    /// inside const functions and associated consts because they are evaluated only when referenced.
+    /// Therefore, this lint serves as an extra layer of defense to prevent any undefined behavior
+    /// from compiling without any warnings or errors.
+    ///
+    /// See [std::mem::transmute] in the reference for more details.
+    ///
+    /// [std::mem::transmute]: https://doc.rust-lang.org/std/mem/fn.transmute.html
+    pub PTR_TO_INTEGER_TRANSMUTE_IN_CONSTS,
+    Warn,
+    "detects pointer to integer transmutes in const functions and associated constants",
+}
diff --git a/compiler/rustc_mir_transform/messages.ftl b/compiler/rustc_mir_transform/messages.ftl
index f9b79d72b05..b81c0906734 100644
--- a/compiler/rustc_mir_transform/messages.ftl
+++ b/compiler/rustc_mir_transform/messages.ftl
@@ -27,3 +27,8 @@ mir_transform_unaligned_packed_ref = reference to packed field is unaligned
     .note = packed structs are only aligned by one byte, and many modern architectures penalize unaligned field accesses
     .note_ub = creating a misaligned reference is undefined behavior (even if that reference is never dereferenced)
     .help = copy the field contents to a local variable, or replace the reference with a raw pointer and use `read_unaligned`/`write_unaligned` (loads and stores via `*p` must be properly aligned even when using raw pointers)
+
+mir_transform_undefined_transmute = pointers cannot be transmuted to integers during const eval
+    .note = at compile-time, pointers do not have an integer value
+    .note2 = avoiding this restriction via `union` or raw pointers leads to compile-time undefined behavior
+    .help = for more information, see https://doc.rust-lang.org/std/mem/fn.transmute.html
diff --git a/compiler/rustc_mir_transform/src/check_undefined_transmutes.rs b/compiler/rustc_mir_transform/src/check_undefined_transmutes.rs
new file mode 100644
index 00000000000..8ba14a1158e
--- /dev/null
+++ b/compiler/rustc_mir_transform/src/check_undefined_transmutes.rs
@@ -0,0 +1,77 @@
+use rustc_middle::mir::visit::Visitor;
+use rustc_middle::mir::{Body, Location, Operand, Terminator, TerminatorKind};
+use rustc_middle::ty::{AssocItem, AssocKind, TyCtxt};
+use rustc_session::lint::builtin::PTR_TO_INTEGER_TRANSMUTE_IN_CONSTS;
+use rustc_span::sym;
+
+use crate::errors;
+
+/// Check for transmutes that exhibit undefined behavior.
+/// For example, transmuting pointers to integers in a const context.
+pub(super) struct CheckUndefinedTransmutes;
+
+impl<'tcx> crate::MirLint<'tcx> for CheckUndefinedTransmutes {
+    fn run_lint(&self, tcx: TyCtxt<'tcx>, body: &Body<'tcx>) {
+        let mut checker = UndefinedTransmutesChecker { body, tcx };
+        checker.visit_body(body);
+    }
+}
+
+struct UndefinedTransmutesChecker<'a, 'tcx> {
+    body: &'a Body<'tcx>,
+    tcx: TyCtxt<'tcx>,
+}
+
+impl<'a, 'tcx> UndefinedTransmutesChecker<'a, 'tcx> {
+    // This functions checks two things:
+    // 1. `function` takes a raw pointer as input and returns an integer as output.
+    // 2. `function` is called from a const function or an associated constant.
+    //
+    // Why do we consider const functions and associated constants only?
+    //
+    // Generally, undefined behavior in const items are handled by the evaluator.
+    // But, const functions and associated constants are evaluated only when referenced.
+    // This can result in undefined behavior in a library going unnoticed until
+    // the function or constant is actually used.
+    //
+    // Therefore, we only consider const functions and associated constants here and leave
+    // other const items to be handled by the evaluator.
+    fn is_ptr_to_int_in_const(&self, function: &Operand<'tcx>) -> bool {
+        let def_id = self.body.source.def_id();
+
+        if self.tcx.is_const_fn(def_id)
+            || matches!(
+                self.tcx.opt_associated_item(def_id),
+                Some(AssocItem { kind: AssocKind::Const, .. })
+            )
+        {
+            let fn_sig = function.ty(self.body, self.tcx).fn_sig(self.tcx).skip_binder();
+            if let [input] = fn_sig.inputs() {
+                return input.is_unsafe_ptr() && fn_sig.output().is_integral();
+            }
+        }
+        false
+    }
+}
+
+impl<'tcx> Visitor<'tcx> for UndefinedTransmutesChecker<'_, 'tcx> {
+    // Check each block's terminator for calls to pointer to integer transmutes
+    // in const functions or associated constants and emit a lint.
+    fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, location: Location) {
+        if let TerminatorKind::Call { func, .. } = &terminator.kind
+            && let Some((func_def_id, _)) = func.const_fn_def()
+            && self.tcx.is_intrinsic(func_def_id, sym::transmute)
+            && self.is_ptr_to_int_in_const(func)
+            && let Some(call_id) = self.body.source.def_id().as_local()
+        {
+            let hir_id = self.tcx.local_def_id_to_hir_id(call_id);
+            let span = self.body.source_info(location).span;
+            self.tcx.emit_node_span_lint(
+                PTR_TO_INTEGER_TRANSMUTE_IN_CONSTS,
+                hir_id,
+                span,
+                errors::UndefinedTransmute,
+            );
+        }
+    }
+}
diff --git a/compiler/rustc_mir_transform/src/errors.rs b/compiler/rustc_mir_transform/src/errors.rs
index 84d44c2ab4c..fa279ace431 100644
--- a/compiler/rustc_mir_transform/src/errors.rs
+++ b/compiler/rustc_mir_transform/src/errors.rs
@@ -121,3 +121,10 @@ pub(crate) struct MustNotSuspendReason {
     pub span: Span,
     pub reason: String,
 }
+
+#[derive(LintDiagnostic)]
+#[diag(mir_transform_undefined_transmute)]
+#[note]
+#[note(mir_transform_note2)]
+#[help]
+pub(crate) struct UndefinedTransmute;
diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs
index 4c090665992..d184328748f 100644
--- a/compiler/rustc_mir_transform/src/lib.rs
+++ b/compiler/rustc_mir_transform/src/lib.rs
@@ -51,6 +51,7 @@ mod add_subtyping_projections;
 mod check_alignment;
 mod check_const_item_mutation;
 mod check_packed_ref;
+mod check_undefined_transmutes;
 // This pass is public to allow external drivers to perform MIR cleanup
 pub mod cleanup_post_borrowck;
 mod copy_prop;
@@ -298,6 +299,7 @@ fn mir_built(tcx: TyCtxt<'_>, def: LocalDefId) -> &Steal<Body<'_>> {
             &Lint(check_packed_ref::CheckPackedRef),
             &Lint(check_const_item_mutation::CheckConstItemMutation),
             &Lint(function_item_references::FunctionItemReferences),
+            &Lint(check_undefined_transmutes::CheckUndefinedTransmutes),
             // What we need to do constant evaluation.
             &simplify::SimplifyCfg::Initial,
             &Lint(sanity_check::SanityCheck),
diff --git a/library/core/src/ptr/mod.rs b/library/core/src/ptr/mod.rs
index 205b25f2d1a..67f1b0cd16d 100644
--- a/library/core/src/ptr/mod.rs
+++ b/library/core/src/ptr/mod.rs
@@ -1909,6 +1909,7 @@ pub unsafe fn write_volatile<T>(dst: *mut T, src: T) {
 /// than trying to adapt this to accommodate that change.
 ///
 /// Any questions go to @nagisa.
+#[cfg_attr(not(bootstrap), allow(ptr_to_integer_transmute_in_consts))]
 #[lang = "align_offset"]
 pub(crate) const unsafe fn align_offset<T: Sized>(p: *const T, a: usize) -> usize {
     // FIXME(#75598): Direct use of these intrinsics improves codegen significantly at opt-level <=
diff --git a/src/tools/clippy/tests/ui/transmutes_expressible_as_ptr_casts.fixed b/src/tools/clippy/tests/ui/transmutes_expressible_as_ptr_casts.fixed
index e95054a7ccb..617d32d1fa7 100644
--- a/src/tools/clippy/tests/ui/transmutes_expressible_as_ptr_casts.fixed
+++ b/src/tools/clippy/tests/ui/transmutes_expressible_as_ptr_casts.fixed
@@ -84,8 +84,11 @@ fn issue_10449() {
 }
 
 // Pointers cannot be cast to integers in const contexts
+#[allow(ptr_to_integer_transmute_in_consts, reason = "This is tested in the compiler test suite")]
 const fn issue_12402<P>(ptr: *const P) {
-    unsafe { transmute::<*const i32, usize>(&42i32) };
-    unsafe { transmute::<fn(*const P), usize>(issue_12402) };
-    let _ = unsafe { transmute::<_, usize>(ptr) };
+    // This test exists even though the compiler lints against it
+    // to test that clippy's transmute lints do not trigger on this.
+    unsafe { std::mem::transmute::<*const i32, usize>(&42i32) };
+    unsafe { std::mem::transmute::<fn(*const P), usize>(issue_12402) };
+    let _ = unsafe { std::mem::transmute::<_, usize>(ptr) };
 }
diff --git a/src/tools/clippy/tests/ui/transmutes_expressible_as_ptr_casts.rs b/src/tools/clippy/tests/ui/transmutes_expressible_as_ptr_casts.rs
index e5fcdef7a1c..d68db3c2deb 100644
--- a/src/tools/clippy/tests/ui/transmutes_expressible_as_ptr_casts.rs
+++ b/src/tools/clippy/tests/ui/transmutes_expressible_as_ptr_casts.rs
@@ -84,8 +84,11 @@ fn issue_10449() {
 }
 
 // Pointers cannot be cast to integers in const contexts
+#[allow(ptr_to_integer_transmute_in_consts, reason = "This is tested in the compiler test suite")]
 const fn issue_12402<P>(ptr: *const P) {
-    unsafe { transmute::<*const i32, usize>(&42i32) };
-    unsafe { transmute::<fn(*const P), usize>(issue_12402) };
-    let _ = unsafe { transmute::<_, usize>(ptr) };
+    // This test exists even though the compiler lints against it
+    // to test that clippy's transmute lints do not trigger on this.
+    unsafe { std::mem::transmute::<*const i32, usize>(&42i32) };
+    unsafe { std::mem::transmute::<fn(*const P), usize>(issue_12402) };
+    let _ = unsafe { std::mem::transmute::<_, usize>(ptr) };
 }
diff --git a/tests/ui/consts/const-eval/ptr-to-int-transmute-in-consts-issue-87525.rs b/tests/ui/consts/const-eval/ptr-to-int-transmute-in-consts-issue-87525.rs
new file mode 100644
index 00000000000..19c78f019aa
--- /dev/null
+++ b/tests/ui/consts/const-eval/ptr-to-int-transmute-in-consts-issue-87525.rs
@@ -0,0 +1,70 @@
+const fn foo(ptr: *const u8) -> usize {
+    unsafe {
+        std::mem::transmute(ptr)
+        //~^ WARN pointers cannot be transmuted to integers
+    }
+}
+
+trait Human {
+    const ID: usize = {
+        let value = 10;
+        let ptr: *const usize = &value;
+        unsafe {
+            std::mem::transmute(ptr)
+            //~^ WARN pointers cannot be transmuted to integers
+        }
+    };
+
+    fn id_plus_one() -> usize {
+        Self::ID + 1
+    }
+}
+
+struct Type<T>(T);
+
+impl<T> Type<T> {
+    const ID: usize = {
+        let value = 10;
+        let ptr: *const usize = &value;
+        unsafe {
+            std::mem::transmute(ptr)
+            //~^ WARN pointers cannot be transmuted to integers
+        }
+    };
+
+    fn id_plus_one() -> usize {
+        Self::ID + 1
+    }
+}
+
+fn control(ptr: *const u8) -> usize {
+    unsafe {
+        std::mem::transmute(ptr)
+    }
+}
+
+struct ControlStruct;
+
+impl ControlStruct {
+    fn new() -> usize {
+        let value = 10;
+        let ptr: *const i32 = &value;
+        unsafe {
+            std::mem::transmute(ptr)
+        }
+    }
+}
+
+
+const fn zoom(ptr: *const u8) -> usize {
+    unsafe {
+        std::mem::transmute(ptr)
+        //~^ WARN pointers cannot be transmuted to integers
+    }
+}
+
+fn main() {
+    const a: u8 = 10;
+    const value: usize = zoom(&a);
+    //~^ ERROR evaluation of constant value failed
+}
diff --git a/tests/ui/consts/const-eval/ptr-to-int-transmute-in-consts-issue-87525.stderr b/tests/ui/consts/const-eval/ptr-to-int-transmute-in-consts-issue-87525.stderr
new file mode 100644
index 00000000000..ca6ad9408ab
--- /dev/null
+++ b/tests/ui/consts/const-eval/ptr-to-int-transmute-in-consts-issue-87525.stderr
@@ -0,0 +1,53 @@
+warning: pointers cannot be transmuted to integers during const eval
+  --> $DIR/ptr-to-int-transmute-in-consts-issue-87525.rs:61:9
+   |
+LL |         std::mem::transmute(ptr)
+   |         ^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: at compile-time, pointers do not have an integer value
+   = note: avoiding this restriction via `union` or raw pointers leads to compile-time undefined behavior
+   = help: for more information, see https://doc.rust-lang.org/std/mem/fn.transmute.html
+   = note: `#[warn(ptr_to_integer_transmute_in_consts)]` on by default
+
+error[E0080]: evaluation of constant value failed
+  --> $DIR/ptr-to-int-transmute-in-consts-issue-87525.rs:68:26
+   |
+LL |     const value: usize = zoom(&a);
+   |                          ^^^^^^^^ unable to turn pointer into integer
+   |
+   = help: this code performed an operation that depends on the underlying bytes representing a pointer
+   = help: the absolute address of a pointer is not known at compile-time, so such operations are not supported
+
+warning: pointers cannot be transmuted to integers during const eval
+  --> $DIR/ptr-to-int-transmute-in-consts-issue-87525.rs:3:9
+   |
+LL |         std::mem::transmute(ptr)
+   |         ^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: at compile-time, pointers do not have an integer value
+   = note: avoiding this restriction via `union` or raw pointers leads to compile-time undefined behavior
+   = help: for more information, see https://doc.rust-lang.org/std/mem/fn.transmute.html
+
+warning: pointers cannot be transmuted to integers during const eval
+  --> $DIR/ptr-to-int-transmute-in-consts-issue-87525.rs:13:13
+   |
+LL |             std::mem::transmute(ptr)
+   |             ^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: at compile-time, pointers do not have an integer value
+   = note: avoiding this restriction via `union` or raw pointers leads to compile-time undefined behavior
+   = help: for more information, see https://doc.rust-lang.org/std/mem/fn.transmute.html
+
+warning: pointers cannot be transmuted to integers during const eval
+  --> $DIR/ptr-to-int-transmute-in-consts-issue-87525.rs:30:13
+   |
+LL |             std::mem::transmute(ptr)
+   |             ^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: at compile-time, pointers do not have an integer value
+   = note: avoiding this restriction via `union` or raw pointers leads to compile-time undefined behavior
+   = help: for more information, see https://doc.rust-lang.org/std/mem/fn.transmute.html
+
+error: aborting due to 1 previous error; 4 warnings emitted
+
+For more information about this error, try `rustc --explain E0080`.