Rollup merge of #102044 - ChrisDenton:BCrypt-system-rand, r=thomcc

Remove `RtlGenRandom` (take two)

First try to use the system preferred RNG but if that fails (e.g. due to a broken system configuration) then fallback to manually opening an algorithm handle.

2 files changed, 29 insertions, 53 deletions

diff --git a/library/std/src/sys/windows/c.rs b/library/std/src/sys/windows/c.rs
index 89d0ab59be8..c61a7e7d1e4 100644
--- a/library/std/src/sys/windows/c.rs
+++ b/library/std/src/sys/windows/c.rs
@@ -279,7 +279,6 @@ pub const STATUS_INVALID_PARAMETER: NTSTATUS = 0xc000000d_u32 as _;
 pub const STATUS_PENDING: NTSTATUS = 0x103 as _;
 pub const STATUS_END_OF_FILE: NTSTATUS = 0xC0000011_u32 as _;
 pub const STATUS_NOT_IMPLEMENTED: NTSTATUS = 0xC0000002_u32 as _;
-pub const STATUS_NOT_SUPPORTED: NTSTATUS = 0xC00000BB_u32 as _;
 
 // Equivalent to the `NT_SUCCESS` C preprocessor macro.
 // See: https://docs.microsoft.com/en-us/windows-hardware/drivers/kernel/using-ntstatus-values
@@ -289,6 +288,7 @@ pub fn nt_success(status: NTSTATUS) -> bool {
 
 // "RNG\0"
 pub const BCRYPT_RNG_ALGORITHM: &[u16] = &[b'R' as u16, b'N' as u16, b'G' as u16, 0];
+pub const BCRYPT_USE_SYSTEM_PREFERRED_RNG: DWORD = 0x00000002;
 
 #[repr(C)]
 pub struct UNICODE_STRING {
@@ -817,10 +817,6 @@ if #[cfg(not(target_vendor = "uwp"))] {
 
     #[link(name = "advapi32")]
     extern "system" {
-        // Forbidden when targeting UWP
-        #[link_name = "SystemFunction036"]
-        pub fn RtlGenRandom(RandomBuffer: *mut u8, RandomBufferLength: ULONG) -> BOOLEAN;
-
         // Allowed but unused by UWP
         pub fn OpenProcessToken(
             ProcessHandle: HANDLE,
diff --git a/library/std/src/sys/windows/rand.rs b/library/std/src/sys/windows/rand.rs
index d6cd8f80271..b5a49489d3f 100644
--- a/library/std/src/sys/windows/rand.rs
+++ b/library/std/src/sys/windows/rand.rs
@@ -13,15 +13,12 @@
 //! but significant number of users to experience panics caused by a failure of
 //! this function. See [#94098].
 //!
-//! The current version changes this to use the `BCRYPT_RNG_ALG_HANDLE`
-//! [Pseudo-handle], which gets the default RNG algorithm without querying the
-//! system preference thus hopefully avoiding the previous issue.
-//! This is only supported on Windows 10+ so a fallback is used for older versions.
+//! The current version falls back to using `BCryptOpenAlgorithmProvider` if
+//! `BCRYPT_USE_SYSTEM_PREFERRED_RNG` fails for any reason.
 //!
 //! [#94098]: https://github.com/rust-lang/rust/issues/94098
 //! [`RtlGenRandom`]: https://docs.microsoft.com/en-us/windows/win32/api/ntsecapi/nf-ntsecapi-rtlgenrandom
 //! [`BCryptGenRandom`]: https://docs.microsoft.com/en-us/windows/win32/api/bcrypt/nf-bcrypt-bcryptgenrandom
-//! [Pseudo-handle]: https://docs.microsoft.com/en-us/windows/win32/seccng/cng-algorithm-pseudo-handles
 use crate::mem;
 use crate::ptr;
 use crate::sys::c;
@@ -33,37 +30,35 @@ use crate::sys::c;
 /// [`HashMap`]: crate::collections::HashMap
 /// [`RandomState`]: crate::collections::hash_map::RandomState
 pub fn hashmap_random_keys() -> (u64, u64) {
-    Rng::open().and_then(|rng| rng.gen_random_keys()).unwrap_or_else(fallback_rng)
+    Rng::SYSTEM.gen_random_keys().unwrap_or_else(fallback_rng)
 }
 
-struct Rng(c::BCRYPT_ALG_HANDLE);
+struct Rng {
+    algorithm: c::BCRYPT_ALG_HANDLE,
+    flags: u32,
+}
 impl Rng {
-    #[cfg(miri)]
-    fn open() -> Result<Self, c::NTSTATUS> {
-        const BCRYPT_RNG_ALG_HANDLE: c::BCRYPT_ALG_HANDLE = ptr::invalid_mut(0x81);
-        let _ = (
-            c::BCryptOpenAlgorithmProvider,
-            c::BCryptCloseAlgorithmProvider,
-            c::BCRYPT_RNG_ALGORITHM,
-            c::STATUS_NOT_SUPPORTED,
-        );
-        Ok(Self(BCRYPT_RNG_ALG_HANDLE))
+    const SYSTEM: Self = unsafe { Self::new(ptr::null_mut(), c::BCRYPT_USE_SYSTEM_PREFERRED_RNG) };
+
+    /// Create the RNG from an existing algorithm handle.
+    ///
+    /// # Safety
+    ///
+    /// The handle must either be null or a valid algorithm handle.
+    const unsafe fn new(algorithm: c::BCRYPT_ALG_HANDLE, flags: u32) -> Self {
+        Self { algorithm, flags }
     }
-    #[cfg(not(miri))]
-    // Open a handle to the RNG algorithm.
+
+    /// Open a handle to the RNG algorithm.
     fn open() -> Result<Self, c::NTSTATUS> {
         use crate::sync::atomic::AtomicPtr;
         use crate::sync::atomic::Ordering::{Acquire, Release};
-        const ERROR_VALUE: c::LPVOID = ptr::invalid_mut(usize::MAX);
 
         // An atomic is used so we don't need to reopen the handle every time.
         static HANDLE: AtomicPtr<crate::ffi::c_void> = AtomicPtr::new(ptr::null_mut());
 
         let mut handle = HANDLE.load(Acquire);
-        // We use a sentinel value to designate an error occurred last time.
-        if handle == ERROR_VALUE {
-            Err(c::STATUS_NOT_SUPPORTED)
-        } else if handle.is_null() {
+        if handle.is_null() {
             let status = unsafe {
                 c::BCryptOpenAlgorithmProvider(
                     &mut handle,
@@ -80,13 +75,12 @@ impl Rng {
                     unsafe { c::BCryptCloseAlgorithmProvider(handle, 0) };
                     handle = previous_handle;
                 }
-                Ok(Self(handle))
+                Ok(unsafe { Self::new(handle, 0) })
             } else {
-                HANDLE.store(ERROR_VALUE, Release);
                 Err(status)
             }
         } else {
-            Ok(Self(handle))
+            Ok(unsafe { Self::new(handle, 0) })
         }
     }
 
@@ -94,33 +88,19 @@ impl Rng {
         let mut v = (0, 0);
         let status = unsafe {
             let size = mem::size_of_val(&v).try_into().unwrap();
-            c::BCryptGenRandom(self.0, ptr::addr_of_mut!(v).cast(), size, 0)
+            c::BCryptGenRandom(self.algorithm, ptr::addr_of_mut!(v).cast(), size, self.flags)
         };
         if c::nt_success(status) { Ok(v) } else { Err(status) }
     }
 }
 
-/// Generate random numbers using the fallback RNG function (RtlGenRandom)
-#[cfg(not(target_vendor = "uwp"))]
+/// Generate random numbers using the fallback RNG function
 #[inline(never)]
 fn fallback_rng(rng_status: c::NTSTATUS) -> (u64, u64) {
-    let mut v = (0, 0);
-    let ret =
-        unsafe { c::RtlGenRandom(&mut v as *mut _ as *mut u8, mem::size_of_val(&v) as c::ULONG) };
-
-    if ret != 0 {
-        v
-    } else {
-        panic!(
-            "RNG broken: {rng_status:#x}, fallback RNG broken: {}",
-            crate::io::Error::last_os_error()
-        )
+    match Rng::open().and_then(|rng| rng.gen_random_keys()) {
+        Ok(keys) => keys,
+        Err(status) => {
+            panic!("RNG broken: {rng_status:#x}, fallback RNG broken: {status:#x}")
+        }
     }
 }
-
-/// We can't use RtlGenRandom with UWP, so there is no fallback
-#[cfg(target_vendor = "uwp")]
-#[inline(never)]
-fn fallback_rng(rng_status: c::NTSTATUS) -> (u64, u64) {
-    panic!("RNG broken: {rng_status:#x} fallback RNG broken: RtlGenRandom() not supported on UWP");
-}