diff options
15 files changed, 383 insertions, 495 deletions
diff --git a/src/tools/miri/src/concurrency/init_once.rs b/src/tools/miri/src/concurrency/init_once.rs index 488edc91dff..c3c04428c94 100644 --- a/src/tools/miri/src/concurrency/init_once.rs +++ b/src/tools/miri/src/concurrency/init_once.rs @@ -2,7 +2,6 @@ use std::collections::VecDeque; use rustc_index::Idx; -use super::sync::EvalContextExtPriv as _; use super::vector_clock::VClock; use crate::*; @@ -27,22 +26,6 @@ pub(super) struct InitOnce { impl<'tcx> EvalContextExt<'tcx> for crate::MiriInterpCx<'tcx> {} pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { - fn init_once_get_or_create_id( - &mut self, - lock: &MPlaceTy<'tcx>, - offset: u64, - ) -> InterpResult<'tcx, InitOnceId> { - let this = self.eval_context_mut(); - this.get_or_create_id( - lock, - offset, - |ecx| &mut ecx.machine.sync.init_onces, - |_| interp_ok(Default::default()), - )? - .ok_or_else(|| err_ub_format!("init_once has invalid ID")) - .into() - } - #[inline] fn init_once_status(&mut self, id: InitOnceId) -> InitOnceStatus { let this = self.eval_context_ref(); diff --git a/src/tools/miri/src/concurrency/sync.rs b/src/tools/miri/src/concurrency/sync.rs index e1e748e7945..2c6a7bf0f58 100644 --- a/src/tools/miri/src/concurrency/sync.rs +++ b/src/tools/miri/src/concurrency/sync.rs @@ -1,4 +1,3 @@ -use std::any::Any; use std::collections::VecDeque; use std::collections::hash_map::Entry; use std::ops::Not; @@ -12,11 +11,6 @@ use super::init_once::InitOnce; use super::vector_clock::VClock; use crate::*; -pub trait SyncId { - fn from_u32(id: u32) -> Self; - fn to_u32(&self) -> u32; -} - /// We cannot use the `newtype_index!` macro because we have to use 0 as a /// sentinel value meaning that the identifier is not assigned. This is because /// the pthreads static initializers initialize memory with zeros (see the @@ -28,16 +22,6 @@ macro_rules! declare_id { #[derive(Clone, Copy, Debug, PartialOrd, Ord, PartialEq, Eq, Hash)] pub struct $name(std::num::NonZero<u32>); - impl $crate::concurrency::sync::SyncId for $name { - // Panics if `id == 0`. - fn from_u32(id: u32) -> Self { - Self(std::num::NonZero::new(id).unwrap()) - } - fn to_u32(&self) -> u32 { - self.0.get() - } - } - impl $crate::VisitProvenance for $name { fn visit_provenance(&self, _visit: &mut VisitWith<'_>) {} } @@ -56,12 +40,6 @@ macro_rules! declare_id { usize::try_from(self.0.get() - 1).unwrap() } } - - impl $name { - pub fn to_u32_scalar(&self) -> Scalar { - Scalar::from_u32(self.0.get()) - } - } }; } pub(super) use declare_id; @@ -79,9 +57,6 @@ struct Mutex { queue: VecDeque<ThreadId>, /// Mutex clock. This tracks the moment of the last unlock. clock: VClock, - - /// Additional data that can be set by shim implementations. - data: Option<Box<dyn Any>>, } declare_id!(RwLockId); @@ -118,9 +93,6 @@ struct RwLock { /// locks. /// This is only relevant when there is an active reader. clock_current_readers: VClock, - - /// Additional data that can be set by shim implementations. - data: Option<Box<dyn Any>>, } declare_id!(CondvarId); @@ -135,9 +107,6 @@ struct Condvar { /// Contains the clock of the last thread to /// perform a condvar-signal. clock: VClock, - - /// Additional data that can be set by shim implementations. - data: Option<Box<dyn Any>>, } /// The futex state. @@ -167,89 +136,15 @@ pub struct SynchronizationObjects { mutexes: IndexVec<MutexId, Mutex>, rwlocks: IndexVec<RwLockId, RwLock>, condvars: IndexVec<CondvarId, Condvar>, - futexes: FxHashMap<u64, Futex>, pub(super) init_onces: IndexVec<InitOnceId, InitOnce>, + + /// Futex info for the futex at the given address. + futexes: FxHashMap<u64, Futex>, } // Private extension trait for local helper methods impl<'tcx> EvalContextExtPriv<'tcx> for crate::MiriInterpCx<'tcx> {} pub(super) trait EvalContextExtPriv<'tcx>: crate::MiriInterpCxExt<'tcx> { - /// Lazily initialize the ID of this Miri sync structure. - /// If memory stores '0', that indicates uninit and we generate a new instance. - /// Returns `None` if memory stores a non-zero invalid ID. - /// - /// `get_objs` must return the `IndexVec` that stores all the objects of this type. - /// `create_obj` must create the new object if initialization is needed. - #[inline] - fn get_or_create_id<Id: SyncId + Idx, T>( - &mut self, - lock: &MPlaceTy<'tcx>, - offset: u64, - get_objs: impl for<'a> Fn(&'a mut MiriInterpCx<'tcx>) -> &'a mut IndexVec<Id, T>, - create_obj: impl for<'a> FnOnce(&'a mut MiriInterpCx<'tcx>) -> InterpResult<'tcx, T>, - ) -> InterpResult<'tcx, Option<Id>> { - let this = self.eval_context_mut(); - let offset = Size::from_bytes(offset); - assert!(lock.layout.size >= offset + this.machine.layouts.u32.size); - let id_place = lock.offset(offset, this.machine.layouts.u32, this)?; - let next_index = get_objs(this).next_index(); - - // Since we are lazy, this update has to be atomic. - let (old, success) = this - .atomic_compare_exchange_scalar( - &id_place, - &ImmTy::from_uint(0u32, this.machine.layouts.u32), - Scalar::from_u32(next_index.to_u32()), - AtomicRwOrd::Relaxed, // deliberately *no* synchronization - AtomicReadOrd::Relaxed, - false, - )? - .to_scalar_pair(); - - interp_ok(if success.to_bool().expect("compare_exchange's second return value is a bool") { - // We set the in-memory ID to `next_index`, now also create this object in the machine - // state. - let obj = create_obj(this)?; - let new_index = get_objs(this).push(obj); - assert_eq!(next_index, new_index); - Some(new_index) - } else { - let id = Id::from_u32(old.to_u32().expect("layout is u32")); - if get_objs(this).get(id).is_none() { - // The in-memory ID is invalid. - None - } else { - Some(id) - } - }) - } - - /// Eagerly creates a Miri sync structure. - /// - /// `create_id` will store the index of the sync_structure in the memory pointed to by - /// `lock_op`, so that future calls to `get_or_create_id` will see it as initialized. - /// - `lock_op` must hold a pointer to the sync structure. - /// - `lock_layout` must be the memory layout of the sync structure. - /// - `offset` must be the offset inside the sync structure where its miri id will be stored. - /// - `get_objs` is described in `get_or_create_id`. - /// - `obj` must be the new sync object. - fn create_id<Id: SyncId + Idx, T>( - &mut self, - lock: &MPlaceTy<'tcx>, - offset: u64, - get_objs: impl for<'a> Fn(&'a mut MiriInterpCx<'tcx>) -> &'a mut IndexVec<Id, T>, - obj: T, - ) -> InterpResult<'tcx, Id> { - let this = self.eval_context_mut(); - let offset = Size::from_bytes(offset); - assert!(lock.layout.size >= offset + this.machine.layouts.u32.size); - let id_place = lock.offset(offset, this.machine.layouts.u32, this)?; - - let new_index = get_objs(this).push(obj); - this.write_scalar(Scalar::from_u32(new_index.to_u32()), &id_place)?; - interp_ok(new_index) - } - fn condvar_reacquire_mutex( &mut self, mutex: MutexId, @@ -270,126 +165,103 @@ pub(super) trait EvalContextExtPriv<'tcx>: crate::MiriInterpCxExt<'tcx> { } } -// Public interface to synchronization primitives. Please note that in most -// cases, the function calls are infallible and it is the client's (shim -// implementation's) responsibility to detect and deal with erroneous -// situations. -impl<'tcx> EvalContextExt<'tcx> for crate::MiriInterpCx<'tcx> {} -pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { - /// Eagerly create and initialize a new mutex. - fn mutex_create( - &mut self, - lock: &MPlaceTy<'tcx>, - offset: u64, - data: Option<Box<dyn Any>>, - ) -> InterpResult<'tcx, MutexId> { - let this = self.eval_context_mut(); - this.create_id(lock, offset, |ecx| &mut ecx.machine.sync.mutexes, Mutex { - data, - ..Default::default() - }) +impl SynchronizationObjects { + pub fn mutex_create(&mut self) -> MutexId { + self.mutexes.push(Default::default()) } - /// Lazily create a new mutex. - /// `initialize_data` must return any additional data that a user wants to associate with the mutex. - fn mutex_get_or_create_id( - &mut self, - lock: &MPlaceTy<'tcx>, - offset: u64, - initialize_data: impl for<'a> FnOnce( - &'a mut MiriInterpCx<'tcx>, - ) -> InterpResult<'tcx, Option<Box<dyn Any>>>, - ) -> InterpResult<'tcx, MutexId> { - let this = self.eval_context_mut(); - this.get_or_create_id( - lock, - offset, - |ecx| &mut ecx.machine.sync.mutexes, - |ecx| initialize_data(ecx).map(|data| Mutex { data, ..Default::default() }), - )? - .ok_or_else(|| err_ub_format!("mutex has invalid ID")) - .into() + pub fn rwlock_create(&mut self) -> RwLockId { + self.rwlocks.push(Default::default()) } - /// Retrieve the additional data stored for a mutex. - fn mutex_get_data<'a, T: 'static>(&'a mut self, id: MutexId) -> Option<&'a T> - where - 'tcx: 'a, - { - let this = self.eval_context_ref(); - this.machine.sync.mutexes[id].data.as_deref().and_then(|p| p.downcast_ref::<T>()) + pub fn condvar_create(&mut self) -> CondvarId { + self.condvars.push(Default::default()) } - fn rwlock_get_or_create_id( - &mut self, - lock: &MPlaceTy<'tcx>, - offset: u64, - initialize_data: impl for<'a> FnOnce( - &'a mut MiriInterpCx<'tcx>, - ) -> InterpResult<'tcx, Option<Box<dyn Any>>>, - ) -> InterpResult<'tcx, RwLockId> { - let this = self.eval_context_mut(); - this.get_or_create_id( - lock, - offset, - |ecx| &mut ecx.machine.sync.rwlocks, - |ecx| initialize_data(ecx).map(|data| RwLock { data, ..Default::default() }), - )? - .ok_or_else(|| err_ub_format!("rwlock has invalid ID")) - .into() + pub fn init_once_create(&mut self) -> InitOnceId { + self.init_onces.push(Default::default()) } +} - /// Retrieve the additional data stored for a rwlock. - fn rwlock_get_data<'a, T: 'static>(&'a mut self, id: RwLockId) -> Option<&'a T> - where - 'tcx: 'a, - { - let this = self.eval_context_ref(); - this.machine.sync.rwlocks[id].data.as_deref().and_then(|p| p.downcast_ref::<T>()) +impl<'tcx> AllocExtra<'tcx> { + pub fn get_sync<T: 'static>(&self, offset: Size) -> Option<&T> { + self.sync.get(&offset).and_then(|s| s.downcast_ref::<T>()) } +} - /// Eagerly create and initialize a new condvar. - fn condvar_create( - &mut self, - condvar: &MPlaceTy<'tcx>, - offset: u64, - data: Option<Box<dyn Any>>, - ) -> InterpResult<'tcx, CondvarId> { - let this = self.eval_context_mut(); - this.create_id(condvar, offset, |ecx| &mut ecx.machine.sync.condvars, Condvar { - data, - ..Default::default() - }) - } +/// We designate an `init`` field in all primitives. +/// If `init` is set to this, we consider the primitive initialized. +pub const LAZY_INIT_COOKIE: u32 = 0xcafe_affe; + +/// Helper for lazily initialized `alloc_extra.sync` data: +/// this forces an immediate init. +pub fn lazy_sync_init<'tcx, T: 'static + Copy>( + ecx: &mut MiriInterpCx<'tcx>, + primitive: &MPlaceTy<'tcx>, + init_offset: Size, + data: T, +) -> InterpResult<'tcx> { + let (alloc, offset, _) = ecx.ptr_get_alloc_id(primitive.ptr(), 0)?; + let (alloc_extra, _machine) = ecx.get_alloc_extra_mut(alloc)?; + alloc_extra.sync.insert(offset, Box::new(data)); + // Mark this as "initialized". + let init_field = primitive.offset(init_offset, ecx.machine.layouts.u32, ecx)?; + ecx.write_scalar_atomic( + Scalar::from_u32(LAZY_INIT_COOKIE), + &init_field, + AtomicWriteOrd::Relaxed, + )?; + interp_ok(()) +} - fn condvar_get_or_create_id( - &mut self, - lock: &MPlaceTy<'tcx>, - offset: u64, - initialize_data: impl for<'a> FnOnce( - &'a mut MiriInterpCx<'tcx>, - ) -> InterpResult<'tcx, Option<Box<dyn Any>>>, - ) -> InterpResult<'tcx, CondvarId> { - let this = self.eval_context_mut(); - this.get_or_create_id( - lock, - offset, - |ecx| &mut ecx.machine.sync.condvars, - |ecx| initialize_data(ecx).map(|data| Condvar { data, ..Default::default() }), +/// Helper for lazily initialized `alloc_extra.sync` data: +/// Checks if the primitive is initialized, and return its associated data if so. +/// Otherwise, calls `new_data` to initialize the primitive. +pub fn lazy_sync_get_data<'tcx, T: 'static + Copy>( + ecx: &mut MiriInterpCx<'tcx>, + primitive: &MPlaceTy<'tcx>, + init_offset: Size, + name: &str, + new_data: impl FnOnce(&mut MiriInterpCx<'tcx>) -> InterpResult<'tcx, T>, +) -> InterpResult<'tcx, T> { + // Check if this is already initialized. Needs to be atomic because we can race with another + // thread initializing. Needs to be an RMW operation to ensure we read the *latest* value. + // So we just try to replace MUTEX_INIT_COOKIE with itself. + let init_cookie = Scalar::from_u32(LAZY_INIT_COOKIE); + let init_field = primitive.offset(init_offset, ecx.machine.layouts.u32, ecx)?; + let (_init, success) = ecx + .atomic_compare_exchange_scalar( + &init_field, + &ImmTy::from_scalar(init_cookie, ecx.machine.layouts.u32), + init_cookie, + AtomicRwOrd::Relaxed, + AtomicReadOrd::Relaxed, + /* can_fail_spuriously */ false, )? - .ok_or_else(|| err_ub_format!("condvar has invalid ID")) - .into() - } - - /// Retrieve the additional data stored for a condvar. - fn condvar_get_data<'a, T: 'static>(&'a mut self, id: CondvarId) -> Option<&'a T> - where - 'tcx: 'a, - { - let this = self.eval_context_ref(); - this.machine.sync.condvars[id].data.as_deref().and_then(|p| p.downcast_ref::<T>()) + .to_scalar_pair(); + + if success.to_bool()? { + // If it is initialized, it must be found in the "sync primitive" table, + // or else it has been moved illegally. + let (alloc, offset, _) = ecx.ptr_get_alloc_id(primitive.ptr(), 0)?; + let alloc_extra = ecx.get_alloc_extra(alloc)?; + let data = alloc_extra + .get_sync::<T>(offset) + .ok_or_else(|| err_ub_format!("`{name}` can't be moved after first use"))?; + interp_ok(*data) + } else { + let data = new_data(ecx)?; + lazy_sync_init(ecx, primitive, init_offset, data)?; + interp_ok(data) } +} +// Public interface to synchronization primitives. Please note that in most +// cases, the function calls are infallible and it is the client's (shim +// implementation's) responsibility to detect and deal with erroneous +// situations. +impl<'tcx> EvalContextExt<'tcx> for crate::MiriInterpCx<'tcx> {} +pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { #[inline] /// Get the id of the thread that currently owns this lock. fn mutex_get_owner(&mut self, id: MutexId) -> ThreadId { diff --git a/src/tools/miri/src/helpers.rs b/src/tools/miri/src/helpers.rs index 8bd429deefc..9ec325863ff 100644 --- a/src/tools/miri/src/helpers.rs +++ b/src/tools/miri/src/helpers.rs @@ -223,14 +223,13 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { } /// Evaluates the scalar at the specified path. - fn eval_path(&self, path: &[&str]) -> OpTy<'tcx> { + fn eval_path(&self, path: &[&str]) -> MPlaceTy<'tcx> { let this = self.eval_context_ref(); let instance = resolve_path(*this.tcx, path, Namespace::ValueNS); // We don't give a span -- this isn't actually used directly by the program anyway. - let const_val = this.eval_global(instance).unwrap_or_else(|err| { + this.eval_global(instance).unwrap_or_else(|err| { panic!("failed to evaluate required Rust item: {path:?}\n{err:?}") - }); - const_val.into() + }) } fn eval_path_scalar(&self, path: &[&str]) -> Scalar { let this = self.eval_context_ref(); diff --git a/src/tools/miri/src/machine.rs b/src/tools/miri/src/machine.rs index af0ca00a0c0..60d096b92f2 100644 --- a/src/tools/miri/src/machine.rs +++ b/src/tools/miri/src/machine.rs @@ -1,6 +1,7 @@ //! Global machine state as well as implementation of the interpreter engine //! `Machine` trait. +use std::any::Any; use std::borrow::Cow; use std::cell::RefCell; use std::collections::hash_map::Entry; @@ -336,6 +337,11 @@ pub struct AllocExtra<'tcx> { /// if this allocation is leakable. The backtrace is not /// pruned yet; that should be done before printing it. pub backtrace: Option<Vec<FrameInfo<'tcx>>>, + /// Synchronization primitives like to attach extra data to particular addresses. We store that + /// inside the relevant allocation, to ensure that everything is removed when the allocation is + /// freed. + /// This maps offsets to synchronization-primitive-specific data. + pub sync: FxHashMap<Size, Box<dyn Any>>, } // We need a `Clone` impl because the machine passes `Allocation` through `Cow`... @@ -348,7 +354,7 @@ impl<'tcx> Clone for AllocExtra<'tcx> { impl VisitProvenance for AllocExtra<'_> { fn visit_provenance(&self, visit: &mut VisitWith<'_>) { - let AllocExtra { borrow_tracker, data_race, weak_memory, backtrace: _ } = self; + let AllocExtra { borrow_tracker, data_race, weak_memory, backtrace: _, sync: _ } = self; borrow_tracker.visit_provenance(visit); data_race.visit_provenance(visit); @@ -1187,7 +1193,13 @@ impl<'tcx> Machine<'tcx> for MiriMachine<'tcx> { .insert(id, (ecx.machine.current_span(), None)); } - interp_ok(AllocExtra { borrow_tracker, data_race, weak_memory, backtrace }) + interp_ok(AllocExtra { + borrow_tracker, + data_race, + weak_memory, + backtrace, + sync: FxHashMap::default(), + }) } fn adjust_alloc_root_pointer( diff --git a/src/tools/miri/src/shims/unix/macos/sync.rs b/src/tools/miri/src/shims/unix/macos/sync.rs index 2f96849d0d2..cd5b0ed1d07 100644 --- a/src/tools/miri/src/shims/unix/macos/sync.rs +++ b/src/tools/miri/src/shims/unix/macos/sync.rs @@ -12,15 +12,26 @@ use crate::*; +struct MacOsUnfairLock { + id: MutexId, +} + impl<'tcx> EvalContextExtPriv<'tcx> for crate::MiriInterpCx<'tcx> {} trait EvalContextExtPriv<'tcx>: crate::MiriInterpCxExt<'tcx> { fn os_unfair_lock_getid(&mut self, lock_ptr: &OpTy<'tcx>) -> InterpResult<'tcx, MutexId> { let this = self.eval_context_mut(); let lock = this.deref_pointer(lock_ptr)?; - // os_unfair_lock holds a 32-bit value, is initialized with zero and - // must be assumed to be opaque. Therefore, we can just store our - // internal mutex ID in the structure without anyone noticing. - this.mutex_get_or_create_id(&lock, 0, |_| interp_ok(None)) + // We store the mutex ID in the `sync` metadata. This means that when the lock is moved, + // that's just implicitly creating a new lock at the new location. + let (alloc, offset, _) = this.ptr_get_alloc_id(lock.ptr(), 0)?; + let (alloc_extra, machine) = this.get_alloc_extra_mut(alloc)?; + if let Some(data) = alloc_extra.get_sync::<MacOsUnfairLock>(offset) { + interp_ok(data.id) + } else { + let id = machine.sync.mutex_create(); + alloc_extra.sync.insert(offset, Box::new(MacOsUnfairLock { id })); + interp_ok(id) + } } } diff --git a/src/tools/miri/src/shims/unix/sync.rs b/src/tools/miri/src/shims/unix/sync.rs index b05f340861e..913f53adbbb 100644 --- a/src/tools/miri/src/shims/unix/sync.rs +++ b/src/tools/miri/src/shims/unix/sync.rs @@ -2,10 +2,42 @@ use std::sync::atomic::{AtomicBool, Ordering}; use rustc_target::abi::Size; +use crate::concurrency::sync::{LAZY_INIT_COOKIE, lazy_sync_get_data, lazy_sync_init}; use crate::*; -// pthread_mutexattr_t is either 4 or 8 bytes, depending on the platform. -// We ignore the platform layout and store our own fields: +/// Do a bytewise comparison of the two places, using relaxed atomic reads. This is used to check if +/// a synchronization primitive matches its static initializer value. +/// +/// The reads happen in chunks of 4, so all racing accesses must also use that access size. +fn bytewise_equal_atomic_relaxed<'tcx>( + ecx: &MiriInterpCx<'tcx>, + left: &MPlaceTy<'tcx>, + right: &MPlaceTy<'tcx>, +) -> InterpResult<'tcx, bool> { + let size = left.layout.size; + assert_eq!(size, right.layout.size); + + // We do this in chunks of 4, so that we are okay to race with (sufficiently aligned) + // 4-byte atomic accesses. + assert!(size.bytes() % 4 == 0); + for i in 0..(size.bytes() / 4) { + let offset = Size::from_bytes(i.strict_mul(4)); + let load = |place: &MPlaceTy<'tcx>| { + let byte = place.offset(offset, ecx.machine.layouts.u32, ecx)?; + ecx.read_scalar_atomic(&byte, AtomicReadOrd::Relaxed)?.to_u32() + }; + let left = load(left)?; + let right = load(right)?; + if left != right { + return interp_ok(false); + } + } + + interp_ok(true) +} + +// # pthread_mutexattr_t +// We store some data directly inside the type, ignoring the platform layout: // - kind: i32 #[inline] @@ -49,52 +81,72 @@ fn mutexattr_set_kind<'tcx>( /// field *not* PTHREAD_MUTEX_DEFAULT but this special flag. const PTHREAD_MUTEX_KIND_UNCHANGED: i32 = 0x8000000; +/// Translates the mutex kind from what is stored in pthread_mutexattr_t to our enum. +fn mutexattr_translate_kind<'tcx>( + ecx: &MiriInterpCx<'tcx>, + kind: i32, +) -> InterpResult<'tcx, MutexKind> { + interp_ok(if kind == (ecx.eval_libc_i32("PTHREAD_MUTEX_NORMAL")) { + MutexKind::Normal + } else if kind == ecx.eval_libc_i32("PTHREAD_MUTEX_ERRORCHECK") { + MutexKind::ErrorCheck + } else if kind == ecx.eval_libc_i32("PTHREAD_MUTEX_RECURSIVE") { + MutexKind::Recursive + } else if kind == ecx.eval_libc_i32("PTHREAD_MUTEX_DEFAULT") + || kind == PTHREAD_MUTEX_KIND_UNCHANGED + { + // We check this *last* since PTHREAD_MUTEX_DEFAULT may be numerically equal to one of the + // others, and we want an explicit `mutexattr_settype` to work as expected. + MutexKind::Default + } else { + throw_unsup_format!("unsupported type of mutex: {kind}"); + }) +} + +// # pthread_mutex_t +// We store some data directly inside the type, ignoring the platform layout: +// - init: u32 + /// The mutex kind. #[derive(Debug, Clone, Copy)] -pub enum MutexKind { +enum MutexKind { Normal, Default, Recursive, ErrorCheck, } -#[derive(Debug)] -/// Additional data that we attach with each mutex instance. -pub struct AdditionalMutexData { - /// The mutex kind, used by some mutex implementations like pthreads mutexes. - pub kind: MutexKind, - - /// The address of the mutex. - pub address: u64, +#[derive(Debug, Clone, Copy)] +struct PthreadMutex { + id: MutexId, + kind: MutexKind, } -// pthread_mutex_t is between 4 and 48 bytes, depending on the platform. -// We ignore the platform layout and store our own fields: -// - id: u32 - -fn mutex_id_offset<'tcx>(ecx: &MiriInterpCx<'tcx>) -> InterpResult<'tcx, u64> { - // When adding a new OS, make sure we also support all its static initializers in - // `mutex_kind_from_static_initializer`! +/// To ensure an initialized mutex that was moved somewhere else can be distinguished from +/// a statically initialized mutex that is used the first time, we pick some offset within +/// `pthread_mutex_t` and use it as an "initialized" flag. +fn mutex_init_offset<'tcx>(ecx: &MiriInterpCx<'tcx>) -> InterpResult<'tcx, Size> { let offset = match &*ecx.tcx.sess.target.os { "linux" | "illumos" | "solaris" | "freebsd" | "android" => 0, - // macOS stores a signature in the first bytes, so we have to move to offset 4. + // macOS stores a signature in the first bytes, so we move to offset 4. "macos" => 4, os => throw_unsup_format!("`pthread_mutex` is not supported on {os}"), }; + let offset = Size::from_bytes(offset); // Sanity-check this against PTHREAD_MUTEX_INITIALIZER (but only once): - // the id must start out as 0. - // FIXME on some platforms (e.g linux) there are more static initializers for - // recursive or error checking mutexes. We should also add thme in this sanity check. + // the `init` field must start out not equal to INIT_COOKIE. static SANITY: AtomicBool = AtomicBool::new(false); if !SANITY.swap(true, Ordering::Relaxed) { let check_static_initializer = |name| { let static_initializer = ecx.eval_path(&["libc", name]); - let id_field = static_initializer - .offset(Size::from_bytes(offset), ecx.machine.layouts.u32, ecx) - .unwrap(); - let id = ecx.read_scalar(&id_field).unwrap().to_u32().unwrap(); - assert_eq!(id, 0, "{name} is incompatible with our pthread_mutex layout: id is not 0"); + let init_field = + static_initializer.offset(offset, ecx.machine.layouts.u32, ecx).unwrap(); + let init = ecx.read_scalar(&init_field).unwrap().to_u32().unwrap(); + assert_ne!( + init, LAZY_INIT_COOKIE, + "{name} is incompatible with our initialization cookie" + ); }; check_static_initializer("PTHREAD_MUTEX_INITIALIZER"); @@ -120,42 +172,28 @@ fn mutex_create<'tcx>( ecx: &mut MiriInterpCx<'tcx>, mutex_ptr: &OpTy<'tcx>, kind: MutexKind, -) -> InterpResult<'tcx> { +) -> InterpResult<'tcx, PthreadMutex> { let mutex = ecx.deref_pointer(mutex_ptr)?; - let address = mutex.ptr().addr().bytes(); - let data = Box::new(AdditionalMutexData { address, kind }); - ecx.mutex_create(&mutex, mutex_id_offset(ecx)?, Some(data))?; - interp_ok(()) + let id = ecx.machine.sync.mutex_create(); + let data = PthreadMutex { id, kind }; + lazy_sync_init(ecx, &mutex, mutex_init_offset(ecx)?, data)?; + interp_ok(data) } /// Returns the `MutexId` of the mutex stored at `mutex_op`. /// /// `mutex_get_id` will also check if the mutex has been moved since its first use and /// return an error if it has. -fn mutex_get_id<'tcx>( - ecx: &mut MiriInterpCx<'tcx>, +fn mutex_get_data<'tcx, 'a>( + ecx: &'a mut MiriInterpCx<'tcx>, mutex_ptr: &OpTy<'tcx>, -) -> InterpResult<'tcx, MutexId> { +) -> InterpResult<'tcx, PthreadMutex> { let mutex = ecx.deref_pointer(mutex_ptr)?; - let address = mutex.ptr().addr().bytes(); - - let id = ecx.mutex_get_or_create_id(&mutex, mutex_id_offset(ecx)?, |ecx| { - // This is called if a static initializer was used and the lock has not been assigned - // an ID yet. We have to determine the mutex kind from the static initializer. + lazy_sync_get_data(ecx, &mutex, mutex_init_offset(ecx)?, "pthread_mutex_t", |ecx| { let kind = mutex_kind_from_static_initializer(ecx, &mutex)?; - - interp_ok(Some(Box::new(AdditionalMutexData { kind, address }))) - })?; - - // Check that the mutex has not been moved since last use. - let data = ecx - .mutex_get_data::<AdditionalMutexData>(id) - .expect("data should always exist for pthreads"); - if data.address != address { - throw_ub_format!("pthread_mutex_t can't be moved after first use") - } - - interp_ok(id) + let id = ecx.machine.sync.mutex_create(); + interp_ok(PthreadMutex { id, kind }) + }) } /// Returns the kind of a static initializer. @@ -163,107 +201,81 @@ fn mutex_kind_from_static_initializer<'tcx>( ecx: &MiriInterpCx<'tcx>, mutex: &MPlaceTy<'tcx>, ) -> InterpResult<'tcx, MutexKind> { - interp_ok(match &*ecx.tcx.sess.target.os { - // Only linux has static initializers other than PTHREAD_MUTEX_DEFAULT. - "linux" => { - let offset = if ecx.pointer_size().bytes() == 8 { 16 } else { 12 }; - let kind_place = - mutex.offset(Size::from_bytes(offset), ecx.machine.layouts.i32, ecx)?; - let kind = ecx.read_scalar(&kind_place)?.to_i32()?; - // Here we give PTHREAD_MUTEX_DEFAULT priority so that - // PTHREAD_MUTEX_INITIALIZER behaves like `pthread_mutex_init` with a NULL argument. - if kind == ecx.eval_libc_i32("PTHREAD_MUTEX_DEFAULT") { - MutexKind::Default - } else { - mutex_translate_kind(ecx, kind)? - } - } - _ => MutexKind::Default, - }) -} + // All the static initializers recognized here *must* be checked in `mutex_init_offset`! + let is_initializer = + |name| bytewise_equal_atomic_relaxed(ecx, mutex, &ecx.eval_path(&["libc", name])); -fn mutex_translate_kind<'tcx>( - ecx: &MiriInterpCx<'tcx>, - kind: i32, -) -> InterpResult<'tcx, MutexKind> { - interp_ok(if kind == (ecx.eval_libc_i32("PTHREAD_MUTEX_NORMAL")) { - MutexKind::Normal - } else if kind == ecx.eval_libc_i32("PTHREAD_MUTEX_ERRORCHECK") { - MutexKind::ErrorCheck - } else if kind == ecx.eval_libc_i32("PTHREAD_MUTEX_RECURSIVE") { - MutexKind::Recursive - } else if kind == ecx.eval_libc_i32("PTHREAD_MUTEX_DEFAULT") - || kind == PTHREAD_MUTEX_KIND_UNCHANGED - { - // We check this *last* since PTHREAD_MUTEX_DEFAULT may be numerically equal to one of the - // others, and we want an explicit `mutexattr_settype` to work as expected. - MutexKind::Default - } else { - throw_unsup_format!("unsupported type of mutex: {kind}"); - }) + // PTHREAD_MUTEX_INITIALIZER is recognized on all targets. + if is_initializer("PTHREAD_MUTEX_INITIALIZER")? { + return interp_ok(MutexKind::Default); + } + // Support additional platform-specific initializers. + match &*ecx.tcx.sess.target.os { + "linux" => + if is_initializer("PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP")? { + return interp_ok(MutexKind::Recursive); + } else if is_initializer("PTHREAD_ERRORCHECK_MUTEX_INITIALIZER_NP")? { + return interp_ok(MutexKind::ErrorCheck); + }, + _ => {} + } + throw_unsup_format!("unsupported static initializer used for `pthread_mutex_t`"); } -// pthread_rwlock_t is between 4 and 56 bytes, depending on the platform. -// We ignore the platform layout and store our own fields: -// - id: u32 +// # pthread_rwlock_t +// We store some data directly inside the type, ignoring the platform layout: +// - init: u32 -#[derive(Debug)] -/// Additional data that we attach with each rwlock instance. -pub struct AdditionalRwLockData { - /// The address of the rwlock. - pub address: u64, +#[derive(Debug, Copy, Clone)] +struct PthreadRwLock { + id: RwLockId, } -fn rwlock_id_offset<'tcx>(ecx: &MiriInterpCx<'tcx>) -> InterpResult<'tcx, u64> { +fn rwlock_init_offset<'tcx>(ecx: &MiriInterpCx<'tcx>) -> InterpResult<'tcx, Size> { let offset = match &*ecx.tcx.sess.target.os { "linux" | "illumos" | "solaris" | "freebsd" | "android" => 0, - // macOS stores a signature in the first bytes, so we have to move to offset 4. + // macOS stores a signature in the first bytes, so we move to offset 4. "macos" => 4, os => throw_unsup_format!("`pthread_rwlock` is not supported on {os}"), }; + let offset = Size::from_bytes(offset); // Sanity-check this against PTHREAD_RWLOCK_INITIALIZER (but only once): - // the id must start out as 0. + // the `init` field must start out not equal to LAZY_INIT_COOKIE. static SANITY: AtomicBool = AtomicBool::new(false); if !SANITY.swap(true, Ordering::Relaxed) { let static_initializer = ecx.eval_path(&["libc", "PTHREAD_RWLOCK_INITIALIZER"]); - let id_field = static_initializer - .offset(Size::from_bytes(offset), ecx.machine.layouts.u32, ecx) - .unwrap(); - let id = ecx.read_scalar(&id_field).unwrap().to_u32().unwrap(); - assert_eq!( - id, 0, - "PTHREAD_RWLOCK_INITIALIZER is incompatible with our pthread_rwlock layout: id is not 0" + let init_field = static_initializer.offset(offset, ecx.machine.layouts.u32, ecx).unwrap(); + let init = ecx.read_scalar(&init_field).unwrap().to_u32().unwrap(); + assert_ne!( + init, LAZY_INIT_COOKIE, + "PTHREAD_RWLOCK_INITIALIZER is incompatible with our initialization cookie" ); } interp_ok(offset) } -fn rwlock_get_id<'tcx>( +fn rwlock_get_data<'tcx>( ecx: &mut MiriInterpCx<'tcx>, rwlock_ptr: &OpTy<'tcx>, -) -> InterpResult<'tcx, RwLockId> { +) -> InterpResult<'tcx, PthreadRwLock> { let rwlock = ecx.deref_pointer(rwlock_ptr)?; - let address = rwlock.ptr().addr().bytes(); - - let id = ecx.rwlock_get_or_create_id(&rwlock, rwlock_id_offset(ecx)?, |_| { - interp_ok(Some(Box::new(AdditionalRwLockData { address }))) - })?; - - // Check that the rwlock has not been moved since last use. - let data = ecx - .rwlock_get_data::<AdditionalRwLockData>(id) - .expect("data should always exist for pthreads"); - if data.address != address { - throw_ub_format!("pthread_rwlock_t can't be moved after first use") - } - - interp_ok(id) + lazy_sync_get_data(ecx, &rwlock, rwlock_init_offset(ecx)?, "pthread_rwlock_t", |ecx| { + if !bytewise_equal_atomic_relaxed( + ecx, + &rwlock, + &ecx.eval_path(&["libc", "PTHREAD_RWLOCK_INITIALIZER"]), + )? { + throw_unsup_format!("unsupported static initializer used for `pthread_rwlock_t`"); + } + let id = ecx.machine.sync.rwlock_create(); + interp_ok(PthreadRwLock { id }) + }) } -// pthread_condattr_t. -// We ignore the platform layout and store our own fields: +// # pthread_condattr_t +// We store some data directly inside the type, ignoring the platform layout: // - clock: i32 #[inline] @@ -288,19 +300,6 @@ fn condattr_get_clock_id<'tcx>( .to_i32() } -fn cond_translate_clock_id<'tcx>( - ecx: &MiriInterpCx<'tcx>, - raw_id: i32, -) -> InterpResult<'tcx, ClockId> { - interp_ok(if raw_id == ecx.eval_libc_i32("CLOCK_REALTIME") { - ClockId::Realtime - } else if raw_id == ecx.eval_libc_i32("CLOCK_MONOTONIC") { - ClockId::Monotonic - } else { - throw_unsup_format!("unsupported clock id: {raw_id}"); - }) -} - fn condattr_set_clock_id<'tcx>( ecx: &mut MiriInterpCx<'tcx>, attr_ptr: &OpTy<'tcx>, @@ -315,30 +314,43 @@ fn condattr_set_clock_id<'tcx>( ) } -// pthread_cond_t can be only 4 bytes in size, depending on the platform. -// We ignore the platform layout and store our own fields: -// - id: u32 +/// Translates the clock from what is stored in pthread_condattr_t to our enum. +fn condattr_translate_clock_id<'tcx>( + ecx: &MiriInterpCx<'tcx>, + raw_id: i32, +) -> InterpResult<'tcx, ClockId> { + interp_ok(if raw_id == ecx.eval_libc_i32("CLOCK_REALTIME") { + ClockId::Realtime + } else if raw_id == ecx.eval_libc_i32("CLOCK_MONOTONIC") { + ClockId::Monotonic + } else { + throw_unsup_format!("unsupported clock id: {raw_id}"); + }) +} + +// # pthread_cond_t +// We store some data directly inside the type, ignoring the platform layout: +// - init: u32 -fn cond_id_offset<'tcx>(ecx: &MiriInterpCx<'tcx>) -> InterpResult<'tcx, u64> { +fn cond_init_offset<'tcx>(ecx: &MiriInterpCx<'tcx>) -> InterpResult<'tcx, Size> { let offset = match &*ecx.tcx.sess.target.os { "linux" | "illumos" | "solaris" | "freebsd" | "android" => 0, - // macOS stores a signature in the first bytes, so we have to move to offset 4. + // macOS stores a signature in the first bytes, so we move to offset 4. "macos" => 4, os => throw_unsup_format!("`pthread_cond` is not supported on {os}"), }; + let offset = Size::from_bytes(offset); // Sanity-check this against PTHREAD_COND_INITIALIZER (but only once): - // the id must start out as 0. + // the `init` field must start out not equal to LAZY_INIT_COOKIE. static SANITY: AtomicBool = AtomicBool::new(false); if !SANITY.swap(true, Ordering::Relaxed) { let static_initializer = ecx.eval_path(&["libc", "PTHREAD_COND_INITIALIZER"]); - let id_field = static_initializer - .offset(Size::from_bytes(offset), ecx.machine.layouts.u32, ecx) - .unwrap(); - let id = ecx.read_scalar(&id_field).unwrap().to_u32().unwrap(); - assert_eq!( - id, 0, - "PTHREAD_COND_INITIALIZER is incompatible with our pthread_cond layout: id is not 0" + let init_field = static_initializer.offset(offset, ecx.machine.layouts.u32, ecx).unwrap(); + let init = ecx.read_scalar(&init_field).unwrap().to_u32().unwrap(); + assert_ne!( + init, LAZY_INIT_COOKIE, + "PTHREAD_COND_INITIALIZER is incompatible with our initialization cookie" ); } @@ -351,36 +363,41 @@ enum ClockId { Monotonic, } -#[derive(Debug)] -/// Additional data that we attach with each cond instance. -struct AdditionalCondData { - /// The address of the cond. - address: u64, +#[derive(Debug, Copy, Clone)] +struct PthreadCondvar { + id: CondvarId, + clock: ClockId, +} - /// The clock id of the cond. - clock_id: ClockId, +fn cond_create<'tcx>( + ecx: &mut MiriInterpCx<'tcx>, + cond_ptr: &OpTy<'tcx>, + clock: ClockId, +) -> InterpResult<'tcx, PthreadCondvar> { + let cond = ecx.deref_pointer(cond_ptr)?; + let id = ecx.machine.sync.condvar_create(); + let data = PthreadCondvar { id, clock }; + lazy_sync_init(ecx, &cond, cond_init_offset(ecx)?, data)?; + interp_ok(data) } -fn cond_get_id<'tcx>( +fn cond_get_data<'tcx>( ecx: &mut MiriInterpCx<'tcx>, cond_ptr: &OpTy<'tcx>, -) -> InterpResult<'tcx, CondvarId> { +) -> InterpResult<'tcx, PthreadCondvar> { let cond = ecx.deref_pointer(cond_ptr)?; - let address = cond.ptr().addr().bytes(); - let id = ecx.condvar_get_or_create_id(&cond, cond_id_offset(ecx)?, |_ecx| { + lazy_sync_get_data(ecx, &cond, cond_init_offset(ecx)?, "pthread_cond_t", |ecx| { + if !bytewise_equal_atomic_relaxed( + ecx, + &cond, + &ecx.eval_path(&["libc", "PTHREAD_COND_INITIALIZER"]), + )? { + throw_unsup_format!("unsupported static initializer used for `pthread_cond_t`"); + } // This used the static initializer. The clock there is always CLOCK_REALTIME. - interp_ok(Some(Box::new(AdditionalCondData { address, clock_id: ClockId::Realtime }))) - })?; - - // Check that the mutex has not been moved since last use. - let data = ecx - .condvar_get_data::<AdditionalCondData>(id) - .expect("data should always exist for pthreads"); - if data.address != address { - throw_ub_format!("pthread_cond_t can't be moved after first use") - } - - interp_ok(id) + let id = ecx.machine.sync.condvar_create(); + interp_ok(PthreadCondvar { id, clock: ClockId::Realtime }) + }) } impl<'tcx> EvalContextExt<'tcx> for crate::MiriInterpCx<'tcx> {} @@ -453,7 +470,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { let kind = if this.ptr_is_null(attr)? { MutexKind::Default } else { - mutex_translate_kind(this, mutexattr_get_kind(this, attr_op)?)? + mutexattr_translate_kind(this, mutexattr_get_kind(this, attr_op)?)? }; mutex_create(this, mutex_op, kind)?; @@ -468,20 +485,16 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { ) -> InterpResult<'tcx> { let this = self.eval_context_mut(); - let id = mutex_get_id(this, mutex_op)?; - let kind = this - .mutex_get_data::<AdditionalMutexData>(id) - .expect("data should always exist for pthread mutexes") - .kind; + let mutex = mutex_get_data(this, mutex_op)?; - let ret = if this.mutex_is_locked(id) { - let owner_thread = this.mutex_get_owner(id); + let ret = if this.mutex_is_locked(mutex.id) { + let owner_thread = this.mutex_get_owner(mutex.id); if owner_thread != this.active_thread() { - this.mutex_enqueue_and_block(id, Some((Scalar::from_i32(0), dest.clone()))); + this.mutex_enqueue_and_block(mutex.id, Some((Scalar::from_i32(0), dest.clone()))); return interp_ok(()); } else { // Trying to acquire the same mutex again. - match kind { + match mutex.kind { MutexKind::Default => throw_ub_format!( "trying to acquire default mutex already locked by the current thread" @@ -489,14 +502,14 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { MutexKind::Normal => throw_machine_stop!(TerminationInfo::Deadlock), MutexKind::ErrorCheck => this.eval_libc_i32("EDEADLK"), MutexKind::Recursive => { - this.mutex_lock(id); + this.mutex_lock(mutex.id); 0 } } } } else { // The mutex is unlocked. Let's lock it. - this.mutex_lock(id); + this.mutex_lock(mutex.id); 0 }; this.write_scalar(Scalar::from_i32(ret), dest)?; @@ -506,29 +519,25 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { fn pthread_mutex_trylock(&mut self, mutex_op: &OpTy<'tcx>) -> InterpResult<'tcx, Scalar> { let this = self.eval_context_mut(); - let id = mutex_get_id(this, mutex_op)?; - let kind = this - .mutex_get_data::<AdditionalMutexData>(id) - .expect("data should always exist for pthread mutexes") - .kind; + let mutex = mutex_get_data(this, mutex_op)?; - interp_ok(Scalar::from_i32(if this.mutex_is_locked(id) { - let owner_thread = this.mutex_get_owner(id); + interp_ok(Scalar::from_i32(if this.mutex_is_locked(mutex.id) { + let owner_thread = this.mutex_get_owner(mutex.id); if owner_thread != this.active_thread() { this.eval_libc_i32("EBUSY") } else { - match kind { + match mutex.kind { MutexKind::Default | MutexKind::Normal | MutexKind::ErrorCheck => this.eval_libc_i32("EBUSY"), MutexKind::Recursive => { - this.mutex_lock(id); + this.mutex_lock(mutex.id); 0 } } } } else { // The mutex is unlocked. Let's lock it. - this.mutex_lock(id); + this.mutex_lock(mutex.id); 0 })) } @@ -536,20 +545,16 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { fn pthread_mutex_unlock(&mut self, mutex_op: &OpTy<'tcx>) -> InterpResult<'tcx, Scalar> { let this = self.eval_context_mut(); - let id = mutex_get_id(this, mutex_op)?; - let kind = this - .mutex_get_data::<AdditionalMutexData>(id) - .expect("data should always exist for pthread mutexes") - .kind; + let mutex = mutex_get_data(this, mutex_op)?; - if let Some(_old_locked_count) = this.mutex_unlock(id)? { + if let Some(_old_locked_count) = this.mutex_unlock(mutex.id)? { // The mutex was locked by the current thread. interp_ok(Scalar::from_i32(0)) } else { // The mutex was locked by another thread or not locked at all. See // the “Unlock When Not Owner” column in // https://pubs.opengroup.org/onlinepubs/9699919799/functions/pthread_mutex_unlock.html. - match kind { + match mutex.kind { MutexKind::Default => throw_ub_format!( "unlocked a default mutex that was not locked by the current thread" @@ -569,9 +574,9 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { // Reading the field also has the side-effect that we detect double-`destroy` // since we make the field unint below. - let id = mutex_get_id(this, mutex_op)?; + let mutex = mutex_get_data(this, mutex_op)?; - if this.mutex_is_locked(id) { + if this.mutex_is_locked(mutex.id) { throw_ub_format!("destroyed a locked mutex"); } @@ -591,7 +596,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { ) -> InterpResult<'tcx> { let this = self.eval_context_mut(); - let id = rwlock_get_id(this, rwlock_op)?; + let id = rwlock_get_data(this, rwlock_op)?.id; if this.rwlock_is_write_locked(id) { this.rwlock_enqueue_and_block_reader(id, Scalar::from_i32(0), dest.clone()); @@ -606,7 +611,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { fn pthread_rwlock_tryrdlock(&mut self, rwlock_op: &OpTy<'tcx>) -> InterpResult<'tcx, Scalar> { let this = self.eval_context_mut(); - let id = rwlock_get_id(this, rwlock_op)?; + let id = rwlock_get_data(this, rwlock_op)?.id; if this.rwlock_is_write_locked(id) { interp_ok(Scalar::from_i32(this.eval_libc_i32("EBUSY"))) @@ -623,7 +628,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { ) -> InterpResult<'tcx> { let this = self.eval_context_mut(); - let id = rwlock_get_id(this, rwlock_op)?; + let id = rwlock_get_data(this, rwlock_op)?.id; if this.rwlock_is_locked(id) { // Note: this will deadlock if the lock is already locked by this @@ -650,7 +655,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { fn pthread_rwlock_trywrlock(&mut self, rwlock_op: &OpTy<'tcx>) -> InterpResult<'tcx, Scalar> { let this = self.eval_context_mut(); - let id = rwlock_get_id(this, rwlock_op)?; + let id = rwlock_get_data(this, rwlock_op)?.id; if this.rwlock_is_locked(id) { interp_ok(Scalar::from_i32(this.eval_libc_i32("EBUSY"))) @@ -663,7 +668,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { fn pthread_rwlock_unlock(&mut self, rwlock_op: &OpTy<'tcx>) -> InterpResult<'tcx, ()> { let this = self.eval_context_mut(); - let id = rwlock_get_id(this, rwlock_op)?; + let id = rwlock_get_data(this, rwlock_op)?.id; #[allow(clippy::if_same_then_else)] if this.rwlock_reader_unlock(id)? || this.rwlock_writer_unlock(id)? { @@ -678,7 +683,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { // Reading the field also has the side-effect that we detect double-`destroy` // since we make the field unint below. - let id = rwlock_get_id(this, rwlock_op)?; + let id = rwlock_get_data(this, rwlock_op)?.id; if this.rwlock_is_locked(id) { throw_ub_format!("destroyed a locked rwlock"); @@ -773,29 +778,23 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { } else { condattr_get_clock_id(this, attr_op)? }; - let clock_id = cond_translate_clock_id(this, clock_id)?; + let clock_id = condattr_translate_clock_id(this, clock_id)?; - let cond = this.deref_pointer(cond_op)?; - let address = cond.ptr().addr().bytes(); - this.condvar_create( - &cond, - cond_id_offset(this)?, - Some(Box::new(AdditionalCondData { address, clock_id })), - )?; + cond_create(this, cond_op, clock_id)?; interp_ok(()) } fn pthread_cond_signal(&mut self, cond_op: &OpTy<'tcx>) -> InterpResult<'tcx, ()> { let this = self.eval_context_mut(); - let id = cond_get_id(this, cond_op)?; + let id = cond_get_data(this, cond_op)?.id; this.condvar_signal(id)?; interp_ok(()) } fn pthread_cond_broadcast(&mut self, cond_op: &OpTy<'tcx>) -> InterpResult<'tcx, ()> { let this = self.eval_context_mut(); - let id = cond_get_id(this, cond_op)?; + let id = cond_get_data(this, cond_op)?.id; while this.condvar_signal(id)? {} interp_ok(()) } @@ -808,11 +807,11 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { ) -> InterpResult<'tcx> { let this = self.eval_context_mut(); - let id = cond_get_id(this, cond_op)?; - let mutex_id = mutex_get_id(this, mutex_op)?; + let data = cond_get_data(this, cond_op)?; + let mutex_id = mutex_get_data(this, mutex_op)?.id; this.condvar_wait( - id, + data.id, mutex_id, None, // no timeout Scalar::from_i32(0), @@ -832,14 +831,10 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { ) -> InterpResult<'tcx> { let this = self.eval_context_mut(); - let id = cond_get_id(this, cond_op)?; - let mutex_id = mutex_get_id(this, mutex_op)?; + let data = cond_get_data(this, cond_op)?; + let mutex_id = mutex_get_data(this, mutex_op)?.id; // Extract the timeout. - let clock_id = this - .condvar_get_data::<AdditionalCondData>(id) - .expect("additional data should always be present for pthreads") - .clock_id; let duration = match this .read_timespec(&this.deref_pointer_as(abstime_op, this.libc_ty_layout("timespec"))?)? { @@ -850,7 +845,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { return interp_ok(()); } }; - let timeout_clock = match clock_id { + let timeout_clock = match data.clock { ClockId::Realtime => { this.check_no_isolation("`pthread_cond_timedwait` with `CLOCK_REALTIME`")?; TimeoutClock::RealTime @@ -859,7 +854,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { }; this.condvar_wait( - id, + data.id, mutex_id, Some((timeout_clock, TimeoutAnchor::Absolute, duration)), Scalar::from_i32(0), @@ -875,7 +870,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { // Reading the field also has the side-effect that we detect double-`destroy` // since we make the field unint below. - let id = cond_get_id(this, cond_op)?; + let id = cond_get_data(this, cond_op)?.id; if this.condvar_is_awaited(id) { throw_ub_format!("destroying an awaited conditional variable"); } diff --git a/src/tools/miri/src/shims/windows/sync.rs b/src/tools/miri/src/shims/windows/sync.rs index 6755c23039e..8771bb4a8ac 100644 --- a/src/tools/miri/src/shims/windows/sync.rs +++ b/src/tools/miri/src/shims/windows/sync.rs @@ -3,17 +3,33 @@ use std::time::Duration; use rustc_target::abi::Size; use crate::concurrency::init_once::InitOnceStatus; +use crate::concurrency::sync::lazy_sync_get_data; use crate::*; +#[derive(Copy, Clone)] +struct WindowsInitOnce { + id: InitOnceId, +} + impl<'tcx> EvalContextExtPriv<'tcx> for crate::MiriInterpCx<'tcx> {} trait EvalContextExtPriv<'tcx>: crate::MiriInterpCxExt<'tcx> { // Windows sync primitives are pointer sized. // We only use the first 4 bytes for the id. - fn init_once_get_id(&mut self, init_once_ptr: &OpTy<'tcx>) -> InterpResult<'tcx, InitOnceId> { + fn init_once_get_data( + &mut self, + init_once_ptr: &OpTy<'tcx>, + ) -> InterpResult<'tcx, WindowsInitOnce> { let this = self.eval_context_mut(); + let init_once = this.deref_pointer(init_once_ptr)?; - this.init_once_get_or_create_id(&init_once, 0) + let init_offset = Size::ZERO; + + lazy_sync_get_data(this, &init_once, init_offset, "INIT_ONCE", |this| { + // TODO: check that this is still all-zero. + let id = this.machine.sync.init_once_create(); + interp_ok(WindowsInitOnce { id }) + }) } /// Returns `true` if we were succssful, `false` if we would block. @@ -55,7 +71,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { ) -> InterpResult<'tcx> { let this = self.eval_context_mut(); - let id = this.init_once_get_id(init_once_op)?; + let id = this.init_once_get_data(init_once_op)?.id; let flags = this.read_scalar(flags_op)?.to_u32()?; let pending_place = this.deref_pointer(pending_op)?; let context = this.read_pointer(context_op)?; @@ -101,7 +117,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { ) -> InterpResult<'tcx, Scalar> { let this = self.eval_context_mut(); - let id = this.init_once_get_id(init_once_op)?; + let id = this.init_once_get_data(init_once_op)?.id; let flags = this.read_scalar(flags_op)?.to_u32()?; let context = this.read_pointer(context_op)?; diff --git a/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_cond_move.init.stderr b/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_cond_move.init.stderr index 6e90c490a23..9a8ddc0b523 100644 --- a/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_cond_move.init.stderr +++ b/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_cond_move.init.stderr @@ -1,8 +1,8 @@ -error: Undefined Behavior: pthread_cond_t can't be moved after first use +error: Undefined Behavior: `pthread_cond_t` can't be moved after first use --> tests/fail-dep/concurrency/libc_pthread_cond_move.rs:LL:CC | LL | libc::pthread_cond_destroy(cond2.as_mut_ptr()); - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pthread_cond_t can't be moved after first use + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `pthread_cond_t` can't be moved after first use | = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information diff --git a/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_cond_move.rs b/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_cond_move.rs index 8fd0caac751..4db904ab5e2 100644 --- a/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_cond_move.rs +++ b/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_cond_move.rs @@ -18,7 +18,7 @@ fn check() { // move pthread_cond_t let mut cond2 = cond; - libc::pthread_cond_destroy(cond2.as_mut_ptr()); //~[init] ERROR: pthread_cond_t can't be moved after first use + libc::pthread_cond_destroy(cond2.as_mut_ptr()); //~[init] ERROR: can't be moved after first use } } @@ -32,6 +32,6 @@ fn check() { // move pthread_cond_t let mut cond2 = cond; - libc::pthread_cond_destroy(&mut cond2 as *mut _); //~[static_initializer] ERROR: pthread_cond_t can't be moved after first use + libc::pthread_cond_destroy(&mut cond2 as *mut _); //~[static_initializer] ERROR: can't be moved after first use } } diff --git a/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_cond_move.static_initializer.stderr b/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_cond_move.static_initializer.stderr index ba726ac7f38..8a7c0dee127 100644 --- a/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_cond_move.static_initializer.stderr +++ b/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_cond_move.static_initializer.stderr @@ -1,8 +1,8 @@ -error: Undefined Behavior: pthread_cond_t can't be moved after first use +error: Undefined Behavior: `pthread_cond_t` can't be moved after first use --> tests/fail-dep/concurrency/libc_pthread_cond_move.rs:LL:CC | LL | libc::pthread_cond_destroy(&mut cond2 as *mut _); - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pthread_cond_t can't be moved after first use + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `pthread_cond_t` can't be moved after first use | = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information diff --git a/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_move.init.stderr b/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_move.init.stderr index 15f397d4ac2..7df8e8be580 100644 --- a/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_move.init.stderr +++ b/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_move.init.stderr @@ -1,8 +1,8 @@ -error: Undefined Behavior: pthread_mutex_t can't be moved after first use +error: Undefined Behavior: `pthread_mutex_t` can't be moved after first use --> tests/fail-dep/concurrency/libc_pthread_mutex_move.rs:LL:CC | LL | libc::pthread_mutex_lock(&mut m2 as *mut _); - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pthread_mutex_t can't be moved after first use + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `pthread_mutex_t` can't be moved after first use | = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information diff --git a/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_move.rs b/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_move.rs index c12a97a9ca1..6c1f967b2b0 100644 --- a/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_move.rs +++ b/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_move.rs @@ -12,7 +12,7 @@ fn check() { assert_eq!(libc::pthread_mutex_init(&mut m as *mut _, std::ptr::null()), 0); let mut m2 = m; // move the mutex - libc::pthread_mutex_lock(&mut m2 as *mut _); //~[init] ERROR: pthread_mutex_t can't be moved after first use + libc::pthread_mutex_lock(&mut m2 as *mut _); //~[init] ERROR: can't be moved after first use } } @@ -23,6 +23,6 @@ fn check() { libc::pthread_mutex_lock(&mut m as *mut _); let mut m2 = m; // move the mutex - libc::pthread_mutex_unlock(&mut m2 as *mut _); //~[static_initializer] ERROR: pthread_mutex_t can't be moved after first use + libc::pthread_mutex_unlock(&mut m2 as *mut _); //~[static_initializer] ERROR: can't be moved after first use } } diff --git a/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_move.static_initializer.stderr b/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_move.static_initializer.stderr index ebc253bf7a6..acc018cb4ba 100644 --- a/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_move.static_initializer.stderr +++ b/src/tools/miri/tests/fail-dep/concurrency/libc_pthread_mutex_move.static_initializer.stderr @@ -1,8 +1,8 @@ -error: Undefined Behavior: pthread_mutex_t can't be moved after first use +error: Undefined Behavior: `pthread_mutex_t` can't be moved after first use --> tests/fail-dep/concurrency/libc_pthread_mutex_move.rs:LL:CC | LL | libc::pthread_mutex_unlock(&mut m2 as *mut _); - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pthread_mutex_t can't be moved after first use + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `pthread_mutex_t` can't be moved after first use | = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information diff --git a/src/tools/miri/tests/fail-dep/concurrency/libx_pthread_rwlock_moved.rs b/src/tools/miri/tests/fail-dep/concurrency/libx_pthread_rwlock_moved.rs index 540729962a9..6af19b7df9b 100644 --- a/src/tools/miri/tests/fail-dep/concurrency/libx_pthread_rwlock_moved.rs +++ b/src/tools/miri/tests/fail-dep/concurrency/libx_pthread_rwlock_moved.rs @@ -9,6 +9,6 @@ fn main() { // Move rwlock let mut rw2 = rw; - libc::pthread_rwlock_unlock(&mut rw2 as *mut _); //~ ERROR: pthread_rwlock_t can't be moved after first use + libc::pthread_rwlock_unlock(&mut rw2 as *mut _); //~ ERROR: can't be moved after first use } } diff --git a/src/tools/miri/tests/fail-dep/concurrency/libx_pthread_rwlock_moved.stderr b/src/tools/miri/tests/fail-dep/concurrency/libx_pthread_rwlock_moved.stderr index ce08fa8159c..fbc9119f110 100644 --- a/src/tools/miri/tests/fail-dep/concurrency/libx_pthread_rwlock_moved.stderr +++ b/src/tools/miri/tests/fail-dep/concurrency/libx_pthread_rwlock_moved.stderr @@ -1,8 +1,8 @@ -error: Undefined Behavior: pthread_rwlock_t can't be moved after first use +error: Undefined Behavior: `pthread_rwlock_t` can't be moved after first use --> tests/fail-dep/concurrency/libx_pthread_rwlock_moved.rs:LL:CC | LL | libc::pthread_rwlock_unlock(&mut rw2 as *mut _); - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pthread_rwlock_t can't be moved after first use + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `pthread_rwlock_t` can't be moved after first use | = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information |