From 13aa5fbd7866a975a7f56b42aa111bb16dac4b1d Mon Sep 17 00:00:00 2001 From: Yoh Deadfall Date: Mon, 11 Nov 2024 18:20:26 +0300 Subject: Added linux_like module --- .../miri/src/shims/unix/android/foreign_items.rs | 6 +- src/tools/miri/src/shims/unix/fd.rs | 2 +- src/tools/miri/src/shims/unix/linux/epoll.rs | 663 --------------------- src/tools/miri/src/shims/unix/linux/eventfd.rs | 330 ---------- .../miri/src/shims/unix/linux/foreign_items.rs | 6 +- src/tools/miri/src/shims/unix/linux/mod.rs | 4 - src/tools/miri/src/shims/unix/linux/sync.rs | 223 ------- src/tools/miri/src/shims/unix/linux/syscall.rs | 63 -- src/tools/miri/src/shims/unix/linux_like/epoll.rs | 663 +++++++++++++++++++++ .../miri/src/shims/unix/linux_like/eventfd.rs | 330 ++++++++++ src/tools/miri/src/shims/unix/linux_like/mod.rs | 4 + src/tools/miri/src/shims/unix/linux_like/sync.rs | 223 +++++++ .../miri/src/shims/unix/linux_like/syscall.rs | 63 ++ src/tools/miri/src/shims/unix/mod.rs | 3 +- src/tools/miri/src/shims/unix/unnamed_socket.rs | 2 +- 15 files changed, 1293 insertions(+), 1292 deletions(-) delete mode 100644 src/tools/miri/src/shims/unix/linux/epoll.rs delete mode 100644 src/tools/miri/src/shims/unix/linux/eventfd.rs delete mode 100644 src/tools/miri/src/shims/unix/linux/sync.rs delete mode 100644 src/tools/miri/src/shims/unix/linux/syscall.rs create mode 100644 src/tools/miri/src/shims/unix/linux_like/epoll.rs create mode 100644 src/tools/miri/src/shims/unix/linux_like/eventfd.rs create mode 100644 src/tools/miri/src/shims/unix/linux_like/mod.rs create mode 100644 src/tools/miri/src/shims/unix/linux_like/sync.rs create mode 100644 src/tools/miri/src/shims/unix/linux_like/syscall.rs (limited to 'src') diff --git a/src/tools/miri/src/shims/unix/android/foreign_items.rs b/src/tools/miri/src/shims/unix/android/foreign_items.rs index 4a005093b27..f9003885450 100644 --- a/src/tools/miri/src/shims/unix/android/foreign_items.rs +++ b/src/tools/miri/src/shims/unix/android/foreign_items.rs @@ -1,10 +1,10 @@ use rustc_abi::ExternAbi; use rustc_span::Symbol; -use self::shims::unix::linux::epoll::EvalContextExt as _; -use self::shims::unix::linux::eventfd::EvalContextExt as _; use crate::shims::unix::android::thread::prctl; -use crate::shims::unix::linux::syscall::syscall; +use crate::shims::unix::linux_like::epoll::EvalContextExt as _; +use crate::shims::unix::linux_like::eventfd::EvalContextExt as _; +use crate::shims::unix::linux_like::syscall::syscall; use crate::*; pub fn is_dyn_sym(_name: &str) -> bool { diff --git a/src/tools/miri/src/shims/unix/fd.rs b/src/tools/miri/src/shims/unix/fd.rs index 27bdd508f77..4e80630e674 100644 --- a/src/tools/miri/src/shims/unix/fd.rs +++ b/src/tools/miri/src/shims/unix/fd.rs @@ -10,7 +10,7 @@ use std::rc::{Rc, Weak}; use rustc_abi::Size; use crate::helpers::check_min_arg_count; -use crate::shims::unix::linux::epoll::EpollReadyEvents; +use crate::shims::unix::linux_like::epoll::EpollReadyEvents; use crate::shims::unix::*; use crate::*; diff --git a/src/tools/miri/src/shims/unix/linux/epoll.rs b/src/tools/miri/src/shims/unix/linux/epoll.rs deleted file mode 100644 index b20b12528db..00000000000 --- a/src/tools/miri/src/shims/unix/linux/epoll.rs +++ /dev/null @@ -1,663 +0,0 @@ -use std::cell::RefCell; -use std::collections::BTreeMap; -use std::io; -use std::rc::{Rc, Weak}; -use std::time::Duration; - -use crate::concurrency::VClock; -use crate::shims::unix::fd::{FdId, FileDescriptionRef, WeakFileDescriptionRef}; -use crate::shims::unix::*; -use crate::*; - -/// An `Epoll` file descriptor connects file handles and epoll events -#[derive(Clone, Debug, Default)] -struct Epoll { - /// A map of EpollEventInterests registered under this epoll instance. - /// Each entry is differentiated using FdId and file descriptor value. - interest_list: RefCell>>>, - /// A map of EpollEventInstance that will be returned when `epoll_wait` is called. - /// Similar to interest_list, the entry is also differentiated using FdId - /// and file descriptor value. - // This is an Rc because EpollInterest need to hold a reference to update - // it. - ready_list: Rc, - /// A list of thread ids blocked on this epoll instance. - thread_id: RefCell>, -} - -/// EpollEventInstance contains information that will be returned by epoll_wait. -#[derive(Debug)] -pub struct EpollEventInstance { - /// Xor-ed event types that happened to the file description. - events: u32, - /// Original data retrieved from `epoll_event` during `epoll_ctl`. - data: u64, - /// The release clock associated with this event. - clock: VClock, -} - -impl EpollEventInstance { - pub fn new(events: u32, data: u64) -> EpollEventInstance { - EpollEventInstance { events, data, clock: Default::default() } - } -} - -/// EpollEventInterest registers the file description information to an epoll -/// instance during a successful `epoll_ctl` call. It also stores additional -/// information needed to check and update readiness state for `epoll_wait`. -/// -/// `events` and `data` field matches the `epoll_event` struct defined -/// by the epoll_ctl man page. For more information -/// see the man page: -/// -/// -#[derive(Clone, Debug)] -pub struct EpollEventInterest { - /// The file descriptor value of the file description registered. - /// This is only used for ready_list, to inform userspace which FD triggered an event. - /// For that, it is crucial to preserve the original FD number. - /// This FD number must never be "dereferenced" to a file description inside Miri. - fd_num: i32, - /// The events bitmask retrieved from `epoll_event`. - events: u32, - /// The data retrieved from `epoll_event`. - /// libc's data field in epoll_event can store integer or pointer, - /// but only u64 is supported for now. - /// - data: u64, - /// Ready list of the epoll instance under which this EpollEventInterest is registered. - ready_list: Rc, - /// The epoll file description that this EpollEventInterest is registered under. - weak_epfd: WeakFileDescriptionRef, -} - -/// EpollReadyEvents reflects the readiness of a file description. -pub struct EpollReadyEvents { - /// The associated file is available for read(2) operations, in the sense that a read will not block. - /// (I.e., returning EOF is considered "ready".) - pub epollin: bool, - /// The associated file is available for write(2) operations, in the sense that a write will not block. - pub epollout: bool, - /// Stream socket peer closed connection, or shut down writing - /// half of connection. - pub epollrdhup: bool, - /// For stream socket, this event merely indicates that the peer - /// closed its end of the channel. - /// Unlike epollrdhup, this should only be set when the stream is fully closed. - /// epollrdhup also gets set when only the write half is closed, which is possible - /// via `shutdown(_, SHUT_WR)`. - pub epollhup: bool, - /// Error condition happened on the associated file descriptor. - pub epollerr: bool, -} - -#[derive(Debug, Default)] -struct ReadyList { - mapping: RefCell>, -} - -impl EpollReadyEvents { - pub fn new() -> Self { - EpollReadyEvents { - epollin: false, - epollout: false, - epollrdhup: false, - epollhup: false, - epollerr: false, - } - } - - pub fn get_event_bitmask<'tcx>(&self, ecx: &MiriInterpCx<'tcx>) -> u32 { - let epollin = ecx.eval_libc_u32("EPOLLIN"); - let epollout = ecx.eval_libc_u32("EPOLLOUT"); - let epollrdhup = ecx.eval_libc_u32("EPOLLRDHUP"); - let epollhup = ecx.eval_libc_u32("EPOLLHUP"); - let epollerr = ecx.eval_libc_u32("EPOLLERR"); - - let mut bitmask = 0; - if self.epollin { - bitmask |= epollin; - } - if self.epollout { - bitmask |= epollout; - } - if self.epollrdhup { - bitmask |= epollrdhup; - } - if self.epollhup { - bitmask |= epollhup; - } - if self.epollerr { - bitmask |= epollerr; - } - bitmask - } -} - -impl Epoll { - fn get_ready_list(&self) -> Rc { - Rc::clone(&self.ready_list) - } -} - -impl FileDescription for Epoll { - fn name(&self) -> &'static str { - "epoll" - } - - fn close<'tcx>( - self: Box, - _communicate_allowed: bool, - _ecx: &mut MiriInterpCx<'tcx>, - ) -> InterpResult<'tcx, io::Result<()>> { - interp_ok(Ok(())) - } -} - -/// The table of all EpollEventInterest. -/// The BTreeMap key is the FdId of an active file description registered with -/// any epoll instance. The value is a list of EpollEventInterest associated -/// with that file description. -pub struct EpollInterestTable(BTreeMap>>>); - -impl EpollInterestTable { - pub(crate) fn new() -> Self { - EpollInterestTable(BTreeMap::new()) - } - - pub fn insert_epoll_interest(&mut self, id: FdId, fd: Weak>) { - match self.0.get_mut(&id) { - Some(fds) => { - fds.push(fd); - } - None => { - let vec = vec![fd]; - self.0.insert(id, vec); - } - } - } - - pub fn get_epoll_interest(&self, id: FdId) -> Option<&Vec>>> { - self.0.get(&id) - } - - pub fn get_epoll_interest_mut( - &mut self, - id: FdId, - ) -> Option<&mut Vec>>> { - self.0.get_mut(&id) - } - - pub fn remove(&mut self, id: FdId) { - self.0.remove(&id); - } -} - -impl<'tcx> EvalContextExt<'tcx> for crate::MiriInterpCx<'tcx> {} -pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { - /// This function returns a file descriptor referring to the new `Epoll` instance. This file - /// descriptor is used for all subsequent calls to the epoll interface. If the `flags` argument - /// is 0, then this function is the same as `epoll_create()`. - /// - /// - fn epoll_create1(&mut self, flags: &OpTy<'tcx>) -> InterpResult<'tcx, Scalar> { - let this = self.eval_context_mut(); - - let flags = this.read_scalar(flags)?.to_i32()?; - - let epoll_cloexec = this.eval_libc_i32("EPOLL_CLOEXEC"); - - // Miri does not support exec, so EPOLL_CLOEXEC flag has no effect. - if flags != epoll_cloexec && flags != 0 { - throw_unsup_format!( - "epoll_create1: flag {:#x} is unsupported, only 0 or EPOLL_CLOEXEC are allowed", - flags - ); - } - - let fd = this.machine.fds.insert_new(Epoll::default()); - interp_ok(Scalar::from_i32(fd)) - } - - /// This function performs control operations on the `Epoll` instance referred to by the file - /// descriptor `epfd`. It requests that the operation `op` be performed for the target file - /// descriptor, `fd`. - /// - /// Valid values for the op argument are: - /// `EPOLL_CTL_ADD` - Register the target file descriptor `fd` on the `Epoll` instance referred - /// to by the file descriptor `epfd` and associate the event `event` with the internal file - /// linked to `fd`. - /// `EPOLL_CTL_MOD` - Change the event `event` associated with the target file descriptor `fd`. - /// `EPOLL_CTL_DEL` - Deregister the target file descriptor `fd` from the `Epoll` instance - /// referred to by `epfd`. The `event` is ignored and can be null. - /// - /// - fn epoll_ctl( - &mut self, - epfd: &OpTy<'tcx>, - op: &OpTy<'tcx>, - fd: &OpTy<'tcx>, - event: &OpTy<'tcx>, - ) -> InterpResult<'tcx, Scalar> { - let this = self.eval_context_mut(); - - let epfd_value = this.read_scalar(epfd)?.to_i32()?; - let op = this.read_scalar(op)?.to_i32()?; - let fd = this.read_scalar(fd)?.to_i32()?; - let event = this.deref_pointer_as(event, this.libc_ty_layout("epoll_event"))?; - - let epoll_ctl_add = this.eval_libc_i32("EPOLL_CTL_ADD"); - let epoll_ctl_mod = this.eval_libc_i32("EPOLL_CTL_MOD"); - let epoll_ctl_del = this.eval_libc_i32("EPOLL_CTL_DEL"); - let epollin = this.eval_libc_u32("EPOLLIN"); - let epollout = this.eval_libc_u32("EPOLLOUT"); - let epollrdhup = this.eval_libc_u32("EPOLLRDHUP"); - let epollet = this.eval_libc_u32("EPOLLET"); - let epollhup = this.eval_libc_u32("EPOLLHUP"); - let epollerr = this.eval_libc_u32("EPOLLERR"); - - // Throw EINVAL if epfd and fd have the same value. - if epfd_value == fd { - return this.set_last_error_and_return_i32(LibcError("EINVAL")); - } - - // Check if epfd is a valid epoll file descriptor. - let Some(epfd) = this.machine.fds.get(epfd_value) else { - return this.set_last_error_and_return_i32(LibcError("EBADF")); - }; - let epoll_file_description = epfd - .downcast::() - .ok_or_else(|| err_unsup_format!("non-epoll FD passed to `epoll_ctl`"))?; - - let mut interest_list = epoll_file_description.interest_list.borrow_mut(); - let ready_list = &epoll_file_description.ready_list; - - let Some(fd_ref) = this.machine.fds.get(fd) else { - return this.set_last_error_and_return_i32(LibcError("EBADF")); - }; - let id = fd_ref.get_id(); - - if op == epoll_ctl_add || op == epoll_ctl_mod { - // Read event bitmask and data from epoll_event passed by caller. - let mut events = this.read_scalar(&this.project_field(&event, 0)?)?.to_u32()?; - let data = this.read_scalar(&this.project_field(&event, 1)?)?.to_u64()?; - - // Unset the flag we support to discover if any unsupported flags are used. - let mut flags = events; - // epoll_wait(2) will always wait for epollhup and epollerr; it is not - // necessary to set it in events when calling epoll_ctl(). - // So we will always set these two event types. - events |= epollhup; - events |= epollerr; - - if events & epollet != epollet { - // We only support edge-triggered notification for now. - throw_unsup_format!("epoll_ctl: epollet flag must be included."); - } else { - flags &= !epollet; - } - if flags & epollin == epollin { - flags &= !epollin; - } - if flags & epollout == epollout { - flags &= !epollout; - } - if flags & epollrdhup == epollrdhup { - flags &= !epollrdhup; - } - if flags & epollhup == epollhup { - flags &= !epollhup; - } - if flags & epollerr == epollerr { - flags &= !epollerr; - } - if flags != 0 { - throw_unsup_format!( - "epoll_ctl: encountered unknown unsupported flags {:#x}", - flags - ); - } - - let epoll_key = (id, fd); - - // Check the existence of fd in the interest list. - if op == epoll_ctl_add { - if interest_list.contains_key(&epoll_key) { - return this.set_last_error_and_return_i32(LibcError("EEXIST")); - } - } else { - if !interest_list.contains_key(&epoll_key) { - return this.set_last_error_and_return_i32(LibcError("ENOENT")); - } - } - - // Create an epoll_interest. - let interest = Rc::new(RefCell::new(EpollEventInterest { - fd_num: fd, - events, - data, - ready_list: Rc::clone(ready_list), - weak_epfd: epfd.downgrade(), - })); - - if op == epoll_ctl_add { - // Insert an epoll_interest to global epoll_interest list. - this.machine.epoll_interests.insert_epoll_interest(id, Rc::downgrade(&interest)); - interest_list.insert(epoll_key, Rc::clone(&interest)); - } else { - // Directly modify the epoll_interest so the global epoll_event_interest table - // will be updated too. - let mut epoll_interest = interest_list.get_mut(&epoll_key).unwrap().borrow_mut(); - epoll_interest.events = events; - epoll_interest.data = data; - } - - // Notification will be returned for current epfd if there is event in the file - // descriptor we registered. - check_and_update_one_event_interest(&fd_ref, interest, id, this)?; - interp_ok(Scalar::from_i32(0)) - } else if op == epoll_ctl_del { - let epoll_key = (id, fd); - - // Remove epoll_event_interest from interest_list. - let Some(epoll_interest) = interest_list.remove(&epoll_key) else { - return this.set_last_error_and_return_i32(LibcError("ENOENT")); - }; - // All related Weak will fail to upgrade after the drop. - drop(epoll_interest); - - // Remove related epoll_interest from ready list. - ready_list.mapping.borrow_mut().remove(&epoll_key); - - // Remove dangling EpollEventInterest from its global table. - // .unwrap() below should succeed because the file description id must have registered - // at least one epoll_interest, if not, it will fail when removing epoll_interest from - // interest list. - this.machine - .epoll_interests - .get_epoll_interest_mut(id) - .unwrap() - .retain(|event| event.upgrade().is_some()); - - interp_ok(Scalar::from_i32(0)) - } else { - throw_unsup_format!("unsupported epoll_ctl operation: {op}"); - } - } - - /// The `epoll_wait()` system call waits for events on the `Epoll` - /// instance referred to by the file descriptor `epfd`. The buffer - /// pointed to by `events` is used to return information from the ready - /// list about file descriptors in the interest list that have some - /// events available. Up to `maxevents` are returned by `epoll_wait()`. - /// The `maxevents` argument must be greater than zero. - /// - /// The `timeout` argument specifies the number of milliseconds that - /// `epoll_wait()` will block. Time is measured against the - /// CLOCK_MONOTONIC clock. If the timeout is zero, the function will not block, - /// while if the timeout is -1, the function will block - /// until at least one event has been retrieved (or an error - /// occurred). - /// - /// A call to `epoll_wait()` will block until either: - /// • a file descriptor delivers an event; - /// • the call is interrupted by a signal handler; or - /// • the timeout expires. - /// - /// Note that the timeout interval will be rounded up to the system - /// clock granularity, and kernel scheduling delays mean that the - /// blocking interval may overrun by a small amount. Specifying a - /// timeout of -1 causes `epoll_wait()` to block indefinitely, while - /// specifying a timeout equal to zero cause `epoll_wait()` to return - /// immediately, even if no events are available. - /// - /// On success, `epoll_wait()` returns the number of file descriptors - /// ready for the requested I/O, or zero if no file descriptor became - /// ready during the requested timeout milliseconds. On failure, - /// `epoll_wait()` returns -1 and errno is set to indicate the error. - /// - /// - fn epoll_wait( - &mut self, - epfd: &OpTy<'tcx>, - events_op: &OpTy<'tcx>, - maxevents: &OpTy<'tcx>, - timeout: &OpTy<'tcx>, - dest: &MPlaceTy<'tcx>, - ) -> InterpResult<'tcx> { - let this = self.eval_context_mut(); - - let epfd_value = this.read_scalar(epfd)?.to_i32()?; - let events = this.read_immediate(events_op)?; - let maxevents = this.read_scalar(maxevents)?.to_i32()?; - let timeout = this.read_scalar(timeout)?.to_i32()?; - - if epfd_value <= 0 || maxevents <= 0 { - return this.set_last_error_and_return(LibcError("EINVAL"), dest); - } - - // This needs to come after the maxevents value check, or else maxevents.try_into().unwrap() - // will fail. - let event = this.deref_pointer_as( - &events, - this.libc_array_ty_layout("epoll_event", maxevents.try_into().unwrap()), - )?; - - let Some(epfd) = this.machine.fds.get(epfd_value) else { - return this.set_last_error_and_return(LibcError("EBADF"), dest); - }; - // Create a weak ref of epfd and pass it to callback so we will make sure that epfd - // is not close after the thread unblocks. - let weak_epfd = epfd.downgrade(); - - // We just need to know if the ready list is empty and borrow the thread_ids out. - // The whole logic is wrapped inside a block so we don't need to manually drop epfd later. - let ready_list_empty; - let mut thread_ids; - { - let epoll_file_description = epfd - .downcast::() - .ok_or_else(|| err_unsup_format!("non-epoll FD passed to `epoll_wait`"))?; - ready_list_empty = epoll_file_description.ready_list.mapping.borrow().is_empty(); - thread_ids = epoll_file_description.thread_id.borrow_mut(); - } - if timeout == 0 || !ready_list_empty { - // If the ready list is not empty, or the timeout is 0, we can return immediately. - return_ready_list(epfd_value, weak_epfd, dest, &event, this)?; - } else { - // Blocking - let timeout = match timeout { - 0.. => { - let duration = Duration::from_millis(timeout.try_into().unwrap()); - Some((TimeoutClock::Monotonic, TimeoutAnchor::Relative, duration)) - } - -1 => None, - ..-1 => { - throw_unsup_format!( - "epoll_wait: Only timeout values greater than or equal to -1 are supported." - ); - } - }; - thread_ids.push(this.active_thread()); - let dest = dest.clone(); - this.block_thread( - BlockReason::Epoll, - timeout, - callback!( - @capture<'tcx> { - epfd_value: i32, - weak_epfd: WeakFileDescriptionRef, - dest: MPlaceTy<'tcx>, - event: MPlaceTy<'tcx>, - } - @unblock = |this| { - return_ready_list(epfd_value, weak_epfd, &dest, &event, this)?; - interp_ok(()) - } - @timeout = |this| { - // No notification after blocking timeout. - let Some(epfd) = weak_epfd.upgrade() else { - throw_unsup_format!("epoll FD {epfd_value} got closed while blocking.") - }; - // Remove the current active thread_id from the blocked thread_id list. - epfd.downcast::() - .ok_or_else(|| err_unsup_format!("non-epoll FD passed to `epoll_wait`"))? - .thread_id.borrow_mut() - .retain(|&id| id != this.active_thread()); - this.write_int(0, &dest)?; - interp_ok(()) - } - ), - ); - } - interp_ok(()) - } - - /// For a specific file description, get its ready events and update the corresponding ready - /// list. This function should be called whenever an event causes more bytes or an EOF to become - /// newly readable from an FD, and whenever more bytes can be written to an FD or no more future - /// writes are possible. - /// - /// This *will* report an event if anyone is subscribed to it, without any further filtering, so - /// do not call this function when an FD didn't have anything happen to it! - fn check_and_update_readiness( - &mut self, - fd_ref: &FileDescriptionRef, - ) -> InterpResult<'tcx, ()> { - let this = self.eval_context_mut(); - let id = fd_ref.get_id(); - let mut waiter = Vec::new(); - // Get a list of EpollEventInterest that is associated to a specific file description. - if let Some(epoll_interests) = this.machine.epoll_interests.get_epoll_interest(id) { - for weak_epoll_interest in epoll_interests { - if let Some(epoll_interest) = weak_epoll_interest.upgrade() { - let is_updated = check_and_update_one_event_interest( - fd_ref, - epoll_interest.clone(), - id, - this, - )?; - if is_updated { - // Edge-triggered notification only notify one thread even if there are - // multiple threads blocked on the same epfd. - - // This unwrap can never fail because if the current epoll instance were - // closed, the upgrade of weak_epoll_interest - // above would fail. This guarantee holds because only the epoll instance - // holds a strong ref to epoll_interest. - let epfd = epoll_interest.borrow().weak_epfd.upgrade().unwrap(); - // FIXME: We can randomly pick a thread to unblock. - - let epoll = epfd.downcast::().unwrap(); - - if let Some(thread_id) = epoll.thread_id.borrow_mut().pop() { - waiter.push(thread_id); - }; - } - } - } - } - waiter.sort(); - waiter.dedup(); - for thread_id in waiter { - this.unblock_thread(thread_id, BlockReason::Epoll)?; - } - interp_ok(()) - } -} - -/// This function takes in ready list and returns EpollEventInstance with file description -/// that is not closed. -fn ready_list_next( - ecx: &MiriInterpCx<'_>, - ready_list: &mut BTreeMap<(FdId, i32), EpollEventInstance>, -) -> Option { - while let Some((epoll_key, epoll_event_instance)) = ready_list.pop_first() { - // This ensures that we only return events that we are interested. The FD might have been closed since - // the event was generated, in which case we are not interested anymore. - // When a file description is fully closed, it gets removed from `machine.epoll_interests`, - // so we skip events whose FD is not in that map anymore. - if ecx.machine.epoll_interests.get_epoll_interest(epoll_key.0).is_some() { - return Some(epoll_event_instance); - } - } - None -} - -/// This helper function checks whether an epoll notification should be triggered for a specific -/// epoll_interest and, if necessary, triggers the notification, and returns whether the -/// notification was added/updated. Unlike check_and_update_readiness, this function sends a -/// notification to only one epoll instance. -fn check_and_update_one_event_interest<'tcx>( - fd_ref: &FileDescriptionRef, - interest: Rc>, - id: FdId, - ecx: &MiriInterpCx<'tcx>, -) -> InterpResult<'tcx, bool> { - // Get the bitmask of ready events for a file description. - let ready_events_bitmask = fd_ref.get_epoll_ready_events()?.get_event_bitmask(ecx); - let epoll_event_interest = interest.borrow(); - // This checks if any of the events specified in epoll_event_interest.events - // match those in ready_events. - let flags = epoll_event_interest.events & ready_events_bitmask; - // If there is any event that we are interested in being specified as ready, - // insert an epoll_return to the ready list. - if flags != 0 { - let epoll_key = (id, epoll_event_interest.fd_num); - let ready_list = &mut epoll_event_interest.ready_list.mapping.borrow_mut(); - let mut event_instance = EpollEventInstance::new(flags, epoll_event_interest.data); - // If we are tracking data races, remember the current clock so we can sync with it later. - ecx.release_clock(|clock| { - event_instance.clock.clone_from(clock); - }); - // Triggers the notification by inserting it to the ready list. - ready_list.insert(epoll_key, event_instance); - interp_ok(true) - } else { - interp_ok(false) - } -} - -/// Stores the ready list of the `epfd` epoll instance into `events` (which must be an array), -/// and the number of returned events into `dest`. -fn return_ready_list<'tcx>( - epfd_value: i32, - weak_epfd: WeakFileDescriptionRef, - dest: &MPlaceTy<'tcx>, - events: &MPlaceTy<'tcx>, - ecx: &mut MiriInterpCx<'tcx>, -) -> InterpResult<'tcx> { - let Some(epfd) = weak_epfd.upgrade() else { - throw_unsup_format!("epoll FD {epfd_value} got closed while blocking.") - }; - - let epoll_file_description = epfd - .downcast::() - .ok_or_else(|| err_unsup_format!("non-epoll FD passed to `epoll_wait`"))?; - - let ready_list = epoll_file_description.get_ready_list(); - - let mut ready_list = ready_list.mapping.borrow_mut(); - let mut num_of_events: i32 = 0; - let mut array_iter = ecx.project_array_fields(events)?; - - while let Some(des) = array_iter.next(ecx)? { - if let Some(epoll_event_instance) = ready_list_next(ecx, &mut ready_list) { - ecx.write_int_fields_named( - &[ - ("events", epoll_event_instance.events.into()), - ("u64", epoll_event_instance.data.into()), - ], - &des.1, - )?; - // Synchronize waking thread with the event of interest. - ecx.acquire_clock(&epoll_event_instance.clock); - - num_of_events = num_of_events.strict_add(1); - } else { - break; - } - } - ecx.write_int(num_of_events, dest)?; - interp_ok(()) -} diff --git a/src/tools/miri/src/shims/unix/linux/eventfd.rs b/src/tools/miri/src/shims/unix/linux/eventfd.rs deleted file mode 100644 index 6aee9c1d86b..00000000000 --- a/src/tools/miri/src/shims/unix/linux/eventfd.rs +++ /dev/null @@ -1,330 +0,0 @@ -//! Linux `eventfd` implementation. -use std::cell::{Cell, RefCell}; -use std::io; -use std::io::ErrorKind; - -use crate::concurrency::VClock; -use crate::shims::unix::fd::{FileDescriptionRef, WeakFileDescriptionRef}; -use crate::shims::unix::linux::epoll::{EpollReadyEvents, EvalContextExt as _}; -use crate::shims::unix::*; -use crate::*; - -/// Maximum value that the eventfd counter can hold. -const MAX_COUNTER: u64 = u64::MAX - 1; - -/// A kind of file descriptor created by `eventfd`. -/// The `Event` type isn't currently written to by `eventfd`. -/// The interface is meant to keep track of objects associated -/// with a file descriptor. For more information see the man -/// page below: -/// -/// -#[derive(Debug)] -struct Event { - /// The object contains an unsigned 64-bit integer (uint64_t) counter that is maintained by the - /// kernel. This counter is initialized with the value specified in the argument initval. - counter: Cell, - is_nonblock: bool, - clock: RefCell, - /// A list of thread ids blocked on eventfd::read. - blocked_read_tid: RefCell>, - /// A list of thread ids blocked on eventfd::write. - blocked_write_tid: RefCell>, -} - -impl FileDescription for Event { - fn name(&self) -> &'static str { - "event" - } - - fn get_epoll_ready_events<'tcx>(&self) -> InterpResult<'tcx, EpollReadyEvents> { - // We only check the status of EPOLLIN and EPOLLOUT flags for eventfd. If other event flags - // need to be supported in the future, the check should be added here. - - interp_ok(EpollReadyEvents { - epollin: self.counter.get() != 0, - epollout: self.counter.get() != MAX_COUNTER, - ..EpollReadyEvents::new() - }) - } - - fn close<'tcx>( - self: Box, - _communicate_allowed: bool, - _ecx: &mut MiriInterpCx<'tcx>, - ) -> InterpResult<'tcx, io::Result<()>> { - interp_ok(Ok(())) - } - - /// Read the counter in the buffer and return the counter if succeeded. - fn read<'tcx>( - &self, - self_ref: &FileDescriptionRef, - _communicate_allowed: bool, - ptr: Pointer, - len: usize, - dest: &MPlaceTy<'tcx>, - ecx: &mut MiriInterpCx<'tcx>, - ) -> InterpResult<'tcx> { - // We're treating the buffer as a `u64`. - let ty = ecx.machine.layouts.u64; - // Check the size of slice, and return error only if the size of the slice < 8. - if len < ty.size.bytes_usize() { - return ecx.set_last_error_and_return(ErrorKind::InvalidInput, dest); - } - - // eventfd read at the size of u64. - let buf_place = ecx.ptr_to_mplace_unaligned(ptr, ty); - - let weak_eventfd = self_ref.downgrade(); - eventfd_read(buf_place, dest, weak_eventfd, ecx) - } - - /// A write call adds the 8-byte integer value supplied in - /// its buffer (in native endianness) to the counter. The maximum value that may be - /// stored in the counter is the largest unsigned 64-bit value - /// minus 1 (i.e., 0xfffffffffffffffe). If the addition would - /// cause the counter's value to exceed the maximum, then the - /// write either blocks until a read is performed on the - /// file descriptor, or fails with the error EAGAIN if the - /// file descriptor has been made nonblocking. - /// - /// A write fails with the error EINVAL if the size of the - /// supplied buffer is less than 8 bytes, or if an attempt is - /// made to write the value 0xffffffffffffffff. - fn write<'tcx>( - &self, - self_ref: &FileDescriptionRef, - _communicate_allowed: bool, - ptr: Pointer, - len: usize, - dest: &MPlaceTy<'tcx>, - ecx: &mut MiriInterpCx<'tcx>, - ) -> InterpResult<'tcx> { - // We're treating the buffer as a `u64`. - let ty = ecx.machine.layouts.u64; - // Check the size of slice, and return error only if the size of the slice < 8. - if len < ty.layout.size.bytes_usize() { - return ecx.set_last_error_and_return(ErrorKind::InvalidInput, dest); - } - - // Read the user-supplied value from the pointer. - let buf_place = ecx.ptr_to_mplace_unaligned(ptr, ty); - let num = ecx.read_scalar(&buf_place)?.to_u64()?; - - // u64::MAX as input is invalid because the maximum value of counter is u64::MAX - 1. - if num == u64::MAX { - return ecx.set_last_error_and_return(ErrorKind::InvalidInput, dest); - } - // If the addition does not let the counter to exceed the maximum value, update the counter. - // Else, block. - let weak_eventfd = self_ref.downgrade(); - eventfd_write(num, buf_place, dest, weak_eventfd, ecx) - } -} - -impl<'tcx> EvalContextExt<'tcx> for crate::MiriInterpCx<'tcx> {} -pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { - /// This function creates an `Event` that is used as an event wait/notify mechanism by - /// user-space applications, and by the kernel to notify user-space applications of events. - /// The `Event` contains an `u64` counter maintained by the kernel. The counter is initialized - /// with the value specified in the `initval` argument. - /// - /// A new file descriptor referring to the `Event` is returned. The `read`, `write`, `poll`, - /// `select`, and `close` operations can be performed on the file descriptor. For more - /// information on these operations, see the man page linked below. - /// - /// The `flags` are not currently implemented for eventfd. - /// The `flags` may be bitwise ORed to change the behavior of `eventfd`: - /// `EFD_CLOEXEC` - Set the close-on-exec (`FD_CLOEXEC`) flag on the new file descriptor. - /// `EFD_NONBLOCK` - Set the `O_NONBLOCK` file status flag on the new open file description. - /// `EFD_SEMAPHORE` - miri does not support semaphore-like semantics. - /// - /// - fn eventfd(&mut self, val: &OpTy<'tcx>, flags: &OpTy<'tcx>) -> InterpResult<'tcx, Scalar> { - let this = self.eval_context_mut(); - - let val = this.read_scalar(val)?.to_u32()?; - let mut flags = this.read_scalar(flags)?.to_i32()?; - - let efd_cloexec = this.eval_libc_i32("EFD_CLOEXEC"); - let efd_nonblock = this.eval_libc_i32("EFD_NONBLOCK"); - let efd_semaphore = this.eval_libc_i32("EFD_SEMAPHORE"); - - if flags & efd_semaphore == efd_semaphore { - throw_unsup_format!("eventfd: EFD_SEMAPHORE is unsupported"); - } - - let mut is_nonblock = false; - // Unset the flag that we support. - // After unloading, flags != 0 means other flags are used. - if flags & efd_cloexec == efd_cloexec { - // cloexec is ignored because Miri does not support exec. - flags &= !efd_cloexec; - } - if flags & efd_nonblock == efd_nonblock { - flags &= !efd_nonblock; - is_nonblock = true; - } - if flags != 0 { - throw_unsup_format!("eventfd: encountered unknown unsupported flags {:#x}", flags); - } - - let fds = &mut this.machine.fds; - - let fd_value = fds.insert_new(Event { - counter: Cell::new(val.into()), - is_nonblock, - clock: RefCell::new(VClock::default()), - blocked_read_tid: RefCell::new(Vec::new()), - blocked_write_tid: RefCell::new(Vec::new()), - }); - - interp_ok(Scalar::from_i32(fd_value)) - } -} - -/// Block thread if the value addition will exceed u64::MAX -1, -/// else just add the user-supplied value to current counter. -fn eventfd_write<'tcx>( - num: u64, - buf_place: MPlaceTy<'tcx>, - dest: &MPlaceTy<'tcx>, - weak_eventfd: WeakFileDescriptionRef, - ecx: &mut MiriInterpCx<'tcx>, -) -> InterpResult<'tcx> { - let Some(eventfd_ref) = weak_eventfd.upgrade() else { - throw_unsup_format!("eventfd FD got closed while blocking.") - }; - - // Since we pass the weak file description ref, it is guaranteed to be - // an eventfd file description. - let eventfd = eventfd_ref.downcast::().unwrap(); - - match eventfd.counter.get().checked_add(num) { - Some(new_count @ 0..=MAX_COUNTER) => { - // Future `read` calls will synchronize with this write, so update the FD clock. - ecx.release_clock(|clock| { - eventfd.clock.borrow_mut().join(clock); - }); - - // Store new counter value. - eventfd.counter.set(new_count); - - // The state changed; we check and update the status of all supported event - // types for current file description. - ecx.check_and_update_readiness(&eventfd_ref)?; - - // Unblock *all* threads previously blocked on `read`. - // We need to take out the blocked thread ids and unblock them together, - // because `unblock_threads` may block them again and end up re-adding the - // thread to the blocked list. - let waiting_threads = std::mem::take(&mut *eventfd.blocked_read_tid.borrow_mut()); - // FIXME: We can randomize the order of unblocking. - for thread_id in waiting_threads { - ecx.unblock_thread(thread_id, BlockReason::Eventfd)?; - } - - // Return how many bytes we consumed from the user-provided buffer. - return ecx.write_int(buf_place.layout.size.bytes(), dest); - } - None | Some(u64::MAX) => { - // We can't update the state, so we have to block. - if eventfd.is_nonblock { - return ecx.set_last_error_and_return(ErrorKind::WouldBlock, dest); - } - - let dest = dest.clone(); - - eventfd.blocked_write_tid.borrow_mut().push(ecx.active_thread()); - - ecx.block_thread( - BlockReason::Eventfd, - None, - callback!( - @capture<'tcx> { - num: u64, - buf_place: MPlaceTy<'tcx>, - dest: MPlaceTy<'tcx>, - weak_eventfd: WeakFileDescriptionRef, - } - @unblock = |this| { - // When we get unblocked, try again. - eventfd_write(num, buf_place, &dest, weak_eventfd, this) - } - ), - ); - } - }; - interp_ok(()) -} - -/// Block thread if the current counter is 0, -/// else just return the current counter value to the caller and set the counter to 0. -fn eventfd_read<'tcx>( - buf_place: MPlaceTy<'tcx>, - dest: &MPlaceTy<'tcx>, - weak_eventfd: WeakFileDescriptionRef, - ecx: &mut MiriInterpCx<'tcx>, -) -> InterpResult<'tcx> { - let Some(eventfd_ref) = weak_eventfd.upgrade() else { - throw_unsup_format!("eventfd FD got closed while blocking.") - }; - - // Since we pass the weak file description ref to the callback function, it is guaranteed to be - // an eventfd file description. - let eventfd = eventfd_ref.downcast::().unwrap(); - - // Set counter to 0, get old value. - let counter = eventfd.counter.replace(0); - - // Block when counter == 0. - if counter == 0 { - if eventfd.is_nonblock { - return ecx.set_last_error_and_return(ErrorKind::WouldBlock, dest); - } - let dest = dest.clone(); - - eventfd.blocked_read_tid.borrow_mut().push(ecx.active_thread()); - - ecx.block_thread( - BlockReason::Eventfd, - None, - callback!( - @capture<'tcx> { - buf_place: MPlaceTy<'tcx>, - dest: MPlaceTy<'tcx>, - weak_eventfd: WeakFileDescriptionRef, - } - @unblock = |this| { - // When we get unblocked, try again. - eventfd_read(buf_place, &dest, weak_eventfd, this) - } - ), - ); - } else { - // Synchronize with all prior `write` calls to this FD. - ecx.acquire_clock(&eventfd.clock.borrow()); - - // Return old counter value into user-space buffer. - ecx.write_int(counter, &buf_place)?; - - // The state changed; we check and update the status of all supported event - // types for current file description. - ecx.check_and_update_readiness(&eventfd_ref)?; - - // Unblock *all* threads previously blocked on `write`. - // We need to take out the blocked thread ids and unblock them together, - // because `unblock_threads` may block them again and end up re-adding the - // thread to the blocked list. - let waiting_threads = std::mem::take(&mut *eventfd.blocked_write_tid.borrow_mut()); - // FIXME: We can randomize the order of unblocking. - for thread_id in waiting_threads { - ecx.unblock_thread(thread_id, BlockReason::Eventfd)?; - } - - // Tell userspace how many bytes we put into the buffer. - return ecx.write_int(buf_place.layout.size.bytes(), dest); - } - interp_ok(()) -} diff --git a/src/tools/miri/src/shims/unix/linux/foreign_items.rs b/src/tools/miri/src/shims/unix/linux/foreign_items.rs index 810e8d3340a..bc3619090c0 100644 --- a/src/tools/miri/src/shims/unix/linux/foreign_items.rs +++ b/src/tools/miri/src/shims/unix/linux/foreign_items.rs @@ -1,10 +1,10 @@ use rustc_abi::ExternAbi; use rustc_span::Symbol; -use self::shims::unix::linux::epoll::EvalContextExt as _; -use self::shims::unix::linux::eventfd::EvalContextExt as _; use self::shims::unix::linux::mem::EvalContextExt as _; -use self::shims::unix::linux::syscall::syscall; +use self::shims::unix::linux_like::epoll::EvalContextExt as _; +use self::shims::unix::linux_like::eventfd::EvalContextExt as _; +use self::shims::unix::linux_like::syscall::syscall; use crate::machine::{SIGRTMAX, SIGRTMIN}; use crate::shims::unix::foreign_items::EvalContextExt as _; use crate::shims::unix::*; diff --git a/src/tools/miri/src/shims/unix/linux/mod.rs b/src/tools/miri/src/shims/unix/linux/mod.rs index 159e5aca031..c10dc52cb28 100644 --- a/src/tools/miri/src/shims/unix/linux/mod.rs +++ b/src/tools/miri/src/shims/unix/linux/mod.rs @@ -1,6 +1,2 @@ -pub mod epoll; -pub mod eventfd; pub mod foreign_items; pub mod mem; -pub mod sync; -pub mod syscall; diff --git a/src/tools/miri/src/shims/unix/linux/sync.rs b/src/tools/miri/src/shims/unix/linux/sync.rs deleted file mode 100644 index 51124fb2a00..00000000000 --- a/src/tools/miri/src/shims/unix/linux/sync.rs +++ /dev/null @@ -1,223 +0,0 @@ -use crate::concurrency::sync::FutexRef; -use crate::helpers::check_min_arg_count; -use crate::*; - -struct LinuxFutex { - futex: FutexRef, -} - -/// Implementation of the SYS_futex syscall. -/// `args` is the arguments *including* the syscall number. -pub fn futex<'tcx>( - ecx: &mut MiriInterpCx<'tcx>, - args: &[OpTy<'tcx>], - dest: &MPlaceTy<'tcx>, -) -> InterpResult<'tcx> { - // The amount of arguments used depends on the type of futex operation. - // The full futex syscall takes six arguments (excluding the syscall - // number), which is also the maximum amount of arguments a linux syscall - // can take on most architectures. - // However, not all futex operations use all six arguments. The unused ones - // may or may not be left out from the `syscall()` call. - // Therefore we don't use `check_arg_count` here, but only check for the - // number of arguments to fall within a range. - let [_, addr, op, val] = check_min_arg_count("`syscall(SYS_futex, ...)`", args)?; - - // The first three arguments (after the syscall number itself) are the same to all futex operations: - // (int *addr, int op, int val). - // We checked above that these definitely exist. - let addr = ecx.read_pointer(addr)?; - let op = ecx.read_scalar(op)?.to_i32()?; - let val = ecx.read_scalar(val)?.to_i32()?; - - // This is a vararg function so we have to bring our own type for this pointer. - let addr = ecx.ptr_to_mplace(addr, ecx.machine.layouts.i32); - - let futex_private = ecx.eval_libc_i32("FUTEX_PRIVATE_FLAG"); - let futex_wait = ecx.eval_libc_i32("FUTEX_WAIT"); - let futex_wait_bitset = ecx.eval_libc_i32("FUTEX_WAIT_BITSET"); - let futex_wake = ecx.eval_libc_i32("FUTEX_WAKE"); - let futex_wake_bitset = ecx.eval_libc_i32("FUTEX_WAKE_BITSET"); - let futex_realtime = ecx.eval_libc_i32("FUTEX_CLOCK_REALTIME"); - - // FUTEX_PRIVATE enables an optimization that stops it from working across processes. - // Miri doesn't support that anyway, so we ignore that flag. - match op & !futex_private { - // FUTEX_WAIT: (int *addr, int op = FUTEX_WAIT, int val, const timespec *timeout) - // Blocks the thread if *addr still equals val. Wakes up when FUTEX_WAKE is called on the same address, - // or *timeout expires. `timeout == null` for an infinite timeout. - // - // FUTEX_WAIT_BITSET: (int *addr, int op = FUTEX_WAIT_BITSET, int val, const timespec *timeout, int *_ignored, unsigned int bitset) - // This is identical to FUTEX_WAIT, except: - // - The timeout is absolute rather than relative. - // - You can specify the bitset to selecting what WAKE operations to respond to. - op if op & !futex_realtime == futex_wait || op & !futex_realtime == futex_wait_bitset => { - let wait_bitset = op & !futex_realtime == futex_wait_bitset; - - let (timeout, bitset) = if wait_bitset { - let [_, _, _, _, timeout, uaddr2, bitset] = - check_min_arg_count("`syscall(SYS_futex, FUTEX_WAIT_BITSET, ...)`", args)?; - let _timeout = ecx.read_pointer(timeout)?; - let _uaddr2 = ecx.read_pointer(uaddr2)?; - (timeout, ecx.read_scalar(bitset)?.to_u32()?) - } else { - let [_, _, _, _, timeout] = - check_min_arg_count("`syscall(SYS_futex, FUTEX_WAIT, ...)`", args)?; - (timeout, u32::MAX) - }; - - if bitset == 0 { - return ecx.set_last_error_and_return(LibcError("EINVAL"), dest); - } - - let timeout = ecx.deref_pointer_as(timeout, ecx.libc_ty_layout("timespec"))?; - let timeout = if ecx.ptr_is_null(timeout.ptr())? { - None - } else { - let duration = match ecx.read_timespec(&timeout)? { - Some(duration) => duration, - None => { - return ecx.set_last_error_and_return(LibcError("EINVAL"), dest); - } - }; - let timeout_clock = if op & futex_realtime == futex_realtime { - ecx.check_no_isolation( - "`futex` syscall with `op=FUTEX_WAIT` and non-null timeout with `FUTEX_CLOCK_REALTIME`", - )?; - TimeoutClock::RealTime - } else { - TimeoutClock::Monotonic - }; - let timeout_anchor = if wait_bitset { - // FUTEX_WAIT_BITSET uses an absolute timestamp. - TimeoutAnchor::Absolute - } else { - // FUTEX_WAIT uses a relative timestamp. - TimeoutAnchor::Relative - }; - Some((timeout_clock, timeout_anchor, duration)) - }; - // There may be a concurrent thread changing the value of addr - // and then invoking the FUTEX_WAKE syscall. It is critical that the - // effects of this and the other thread are correctly observed, - // otherwise we will deadlock. - // - // There are two scenarios to consider, depending on whether WAIT or WAKE goes first: - // 1. If we (FUTEX_WAIT) execute first, we'll push ourselves into the waiters queue and - // go to sleep. They (FUTEX_WAKE) will see us in the queue and wake us up. It doesn't - // matter how the addr write is ordered. - // 2. If they (FUTEX_WAKE) execute first, that means the addr write is also before us - // (FUTEX_WAIT). It is crucial that we observe addr's new value. If we see an - // outdated value that happens to equal the expected val, then we'll put ourselves to - // sleep with no one to wake us up, so we end up with a deadlock. This is prevented - // by having a SeqCst fence inside FUTEX_WAKE syscall, and another SeqCst fence here - // in FUTEX_WAIT. The atomic read on addr after the SeqCst fence is guaranteed not to - // see any value older than the addr write immediately before calling FUTEX_WAKE. - // We'll see futex_val != val and return without sleeping. - // - // Note that the fences do not create any happens-before relationship. - // The read sees the write immediately before the fence not because - // one happens after the other, but is instead due to a guarantee unique - // to SeqCst fences that restricts what an atomic read placed AFTER the - // fence can see. The read still has to be atomic, otherwise it's a data - // race. This guarantee cannot be achieved with acquire-release fences - // since they only talk about reads placed BEFORE a fence - and places - // no restrictions on what the read itself can see, only that there is - // a happens-before between the fences IF the read happens to see the - // right value. This is useless to us, since we need the read itself - // to see an up-to-date value. - // - // The above case distinction is valid since both FUTEX_WAIT and FUTEX_WAKE - // contain a SeqCst fence, therefore inducing a total order between the operations. - // It is also critical that the fence, the atomic load, and the comparison in FUTEX_WAIT - // altogether happen atomically. If the other thread's fence in FUTEX_WAKE - // gets interleaved after our fence, then we lose the guarantee on the - // atomic load being up-to-date; if the other thread's write on addr and FUTEX_WAKE - // call are interleaved after the load but before the comparison, then we get a TOCTOU - // race condition, and go to sleep thinking the other thread will wake us up, - // even though they have already finished. - // - // Thankfully, preemptions cannot happen inside a Miri shim, so we do not need to - // do anything special to guarantee fence-load-comparison atomicity. - ecx.atomic_fence(AtomicFenceOrd::SeqCst)?; - // Read an `i32` through the pointer, regardless of any wrapper types. - // It's not uncommon for `addr` to be passed as another type than `*mut i32`, such as `*const AtomicI32`. - // We do an acquire read -- it only seems reasonable that if we observe a value here, we - // actually establish an ordering with that value. - let futex_val = ecx.read_scalar_atomic(&addr, AtomicReadOrd::Acquire)?.to_i32()?; - if val == futex_val { - // The value still matches, so we block the thread and make it wait for FUTEX_WAKE. - - // This cannot fail since we already did an atomic acquire read on that pointer. - // Acquire reads are only allowed on mutable memory. - let futex_ref = ecx - .get_sync_or_init(addr.ptr(), |_| LinuxFutex { futex: Default::default() }) - .unwrap() - .futex - .clone(); - - ecx.futex_wait( - futex_ref, - bitset, - timeout, - Scalar::from_target_isize(0, ecx), // retval_succ - Scalar::from_target_isize(-1, ecx), // retval_timeout - dest.clone(), - LibcError("ETIMEDOUT"), // errno_timeout - ); - } else { - // The futex value doesn't match the expected value, so we return failure - // right away without sleeping: -1 and errno set to EAGAIN. - return ecx.set_last_error_and_return(LibcError("EAGAIN"), dest); - } - } - // FUTEX_WAKE: (int *addr, int op = FUTEX_WAKE, int val) - // Wakes at most `val` threads waiting on the futex at `addr`. - // Returns the amount of threads woken up. - // Does not access the futex value at *addr. - // FUTEX_WAKE_BITSET: (int *addr, int op = FUTEX_WAKE, int val, const timespect *_unused, int *_unused, unsigned int bitset) - // Same as FUTEX_WAKE, but allows you to specify a bitset to select which threads to wake up. - op if op == futex_wake || op == futex_wake_bitset => { - let Some(futex_ref) = - ecx.get_sync_or_init(addr.ptr(), |_| LinuxFutex { futex: Default::default() }) - else { - // No AllocId, or no live allocation at that AllocId. - // Return an error code. (That seems nicer than silently doing something non-intuitive.) - // This means that if an address gets reused by a new allocation, - // we'll use an independent futex queue for this... that seems acceptable. - return ecx.set_last_error_and_return(LibcError("EFAULT"), dest); - }; - let futex_ref = futex_ref.futex.clone(); - - let bitset = if op == futex_wake_bitset { - let [_, _, _, _, timeout, uaddr2, bitset] = - check_min_arg_count("`syscall(SYS_futex, FUTEX_WAKE_BITSET, ...)`", args)?; - let _timeout = ecx.read_pointer(timeout)?; - let _uaddr2 = ecx.read_pointer(uaddr2)?; - ecx.read_scalar(bitset)?.to_u32()? - } else { - u32::MAX - }; - if bitset == 0 { - return ecx.set_last_error_and_return(LibcError("EINVAL"), dest); - } - // Together with the SeqCst fence in futex_wait, this makes sure that futex_wait - // will see the latest value on addr which could be changed by our caller - // before doing the syscall. - ecx.atomic_fence(AtomicFenceOrd::SeqCst)?; - let mut n = 0; - #[expect(clippy::arithmetic_side_effects)] - for _ in 0..val { - if ecx.futex_wake(&futex_ref, bitset)? { - n += 1; - } else { - break; - } - } - ecx.write_scalar(Scalar::from_target_isize(n, ecx), dest)?; - } - op => throw_unsup_format!("Miri does not support `futex` syscall with op={}", op), - } - - interp_ok(()) -} diff --git a/src/tools/miri/src/shims/unix/linux/syscall.rs b/src/tools/miri/src/shims/unix/linux/syscall.rs deleted file mode 100644 index 9f6935f096b..00000000000 --- a/src/tools/miri/src/shims/unix/linux/syscall.rs +++ /dev/null @@ -1,63 +0,0 @@ -use rustc_abi::ExternAbi; -use rustc_span::Symbol; - -use self::shims::unix::linux::eventfd::EvalContextExt as _; -use crate::helpers::check_min_arg_count; -use crate::shims::unix::linux::sync::futex; -use crate::*; - -pub fn syscall<'tcx>( - ecx: &mut MiriInterpCx<'tcx>, - link_name: Symbol, - abi: ExternAbi, - args: &[OpTy<'tcx>], - dest: &MPlaceTy<'tcx>, -) -> InterpResult<'tcx> { - // We do not use `check_shim` here because `syscall` is variadic. The argument - // count is checked bellow. - ecx.check_abi_and_shim_symbol_clash(abi, ExternAbi::C { unwind: false }, link_name)?; - // The syscall variadic function is legal to call with more arguments than needed, - // extra arguments are simply ignored. The important check is that when we use an - // argument, we have to also check all arguments *before* it to ensure that they - // have the right type. - - let sys_getrandom = ecx.eval_libc("SYS_getrandom").to_target_usize(ecx)?; - let sys_futex = ecx.eval_libc("SYS_futex").to_target_usize(ecx)?; - let sys_eventfd2 = ecx.eval_libc("SYS_eventfd2").to_target_usize(ecx)?; - - let [op] = check_min_arg_count("syscall", args)?; - match ecx.read_target_usize(op)? { - // `libc::syscall(NR_GETRANDOM, buf.as_mut_ptr(), buf.len(), GRND_NONBLOCK)` - // is called if a `HashMap` is created the regular way (e.g. HashMap). - num if num == sys_getrandom => { - // Used by getrandom 0.1 - // The first argument is the syscall id, so skip over it. - let [_, ptr, len, flags] = check_min_arg_count("syscall(SYS_getrandom, ...)", args)?; - - let ptr = ecx.read_pointer(ptr)?; - let len = ecx.read_target_usize(len)?; - // The only supported flags are GRND_RANDOM and GRND_NONBLOCK, - // neither of which have any effect on our current PRNG. - // See for a discussion of argument sizes. - let _flags = ecx.read_scalar(flags)?.to_i32()?; - - ecx.gen_random(ptr, len)?; - ecx.write_scalar(Scalar::from_target_usize(len, ecx), dest)?; - } - // `futex` is used by some synchronization primitives. - num if num == sys_futex => { - futex(ecx, args, dest)?; - } - num if num == sys_eventfd2 => { - let [_, initval, flags] = check_min_arg_count("syscall(SYS_evetfd2, ...)", args)?; - - let result = ecx.eventfd(initval, flags)?; - ecx.write_int(result.to_i32()?, dest)?; - } - num => { - throw_unsup_format!("syscall: unsupported syscall number {num}"); - } - }; - - interp_ok(()) -} diff --git a/src/tools/miri/src/shims/unix/linux_like/epoll.rs b/src/tools/miri/src/shims/unix/linux_like/epoll.rs new file mode 100644 index 00000000000..b20b12528db --- /dev/null +++ b/src/tools/miri/src/shims/unix/linux_like/epoll.rs @@ -0,0 +1,663 @@ +use std::cell::RefCell; +use std::collections::BTreeMap; +use std::io; +use std::rc::{Rc, Weak}; +use std::time::Duration; + +use crate::concurrency::VClock; +use crate::shims::unix::fd::{FdId, FileDescriptionRef, WeakFileDescriptionRef}; +use crate::shims::unix::*; +use crate::*; + +/// An `Epoll` file descriptor connects file handles and epoll events +#[derive(Clone, Debug, Default)] +struct Epoll { + /// A map of EpollEventInterests registered under this epoll instance. + /// Each entry is differentiated using FdId and file descriptor value. + interest_list: RefCell>>>, + /// A map of EpollEventInstance that will be returned when `epoll_wait` is called. + /// Similar to interest_list, the entry is also differentiated using FdId + /// and file descriptor value. + // This is an Rc because EpollInterest need to hold a reference to update + // it. + ready_list: Rc, + /// A list of thread ids blocked on this epoll instance. + thread_id: RefCell>, +} + +/// EpollEventInstance contains information that will be returned by epoll_wait. +#[derive(Debug)] +pub struct EpollEventInstance { + /// Xor-ed event types that happened to the file description. + events: u32, + /// Original data retrieved from `epoll_event` during `epoll_ctl`. + data: u64, + /// The release clock associated with this event. + clock: VClock, +} + +impl EpollEventInstance { + pub fn new(events: u32, data: u64) -> EpollEventInstance { + EpollEventInstance { events, data, clock: Default::default() } + } +} + +/// EpollEventInterest registers the file description information to an epoll +/// instance during a successful `epoll_ctl` call. It also stores additional +/// information needed to check and update readiness state for `epoll_wait`. +/// +/// `events` and `data` field matches the `epoll_event` struct defined +/// by the epoll_ctl man page. For more information +/// see the man page: +/// +/// +#[derive(Clone, Debug)] +pub struct EpollEventInterest { + /// The file descriptor value of the file description registered. + /// This is only used for ready_list, to inform userspace which FD triggered an event. + /// For that, it is crucial to preserve the original FD number. + /// This FD number must never be "dereferenced" to a file description inside Miri. + fd_num: i32, + /// The events bitmask retrieved from `epoll_event`. + events: u32, + /// The data retrieved from `epoll_event`. + /// libc's data field in epoll_event can store integer or pointer, + /// but only u64 is supported for now. + /// + data: u64, + /// Ready list of the epoll instance under which this EpollEventInterest is registered. + ready_list: Rc, + /// The epoll file description that this EpollEventInterest is registered under. + weak_epfd: WeakFileDescriptionRef, +} + +/// EpollReadyEvents reflects the readiness of a file description. +pub struct EpollReadyEvents { + /// The associated file is available for read(2) operations, in the sense that a read will not block. + /// (I.e., returning EOF is considered "ready".) + pub epollin: bool, + /// The associated file is available for write(2) operations, in the sense that a write will not block. + pub epollout: bool, + /// Stream socket peer closed connection, or shut down writing + /// half of connection. + pub epollrdhup: bool, + /// For stream socket, this event merely indicates that the peer + /// closed its end of the channel. + /// Unlike epollrdhup, this should only be set when the stream is fully closed. + /// epollrdhup also gets set when only the write half is closed, which is possible + /// via `shutdown(_, SHUT_WR)`. + pub epollhup: bool, + /// Error condition happened on the associated file descriptor. + pub epollerr: bool, +} + +#[derive(Debug, Default)] +struct ReadyList { + mapping: RefCell>, +} + +impl EpollReadyEvents { + pub fn new() -> Self { + EpollReadyEvents { + epollin: false, + epollout: false, + epollrdhup: false, + epollhup: false, + epollerr: false, + } + } + + pub fn get_event_bitmask<'tcx>(&self, ecx: &MiriInterpCx<'tcx>) -> u32 { + let epollin = ecx.eval_libc_u32("EPOLLIN"); + let epollout = ecx.eval_libc_u32("EPOLLOUT"); + let epollrdhup = ecx.eval_libc_u32("EPOLLRDHUP"); + let epollhup = ecx.eval_libc_u32("EPOLLHUP"); + let epollerr = ecx.eval_libc_u32("EPOLLERR"); + + let mut bitmask = 0; + if self.epollin { + bitmask |= epollin; + } + if self.epollout { + bitmask |= epollout; + } + if self.epollrdhup { + bitmask |= epollrdhup; + } + if self.epollhup { + bitmask |= epollhup; + } + if self.epollerr { + bitmask |= epollerr; + } + bitmask + } +} + +impl Epoll { + fn get_ready_list(&self) -> Rc { + Rc::clone(&self.ready_list) + } +} + +impl FileDescription for Epoll { + fn name(&self) -> &'static str { + "epoll" + } + + fn close<'tcx>( + self: Box, + _communicate_allowed: bool, + _ecx: &mut MiriInterpCx<'tcx>, + ) -> InterpResult<'tcx, io::Result<()>> { + interp_ok(Ok(())) + } +} + +/// The table of all EpollEventInterest. +/// The BTreeMap key is the FdId of an active file description registered with +/// any epoll instance. The value is a list of EpollEventInterest associated +/// with that file description. +pub struct EpollInterestTable(BTreeMap>>>); + +impl EpollInterestTable { + pub(crate) fn new() -> Self { + EpollInterestTable(BTreeMap::new()) + } + + pub fn insert_epoll_interest(&mut self, id: FdId, fd: Weak>) { + match self.0.get_mut(&id) { + Some(fds) => { + fds.push(fd); + } + None => { + let vec = vec![fd]; + self.0.insert(id, vec); + } + } + } + + pub fn get_epoll_interest(&self, id: FdId) -> Option<&Vec>>> { + self.0.get(&id) + } + + pub fn get_epoll_interest_mut( + &mut self, + id: FdId, + ) -> Option<&mut Vec>>> { + self.0.get_mut(&id) + } + + pub fn remove(&mut self, id: FdId) { + self.0.remove(&id); + } +} + +impl<'tcx> EvalContextExt<'tcx> for crate::MiriInterpCx<'tcx> {} +pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { + /// This function returns a file descriptor referring to the new `Epoll` instance. This file + /// descriptor is used for all subsequent calls to the epoll interface. If the `flags` argument + /// is 0, then this function is the same as `epoll_create()`. + /// + /// + fn epoll_create1(&mut self, flags: &OpTy<'tcx>) -> InterpResult<'tcx, Scalar> { + let this = self.eval_context_mut(); + + let flags = this.read_scalar(flags)?.to_i32()?; + + let epoll_cloexec = this.eval_libc_i32("EPOLL_CLOEXEC"); + + // Miri does not support exec, so EPOLL_CLOEXEC flag has no effect. + if flags != epoll_cloexec && flags != 0 { + throw_unsup_format!( + "epoll_create1: flag {:#x} is unsupported, only 0 or EPOLL_CLOEXEC are allowed", + flags + ); + } + + let fd = this.machine.fds.insert_new(Epoll::default()); + interp_ok(Scalar::from_i32(fd)) + } + + /// This function performs control operations on the `Epoll` instance referred to by the file + /// descriptor `epfd`. It requests that the operation `op` be performed for the target file + /// descriptor, `fd`. + /// + /// Valid values for the op argument are: + /// `EPOLL_CTL_ADD` - Register the target file descriptor `fd` on the `Epoll` instance referred + /// to by the file descriptor `epfd` and associate the event `event` with the internal file + /// linked to `fd`. + /// `EPOLL_CTL_MOD` - Change the event `event` associated with the target file descriptor `fd`. + /// `EPOLL_CTL_DEL` - Deregister the target file descriptor `fd` from the `Epoll` instance + /// referred to by `epfd`. The `event` is ignored and can be null. + /// + /// + fn epoll_ctl( + &mut self, + epfd: &OpTy<'tcx>, + op: &OpTy<'tcx>, + fd: &OpTy<'tcx>, + event: &OpTy<'tcx>, + ) -> InterpResult<'tcx, Scalar> { + let this = self.eval_context_mut(); + + let epfd_value = this.read_scalar(epfd)?.to_i32()?; + let op = this.read_scalar(op)?.to_i32()?; + let fd = this.read_scalar(fd)?.to_i32()?; + let event = this.deref_pointer_as(event, this.libc_ty_layout("epoll_event"))?; + + let epoll_ctl_add = this.eval_libc_i32("EPOLL_CTL_ADD"); + let epoll_ctl_mod = this.eval_libc_i32("EPOLL_CTL_MOD"); + let epoll_ctl_del = this.eval_libc_i32("EPOLL_CTL_DEL"); + let epollin = this.eval_libc_u32("EPOLLIN"); + let epollout = this.eval_libc_u32("EPOLLOUT"); + let epollrdhup = this.eval_libc_u32("EPOLLRDHUP"); + let epollet = this.eval_libc_u32("EPOLLET"); + let epollhup = this.eval_libc_u32("EPOLLHUP"); + let epollerr = this.eval_libc_u32("EPOLLERR"); + + // Throw EINVAL if epfd and fd have the same value. + if epfd_value == fd { + return this.set_last_error_and_return_i32(LibcError("EINVAL")); + } + + // Check if epfd is a valid epoll file descriptor. + let Some(epfd) = this.machine.fds.get(epfd_value) else { + return this.set_last_error_and_return_i32(LibcError("EBADF")); + }; + let epoll_file_description = epfd + .downcast::() + .ok_or_else(|| err_unsup_format!("non-epoll FD passed to `epoll_ctl`"))?; + + let mut interest_list = epoll_file_description.interest_list.borrow_mut(); + let ready_list = &epoll_file_description.ready_list; + + let Some(fd_ref) = this.machine.fds.get(fd) else { + return this.set_last_error_and_return_i32(LibcError("EBADF")); + }; + let id = fd_ref.get_id(); + + if op == epoll_ctl_add || op == epoll_ctl_mod { + // Read event bitmask and data from epoll_event passed by caller. + let mut events = this.read_scalar(&this.project_field(&event, 0)?)?.to_u32()?; + let data = this.read_scalar(&this.project_field(&event, 1)?)?.to_u64()?; + + // Unset the flag we support to discover if any unsupported flags are used. + let mut flags = events; + // epoll_wait(2) will always wait for epollhup and epollerr; it is not + // necessary to set it in events when calling epoll_ctl(). + // So we will always set these two event types. + events |= epollhup; + events |= epollerr; + + if events & epollet != epollet { + // We only support edge-triggered notification for now. + throw_unsup_format!("epoll_ctl: epollet flag must be included."); + } else { + flags &= !epollet; + } + if flags & epollin == epollin { + flags &= !epollin; + } + if flags & epollout == epollout { + flags &= !epollout; + } + if flags & epollrdhup == epollrdhup { + flags &= !epollrdhup; + } + if flags & epollhup == epollhup { + flags &= !epollhup; + } + if flags & epollerr == epollerr { + flags &= !epollerr; + } + if flags != 0 { + throw_unsup_format!( + "epoll_ctl: encountered unknown unsupported flags {:#x}", + flags + ); + } + + let epoll_key = (id, fd); + + // Check the existence of fd in the interest list. + if op == epoll_ctl_add { + if interest_list.contains_key(&epoll_key) { + return this.set_last_error_and_return_i32(LibcError("EEXIST")); + } + } else { + if !interest_list.contains_key(&epoll_key) { + return this.set_last_error_and_return_i32(LibcError("ENOENT")); + } + } + + // Create an epoll_interest. + let interest = Rc::new(RefCell::new(EpollEventInterest { + fd_num: fd, + events, + data, + ready_list: Rc::clone(ready_list), + weak_epfd: epfd.downgrade(), + })); + + if op == epoll_ctl_add { + // Insert an epoll_interest to global epoll_interest list. + this.machine.epoll_interests.insert_epoll_interest(id, Rc::downgrade(&interest)); + interest_list.insert(epoll_key, Rc::clone(&interest)); + } else { + // Directly modify the epoll_interest so the global epoll_event_interest table + // will be updated too. + let mut epoll_interest = interest_list.get_mut(&epoll_key).unwrap().borrow_mut(); + epoll_interest.events = events; + epoll_interest.data = data; + } + + // Notification will be returned for current epfd if there is event in the file + // descriptor we registered. + check_and_update_one_event_interest(&fd_ref, interest, id, this)?; + interp_ok(Scalar::from_i32(0)) + } else if op == epoll_ctl_del { + let epoll_key = (id, fd); + + // Remove epoll_event_interest from interest_list. + let Some(epoll_interest) = interest_list.remove(&epoll_key) else { + return this.set_last_error_and_return_i32(LibcError("ENOENT")); + }; + // All related Weak will fail to upgrade after the drop. + drop(epoll_interest); + + // Remove related epoll_interest from ready list. + ready_list.mapping.borrow_mut().remove(&epoll_key); + + // Remove dangling EpollEventInterest from its global table. + // .unwrap() below should succeed because the file description id must have registered + // at least one epoll_interest, if not, it will fail when removing epoll_interest from + // interest list. + this.machine + .epoll_interests + .get_epoll_interest_mut(id) + .unwrap() + .retain(|event| event.upgrade().is_some()); + + interp_ok(Scalar::from_i32(0)) + } else { + throw_unsup_format!("unsupported epoll_ctl operation: {op}"); + } + } + + /// The `epoll_wait()` system call waits for events on the `Epoll` + /// instance referred to by the file descriptor `epfd`. The buffer + /// pointed to by `events` is used to return information from the ready + /// list about file descriptors in the interest list that have some + /// events available. Up to `maxevents` are returned by `epoll_wait()`. + /// The `maxevents` argument must be greater than zero. + /// + /// The `timeout` argument specifies the number of milliseconds that + /// `epoll_wait()` will block. Time is measured against the + /// CLOCK_MONOTONIC clock. If the timeout is zero, the function will not block, + /// while if the timeout is -1, the function will block + /// until at least one event has been retrieved (or an error + /// occurred). + /// + /// A call to `epoll_wait()` will block until either: + /// • a file descriptor delivers an event; + /// • the call is interrupted by a signal handler; or + /// • the timeout expires. + /// + /// Note that the timeout interval will be rounded up to the system + /// clock granularity, and kernel scheduling delays mean that the + /// blocking interval may overrun by a small amount. Specifying a + /// timeout of -1 causes `epoll_wait()` to block indefinitely, while + /// specifying a timeout equal to zero cause `epoll_wait()` to return + /// immediately, even if no events are available. + /// + /// On success, `epoll_wait()` returns the number of file descriptors + /// ready for the requested I/O, or zero if no file descriptor became + /// ready during the requested timeout milliseconds. On failure, + /// `epoll_wait()` returns -1 and errno is set to indicate the error. + /// + /// + fn epoll_wait( + &mut self, + epfd: &OpTy<'tcx>, + events_op: &OpTy<'tcx>, + maxevents: &OpTy<'tcx>, + timeout: &OpTy<'tcx>, + dest: &MPlaceTy<'tcx>, + ) -> InterpResult<'tcx> { + let this = self.eval_context_mut(); + + let epfd_value = this.read_scalar(epfd)?.to_i32()?; + let events = this.read_immediate(events_op)?; + let maxevents = this.read_scalar(maxevents)?.to_i32()?; + let timeout = this.read_scalar(timeout)?.to_i32()?; + + if epfd_value <= 0 || maxevents <= 0 { + return this.set_last_error_and_return(LibcError("EINVAL"), dest); + } + + // This needs to come after the maxevents value check, or else maxevents.try_into().unwrap() + // will fail. + let event = this.deref_pointer_as( + &events, + this.libc_array_ty_layout("epoll_event", maxevents.try_into().unwrap()), + )?; + + let Some(epfd) = this.machine.fds.get(epfd_value) else { + return this.set_last_error_and_return(LibcError("EBADF"), dest); + }; + // Create a weak ref of epfd and pass it to callback so we will make sure that epfd + // is not close after the thread unblocks. + let weak_epfd = epfd.downgrade(); + + // We just need to know if the ready list is empty and borrow the thread_ids out. + // The whole logic is wrapped inside a block so we don't need to manually drop epfd later. + let ready_list_empty; + let mut thread_ids; + { + let epoll_file_description = epfd + .downcast::() + .ok_or_else(|| err_unsup_format!("non-epoll FD passed to `epoll_wait`"))?; + ready_list_empty = epoll_file_description.ready_list.mapping.borrow().is_empty(); + thread_ids = epoll_file_description.thread_id.borrow_mut(); + } + if timeout == 0 || !ready_list_empty { + // If the ready list is not empty, or the timeout is 0, we can return immediately. + return_ready_list(epfd_value, weak_epfd, dest, &event, this)?; + } else { + // Blocking + let timeout = match timeout { + 0.. => { + let duration = Duration::from_millis(timeout.try_into().unwrap()); + Some((TimeoutClock::Monotonic, TimeoutAnchor::Relative, duration)) + } + -1 => None, + ..-1 => { + throw_unsup_format!( + "epoll_wait: Only timeout values greater than or equal to -1 are supported." + ); + } + }; + thread_ids.push(this.active_thread()); + let dest = dest.clone(); + this.block_thread( + BlockReason::Epoll, + timeout, + callback!( + @capture<'tcx> { + epfd_value: i32, + weak_epfd: WeakFileDescriptionRef, + dest: MPlaceTy<'tcx>, + event: MPlaceTy<'tcx>, + } + @unblock = |this| { + return_ready_list(epfd_value, weak_epfd, &dest, &event, this)?; + interp_ok(()) + } + @timeout = |this| { + // No notification after blocking timeout. + let Some(epfd) = weak_epfd.upgrade() else { + throw_unsup_format!("epoll FD {epfd_value} got closed while blocking.") + }; + // Remove the current active thread_id from the blocked thread_id list. + epfd.downcast::() + .ok_or_else(|| err_unsup_format!("non-epoll FD passed to `epoll_wait`"))? + .thread_id.borrow_mut() + .retain(|&id| id != this.active_thread()); + this.write_int(0, &dest)?; + interp_ok(()) + } + ), + ); + } + interp_ok(()) + } + + /// For a specific file description, get its ready events and update the corresponding ready + /// list. This function should be called whenever an event causes more bytes or an EOF to become + /// newly readable from an FD, and whenever more bytes can be written to an FD or no more future + /// writes are possible. + /// + /// This *will* report an event if anyone is subscribed to it, without any further filtering, so + /// do not call this function when an FD didn't have anything happen to it! + fn check_and_update_readiness( + &mut self, + fd_ref: &FileDescriptionRef, + ) -> InterpResult<'tcx, ()> { + let this = self.eval_context_mut(); + let id = fd_ref.get_id(); + let mut waiter = Vec::new(); + // Get a list of EpollEventInterest that is associated to a specific file description. + if let Some(epoll_interests) = this.machine.epoll_interests.get_epoll_interest(id) { + for weak_epoll_interest in epoll_interests { + if let Some(epoll_interest) = weak_epoll_interest.upgrade() { + let is_updated = check_and_update_one_event_interest( + fd_ref, + epoll_interest.clone(), + id, + this, + )?; + if is_updated { + // Edge-triggered notification only notify one thread even if there are + // multiple threads blocked on the same epfd. + + // This unwrap can never fail because if the current epoll instance were + // closed, the upgrade of weak_epoll_interest + // above would fail. This guarantee holds because only the epoll instance + // holds a strong ref to epoll_interest. + let epfd = epoll_interest.borrow().weak_epfd.upgrade().unwrap(); + // FIXME: We can randomly pick a thread to unblock. + + let epoll = epfd.downcast::().unwrap(); + + if let Some(thread_id) = epoll.thread_id.borrow_mut().pop() { + waiter.push(thread_id); + }; + } + } + } + } + waiter.sort(); + waiter.dedup(); + for thread_id in waiter { + this.unblock_thread(thread_id, BlockReason::Epoll)?; + } + interp_ok(()) + } +} + +/// This function takes in ready list and returns EpollEventInstance with file description +/// that is not closed. +fn ready_list_next( + ecx: &MiriInterpCx<'_>, + ready_list: &mut BTreeMap<(FdId, i32), EpollEventInstance>, +) -> Option { + while let Some((epoll_key, epoll_event_instance)) = ready_list.pop_first() { + // This ensures that we only return events that we are interested. The FD might have been closed since + // the event was generated, in which case we are not interested anymore. + // When a file description is fully closed, it gets removed from `machine.epoll_interests`, + // so we skip events whose FD is not in that map anymore. + if ecx.machine.epoll_interests.get_epoll_interest(epoll_key.0).is_some() { + return Some(epoll_event_instance); + } + } + None +} + +/// This helper function checks whether an epoll notification should be triggered for a specific +/// epoll_interest and, if necessary, triggers the notification, and returns whether the +/// notification was added/updated. Unlike check_and_update_readiness, this function sends a +/// notification to only one epoll instance. +fn check_and_update_one_event_interest<'tcx>( + fd_ref: &FileDescriptionRef, + interest: Rc>, + id: FdId, + ecx: &MiriInterpCx<'tcx>, +) -> InterpResult<'tcx, bool> { + // Get the bitmask of ready events for a file description. + let ready_events_bitmask = fd_ref.get_epoll_ready_events()?.get_event_bitmask(ecx); + let epoll_event_interest = interest.borrow(); + // This checks if any of the events specified in epoll_event_interest.events + // match those in ready_events. + let flags = epoll_event_interest.events & ready_events_bitmask; + // If there is any event that we are interested in being specified as ready, + // insert an epoll_return to the ready list. + if flags != 0 { + let epoll_key = (id, epoll_event_interest.fd_num); + let ready_list = &mut epoll_event_interest.ready_list.mapping.borrow_mut(); + let mut event_instance = EpollEventInstance::new(flags, epoll_event_interest.data); + // If we are tracking data races, remember the current clock so we can sync with it later. + ecx.release_clock(|clock| { + event_instance.clock.clone_from(clock); + }); + // Triggers the notification by inserting it to the ready list. + ready_list.insert(epoll_key, event_instance); + interp_ok(true) + } else { + interp_ok(false) + } +} + +/// Stores the ready list of the `epfd` epoll instance into `events` (which must be an array), +/// and the number of returned events into `dest`. +fn return_ready_list<'tcx>( + epfd_value: i32, + weak_epfd: WeakFileDescriptionRef, + dest: &MPlaceTy<'tcx>, + events: &MPlaceTy<'tcx>, + ecx: &mut MiriInterpCx<'tcx>, +) -> InterpResult<'tcx> { + let Some(epfd) = weak_epfd.upgrade() else { + throw_unsup_format!("epoll FD {epfd_value} got closed while blocking.") + }; + + let epoll_file_description = epfd + .downcast::() + .ok_or_else(|| err_unsup_format!("non-epoll FD passed to `epoll_wait`"))?; + + let ready_list = epoll_file_description.get_ready_list(); + + let mut ready_list = ready_list.mapping.borrow_mut(); + let mut num_of_events: i32 = 0; + let mut array_iter = ecx.project_array_fields(events)?; + + while let Some(des) = array_iter.next(ecx)? { + if let Some(epoll_event_instance) = ready_list_next(ecx, &mut ready_list) { + ecx.write_int_fields_named( + &[ + ("events", epoll_event_instance.events.into()), + ("u64", epoll_event_instance.data.into()), + ], + &des.1, + )?; + // Synchronize waking thread with the event of interest. + ecx.acquire_clock(&epoll_event_instance.clock); + + num_of_events = num_of_events.strict_add(1); + } else { + break; + } + } + ecx.write_int(num_of_events, dest)?; + interp_ok(()) +} diff --git a/src/tools/miri/src/shims/unix/linux_like/eventfd.rs b/src/tools/miri/src/shims/unix/linux_like/eventfd.rs new file mode 100644 index 00000000000..61c91877946 --- /dev/null +++ b/src/tools/miri/src/shims/unix/linux_like/eventfd.rs @@ -0,0 +1,330 @@ +//! Linux `eventfd` implementation. +use std::cell::{Cell, RefCell}; +use std::io; +use std::io::ErrorKind; + +use crate::concurrency::VClock; +use crate::shims::unix::fd::{FileDescriptionRef, WeakFileDescriptionRef}; +use crate::shims::unix::linux_like::epoll::{EpollReadyEvents, EvalContextExt as _}; +use crate::shims::unix::*; +use crate::*; + +/// Maximum value that the eventfd counter can hold. +const MAX_COUNTER: u64 = u64::MAX - 1; + +/// A kind of file descriptor created by `eventfd`. +/// The `Event` type isn't currently written to by `eventfd`. +/// The interface is meant to keep track of objects associated +/// with a file descriptor. For more information see the man +/// page below: +/// +/// +#[derive(Debug)] +struct Event { + /// The object contains an unsigned 64-bit integer (uint64_t) counter that is maintained by the + /// kernel. This counter is initialized with the value specified in the argument initval. + counter: Cell, + is_nonblock: bool, + clock: RefCell, + /// A list of thread ids blocked on eventfd::read. + blocked_read_tid: RefCell>, + /// A list of thread ids blocked on eventfd::write. + blocked_write_tid: RefCell>, +} + +impl FileDescription for Event { + fn name(&self) -> &'static str { + "event" + } + + fn get_epoll_ready_events<'tcx>(&self) -> InterpResult<'tcx, EpollReadyEvents> { + // We only check the status of EPOLLIN and EPOLLOUT flags for eventfd. If other event flags + // need to be supported in the future, the check should be added here. + + interp_ok(EpollReadyEvents { + epollin: self.counter.get() != 0, + epollout: self.counter.get() != MAX_COUNTER, + ..EpollReadyEvents::new() + }) + } + + fn close<'tcx>( + self: Box, + _communicate_allowed: bool, + _ecx: &mut MiriInterpCx<'tcx>, + ) -> InterpResult<'tcx, io::Result<()>> { + interp_ok(Ok(())) + } + + /// Read the counter in the buffer and return the counter if succeeded. + fn read<'tcx>( + &self, + self_ref: &FileDescriptionRef, + _communicate_allowed: bool, + ptr: Pointer, + len: usize, + dest: &MPlaceTy<'tcx>, + ecx: &mut MiriInterpCx<'tcx>, + ) -> InterpResult<'tcx> { + // We're treating the buffer as a `u64`. + let ty = ecx.machine.layouts.u64; + // Check the size of slice, and return error only if the size of the slice < 8. + if len < ty.size.bytes_usize() { + return ecx.set_last_error_and_return(ErrorKind::InvalidInput, dest); + } + + // eventfd read at the size of u64. + let buf_place = ecx.ptr_to_mplace_unaligned(ptr, ty); + + let weak_eventfd = self_ref.downgrade(); + eventfd_read(buf_place, dest, weak_eventfd, ecx) + } + + /// A write call adds the 8-byte integer value supplied in + /// its buffer (in native endianness) to the counter. The maximum value that may be + /// stored in the counter is the largest unsigned 64-bit value + /// minus 1 (i.e., 0xfffffffffffffffe). If the addition would + /// cause the counter's value to exceed the maximum, then the + /// write either blocks until a read is performed on the + /// file descriptor, or fails with the error EAGAIN if the + /// file descriptor has been made nonblocking. + /// + /// A write fails with the error EINVAL if the size of the + /// supplied buffer is less than 8 bytes, or if an attempt is + /// made to write the value 0xffffffffffffffff. + fn write<'tcx>( + &self, + self_ref: &FileDescriptionRef, + _communicate_allowed: bool, + ptr: Pointer, + len: usize, + dest: &MPlaceTy<'tcx>, + ecx: &mut MiriInterpCx<'tcx>, + ) -> InterpResult<'tcx> { + // We're treating the buffer as a `u64`. + let ty = ecx.machine.layouts.u64; + // Check the size of slice, and return error only if the size of the slice < 8. + if len < ty.layout.size.bytes_usize() { + return ecx.set_last_error_and_return(ErrorKind::InvalidInput, dest); + } + + // Read the user-supplied value from the pointer. + let buf_place = ecx.ptr_to_mplace_unaligned(ptr, ty); + let num = ecx.read_scalar(&buf_place)?.to_u64()?; + + // u64::MAX as input is invalid because the maximum value of counter is u64::MAX - 1. + if num == u64::MAX { + return ecx.set_last_error_and_return(ErrorKind::InvalidInput, dest); + } + // If the addition does not let the counter to exceed the maximum value, update the counter. + // Else, block. + let weak_eventfd = self_ref.downgrade(); + eventfd_write(num, buf_place, dest, weak_eventfd, ecx) + } +} + +impl<'tcx> EvalContextExt<'tcx> for crate::MiriInterpCx<'tcx> {} +pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { + /// This function creates an `Event` that is used as an event wait/notify mechanism by + /// user-space applications, and by the kernel to notify user-space applications of events. + /// The `Event` contains an `u64` counter maintained by the kernel. The counter is initialized + /// with the value specified in the `initval` argument. + /// + /// A new file descriptor referring to the `Event` is returned. The `read`, `write`, `poll`, + /// `select`, and `close` operations can be performed on the file descriptor. For more + /// information on these operations, see the man page linked below. + /// + /// The `flags` are not currently implemented for eventfd. + /// The `flags` may be bitwise ORed to change the behavior of `eventfd`: + /// `EFD_CLOEXEC` - Set the close-on-exec (`FD_CLOEXEC`) flag on the new file descriptor. + /// `EFD_NONBLOCK` - Set the `O_NONBLOCK` file status flag on the new open file description. + /// `EFD_SEMAPHORE` - miri does not support semaphore-like semantics. + /// + /// + fn eventfd(&mut self, val: &OpTy<'tcx>, flags: &OpTy<'tcx>) -> InterpResult<'tcx, Scalar> { + let this = self.eval_context_mut(); + + let val = this.read_scalar(val)?.to_u32()?; + let mut flags = this.read_scalar(flags)?.to_i32()?; + + let efd_cloexec = this.eval_libc_i32("EFD_CLOEXEC"); + let efd_nonblock = this.eval_libc_i32("EFD_NONBLOCK"); + let efd_semaphore = this.eval_libc_i32("EFD_SEMAPHORE"); + + if flags & efd_semaphore == efd_semaphore { + throw_unsup_format!("eventfd: EFD_SEMAPHORE is unsupported"); + } + + let mut is_nonblock = false; + // Unset the flag that we support. + // After unloading, flags != 0 means other flags are used. + if flags & efd_cloexec == efd_cloexec { + // cloexec is ignored because Miri does not support exec. + flags &= !efd_cloexec; + } + if flags & efd_nonblock == efd_nonblock { + flags &= !efd_nonblock; + is_nonblock = true; + } + if flags != 0 { + throw_unsup_format!("eventfd: encountered unknown unsupported flags {:#x}", flags); + } + + let fds = &mut this.machine.fds; + + let fd_value = fds.insert_new(Event { + counter: Cell::new(val.into()), + is_nonblock, + clock: RefCell::new(VClock::default()), + blocked_read_tid: RefCell::new(Vec::new()), + blocked_write_tid: RefCell::new(Vec::new()), + }); + + interp_ok(Scalar::from_i32(fd_value)) + } +} + +/// Block thread if the value addition will exceed u64::MAX -1, +/// else just add the user-supplied value to current counter. +fn eventfd_write<'tcx>( + num: u64, + buf_place: MPlaceTy<'tcx>, + dest: &MPlaceTy<'tcx>, + weak_eventfd: WeakFileDescriptionRef, + ecx: &mut MiriInterpCx<'tcx>, +) -> InterpResult<'tcx> { + let Some(eventfd_ref) = weak_eventfd.upgrade() else { + throw_unsup_format!("eventfd FD got closed while blocking.") + }; + + // Since we pass the weak file description ref, it is guaranteed to be + // an eventfd file description. + let eventfd = eventfd_ref.downcast::().unwrap(); + + match eventfd.counter.get().checked_add(num) { + Some(new_count @ 0..=MAX_COUNTER) => { + // Future `read` calls will synchronize with this write, so update the FD clock. + ecx.release_clock(|clock| { + eventfd.clock.borrow_mut().join(clock); + }); + + // Store new counter value. + eventfd.counter.set(new_count); + + // The state changed; we check and update the status of all supported event + // types for current file description. + ecx.check_and_update_readiness(&eventfd_ref)?; + + // Unblock *all* threads previously blocked on `read`. + // We need to take out the blocked thread ids and unblock them together, + // because `unblock_threads` may block them again and end up re-adding the + // thread to the blocked list. + let waiting_threads = std::mem::take(&mut *eventfd.blocked_read_tid.borrow_mut()); + // FIXME: We can randomize the order of unblocking. + for thread_id in waiting_threads { + ecx.unblock_thread(thread_id, BlockReason::Eventfd)?; + } + + // Return how many bytes we consumed from the user-provided buffer. + return ecx.write_int(buf_place.layout.size.bytes(), dest); + } + None | Some(u64::MAX) => { + // We can't update the state, so we have to block. + if eventfd.is_nonblock { + return ecx.set_last_error_and_return(ErrorKind::WouldBlock, dest); + } + + let dest = dest.clone(); + + eventfd.blocked_write_tid.borrow_mut().push(ecx.active_thread()); + + ecx.block_thread( + BlockReason::Eventfd, + None, + callback!( + @capture<'tcx> { + num: u64, + buf_place: MPlaceTy<'tcx>, + dest: MPlaceTy<'tcx>, + weak_eventfd: WeakFileDescriptionRef, + } + @unblock = |this| { + // When we get unblocked, try again. + eventfd_write(num, buf_place, &dest, weak_eventfd, this) + } + ), + ); + } + }; + interp_ok(()) +} + +/// Block thread if the current counter is 0, +/// else just return the current counter value to the caller and set the counter to 0. +fn eventfd_read<'tcx>( + buf_place: MPlaceTy<'tcx>, + dest: &MPlaceTy<'tcx>, + weak_eventfd: WeakFileDescriptionRef, + ecx: &mut MiriInterpCx<'tcx>, +) -> InterpResult<'tcx> { + let Some(eventfd_ref) = weak_eventfd.upgrade() else { + throw_unsup_format!("eventfd FD got closed while blocking.") + }; + + // Since we pass the weak file description ref to the callback function, it is guaranteed to be + // an eventfd file description. + let eventfd = eventfd_ref.downcast::().unwrap(); + + // Set counter to 0, get old value. + let counter = eventfd.counter.replace(0); + + // Block when counter == 0. + if counter == 0 { + if eventfd.is_nonblock { + return ecx.set_last_error_and_return(ErrorKind::WouldBlock, dest); + } + let dest = dest.clone(); + + eventfd.blocked_read_tid.borrow_mut().push(ecx.active_thread()); + + ecx.block_thread( + BlockReason::Eventfd, + None, + callback!( + @capture<'tcx> { + buf_place: MPlaceTy<'tcx>, + dest: MPlaceTy<'tcx>, + weak_eventfd: WeakFileDescriptionRef, + } + @unblock = |this| { + // When we get unblocked, try again. + eventfd_read(buf_place, &dest, weak_eventfd, this) + } + ), + ); + } else { + // Synchronize with all prior `write` calls to this FD. + ecx.acquire_clock(&eventfd.clock.borrow()); + + // Return old counter value into user-space buffer. + ecx.write_int(counter, &buf_place)?; + + // The state changed; we check and update the status of all supported event + // types for current file description. + ecx.check_and_update_readiness(&eventfd_ref)?; + + // Unblock *all* threads previously blocked on `write`. + // We need to take out the blocked thread ids and unblock them together, + // because `unblock_threads` may block them again and end up re-adding the + // thread to the blocked list. + let waiting_threads = std::mem::take(&mut *eventfd.blocked_write_tid.borrow_mut()); + // FIXME: We can randomize the order of unblocking. + for thread_id in waiting_threads { + ecx.unblock_thread(thread_id, BlockReason::Eventfd)?; + } + + // Tell userspace how many bytes we put into the buffer. + return ecx.write_int(buf_place.layout.size.bytes(), dest); + } + interp_ok(()) +} diff --git a/src/tools/miri/src/shims/unix/linux_like/mod.rs b/src/tools/miri/src/shims/unix/linux_like/mod.rs new file mode 100644 index 00000000000..1a22539a4ee --- /dev/null +++ b/src/tools/miri/src/shims/unix/linux_like/mod.rs @@ -0,0 +1,4 @@ +pub mod epoll; +pub mod eventfd; +pub mod sync; +pub mod syscall; diff --git a/src/tools/miri/src/shims/unix/linux_like/sync.rs b/src/tools/miri/src/shims/unix/linux_like/sync.rs new file mode 100644 index 00000000000..51124fb2a00 --- /dev/null +++ b/src/tools/miri/src/shims/unix/linux_like/sync.rs @@ -0,0 +1,223 @@ +use crate::concurrency::sync::FutexRef; +use crate::helpers::check_min_arg_count; +use crate::*; + +struct LinuxFutex { + futex: FutexRef, +} + +/// Implementation of the SYS_futex syscall. +/// `args` is the arguments *including* the syscall number. +pub fn futex<'tcx>( + ecx: &mut MiriInterpCx<'tcx>, + args: &[OpTy<'tcx>], + dest: &MPlaceTy<'tcx>, +) -> InterpResult<'tcx> { + // The amount of arguments used depends on the type of futex operation. + // The full futex syscall takes six arguments (excluding the syscall + // number), which is also the maximum amount of arguments a linux syscall + // can take on most architectures. + // However, not all futex operations use all six arguments. The unused ones + // may or may not be left out from the `syscall()` call. + // Therefore we don't use `check_arg_count` here, but only check for the + // number of arguments to fall within a range. + let [_, addr, op, val] = check_min_arg_count("`syscall(SYS_futex, ...)`", args)?; + + // The first three arguments (after the syscall number itself) are the same to all futex operations: + // (int *addr, int op, int val). + // We checked above that these definitely exist. + let addr = ecx.read_pointer(addr)?; + let op = ecx.read_scalar(op)?.to_i32()?; + let val = ecx.read_scalar(val)?.to_i32()?; + + // This is a vararg function so we have to bring our own type for this pointer. + let addr = ecx.ptr_to_mplace(addr, ecx.machine.layouts.i32); + + let futex_private = ecx.eval_libc_i32("FUTEX_PRIVATE_FLAG"); + let futex_wait = ecx.eval_libc_i32("FUTEX_WAIT"); + let futex_wait_bitset = ecx.eval_libc_i32("FUTEX_WAIT_BITSET"); + let futex_wake = ecx.eval_libc_i32("FUTEX_WAKE"); + let futex_wake_bitset = ecx.eval_libc_i32("FUTEX_WAKE_BITSET"); + let futex_realtime = ecx.eval_libc_i32("FUTEX_CLOCK_REALTIME"); + + // FUTEX_PRIVATE enables an optimization that stops it from working across processes. + // Miri doesn't support that anyway, so we ignore that flag. + match op & !futex_private { + // FUTEX_WAIT: (int *addr, int op = FUTEX_WAIT, int val, const timespec *timeout) + // Blocks the thread if *addr still equals val. Wakes up when FUTEX_WAKE is called on the same address, + // or *timeout expires. `timeout == null` for an infinite timeout. + // + // FUTEX_WAIT_BITSET: (int *addr, int op = FUTEX_WAIT_BITSET, int val, const timespec *timeout, int *_ignored, unsigned int bitset) + // This is identical to FUTEX_WAIT, except: + // - The timeout is absolute rather than relative. + // - You can specify the bitset to selecting what WAKE operations to respond to. + op if op & !futex_realtime == futex_wait || op & !futex_realtime == futex_wait_bitset => { + let wait_bitset = op & !futex_realtime == futex_wait_bitset; + + let (timeout, bitset) = if wait_bitset { + let [_, _, _, _, timeout, uaddr2, bitset] = + check_min_arg_count("`syscall(SYS_futex, FUTEX_WAIT_BITSET, ...)`", args)?; + let _timeout = ecx.read_pointer(timeout)?; + let _uaddr2 = ecx.read_pointer(uaddr2)?; + (timeout, ecx.read_scalar(bitset)?.to_u32()?) + } else { + let [_, _, _, _, timeout] = + check_min_arg_count("`syscall(SYS_futex, FUTEX_WAIT, ...)`", args)?; + (timeout, u32::MAX) + }; + + if bitset == 0 { + return ecx.set_last_error_and_return(LibcError("EINVAL"), dest); + } + + let timeout = ecx.deref_pointer_as(timeout, ecx.libc_ty_layout("timespec"))?; + let timeout = if ecx.ptr_is_null(timeout.ptr())? { + None + } else { + let duration = match ecx.read_timespec(&timeout)? { + Some(duration) => duration, + None => { + return ecx.set_last_error_and_return(LibcError("EINVAL"), dest); + } + }; + let timeout_clock = if op & futex_realtime == futex_realtime { + ecx.check_no_isolation( + "`futex` syscall with `op=FUTEX_WAIT` and non-null timeout with `FUTEX_CLOCK_REALTIME`", + )?; + TimeoutClock::RealTime + } else { + TimeoutClock::Monotonic + }; + let timeout_anchor = if wait_bitset { + // FUTEX_WAIT_BITSET uses an absolute timestamp. + TimeoutAnchor::Absolute + } else { + // FUTEX_WAIT uses a relative timestamp. + TimeoutAnchor::Relative + }; + Some((timeout_clock, timeout_anchor, duration)) + }; + // There may be a concurrent thread changing the value of addr + // and then invoking the FUTEX_WAKE syscall. It is critical that the + // effects of this and the other thread are correctly observed, + // otherwise we will deadlock. + // + // There are two scenarios to consider, depending on whether WAIT or WAKE goes first: + // 1. If we (FUTEX_WAIT) execute first, we'll push ourselves into the waiters queue and + // go to sleep. They (FUTEX_WAKE) will see us in the queue and wake us up. It doesn't + // matter how the addr write is ordered. + // 2. If they (FUTEX_WAKE) execute first, that means the addr write is also before us + // (FUTEX_WAIT). It is crucial that we observe addr's new value. If we see an + // outdated value that happens to equal the expected val, then we'll put ourselves to + // sleep with no one to wake us up, so we end up with a deadlock. This is prevented + // by having a SeqCst fence inside FUTEX_WAKE syscall, and another SeqCst fence here + // in FUTEX_WAIT. The atomic read on addr after the SeqCst fence is guaranteed not to + // see any value older than the addr write immediately before calling FUTEX_WAKE. + // We'll see futex_val != val and return without sleeping. + // + // Note that the fences do not create any happens-before relationship. + // The read sees the write immediately before the fence not because + // one happens after the other, but is instead due to a guarantee unique + // to SeqCst fences that restricts what an atomic read placed AFTER the + // fence can see. The read still has to be atomic, otherwise it's a data + // race. This guarantee cannot be achieved with acquire-release fences + // since they only talk about reads placed BEFORE a fence - and places + // no restrictions on what the read itself can see, only that there is + // a happens-before between the fences IF the read happens to see the + // right value. This is useless to us, since we need the read itself + // to see an up-to-date value. + // + // The above case distinction is valid since both FUTEX_WAIT and FUTEX_WAKE + // contain a SeqCst fence, therefore inducing a total order between the operations. + // It is also critical that the fence, the atomic load, and the comparison in FUTEX_WAIT + // altogether happen atomically. If the other thread's fence in FUTEX_WAKE + // gets interleaved after our fence, then we lose the guarantee on the + // atomic load being up-to-date; if the other thread's write on addr and FUTEX_WAKE + // call are interleaved after the load but before the comparison, then we get a TOCTOU + // race condition, and go to sleep thinking the other thread will wake us up, + // even though they have already finished. + // + // Thankfully, preemptions cannot happen inside a Miri shim, so we do not need to + // do anything special to guarantee fence-load-comparison atomicity. + ecx.atomic_fence(AtomicFenceOrd::SeqCst)?; + // Read an `i32` through the pointer, regardless of any wrapper types. + // It's not uncommon for `addr` to be passed as another type than `*mut i32`, such as `*const AtomicI32`. + // We do an acquire read -- it only seems reasonable that if we observe a value here, we + // actually establish an ordering with that value. + let futex_val = ecx.read_scalar_atomic(&addr, AtomicReadOrd::Acquire)?.to_i32()?; + if val == futex_val { + // The value still matches, so we block the thread and make it wait for FUTEX_WAKE. + + // This cannot fail since we already did an atomic acquire read on that pointer. + // Acquire reads are only allowed on mutable memory. + let futex_ref = ecx + .get_sync_or_init(addr.ptr(), |_| LinuxFutex { futex: Default::default() }) + .unwrap() + .futex + .clone(); + + ecx.futex_wait( + futex_ref, + bitset, + timeout, + Scalar::from_target_isize(0, ecx), // retval_succ + Scalar::from_target_isize(-1, ecx), // retval_timeout + dest.clone(), + LibcError("ETIMEDOUT"), // errno_timeout + ); + } else { + // The futex value doesn't match the expected value, so we return failure + // right away without sleeping: -1 and errno set to EAGAIN. + return ecx.set_last_error_and_return(LibcError("EAGAIN"), dest); + } + } + // FUTEX_WAKE: (int *addr, int op = FUTEX_WAKE, int val) + // Wakes at most `val` threads waiting on the futex at `addr`. + // Returns the amount of threads woken up. + // Does not access the futex value at *addr. + // FUTEX_WAKE_BITSET: (int *addr, int op = FUTEX_WAKE, int val, const timespect *_unused, int *_unused, unsigned int bitset) + // Same as FUTEX_WAKE, but allows you to specify a bitset to select which threads to wake up. + op if op == futex_wake || op == futex_wake_bitset => { + let Some(futex_ref) = + ecx.get_sync_or_init(addr.ptr(), |_| LinuxFutex { futex: Default::default() }) + else { + // No AllocId, or no live allocation at that AllocId. + // Return an error code. (That seems nicer than silently doing something non-intuitive.) + // This means that if an address gets reused by a new allocation, + // we'll use an independent futex queue for this... that seems acceptable. + return ecx.set_last_error_and_return(LibcError("EFAULT"), dest); + }; + let futex_ref = futex_ref.futex.clone(); + + let bitset = if op == futex_wake_bitset { + let [_, _, _, _, timeout, uaddr2, bitset] = + check_min_arg_count("`syscall(SYS_futex, FUTEX_WAKE_BITSET, ...)`", args)?; + let _timeout = ecx.read_pointer(timeout)?; + let _uaddr2 = ecx.read_pointer(uaddr2)?; + ecx.read_scalar(bitset)?.to_u32()? + } else { + u32::MAX + }; + if bitset == 0 { + return ecx.set_last_error_and_return(LibcError("EINVAL"), dest); + } + // Together with the SeqCst fence in futex_wait, this makes sure that futex_wait + // will see the latest value on addr which could be changed by our caller + // before doing the syscall. + ecx.atomic_fence(AtomicFenceOrd::SeqCst)?; + let mut n = 0; + #[expect(clippy::arithmetic_side_effects)] + for _ in 0..val { + if ecx.futex_wake(&futex_ref, bitset)? { + n += 1; + } else { + break; + } + } + ecx.write_scalar(Scalar::from_target_isize(n, ecx), dest)?; + } + op => throw_unsup_format!("Miri does not support `futex` syscall with op={}", op), + } + + interp_ok(()) +} diff --git a/src/tools/miri/src/shims/unix/linux_like/syscall.rs b/src/tools/miri/src/shims/unix/linux_like/syscall.rs new file mode 100644 index 00000000000..e9a32a26326 --- /dev/null +++ b/src/tools/miri/src/shims/unix/linux_like/syscall.rs @@ -0,0 +1,63 @@ +use rustc_abi::ExternAbi; +use rustc_span::Symbol; + +use crate::helpers::check_min_arg_count; +use crate::shims::unix::linux_like::eventfd::EvalContextExt as _; +use crate::shims::unix::linux_like::sync::futex; +use crate::*; + +pub fn syscall<'tcx>( + ecx: &mut MiriInterpCx<'tcx>, + link_name: Symbol, + abi: ExternAbi, + args: &[OpTy<'tcx>], + dest: &MPlaceTy<'tcx>, +) -> InterpResult<'tcx> { + // We do not use `check_shim` here because `syscall` is variadic. The argument + // count is checked bellow. + ecx.check_abi_and_shim_symbol_clash(abi, ExternAbi::C { unwind: false }, link_name)?; + // The syscall variadic function is legal to call with more arguments than needed, + // extra arguments are simply ignored. The important check is that when we use an + // argument, we have to also check all arguments *before* it to ensure that they + // have the right type. + + let sys_getrandom = ecx.eval_libc("SYS_getrandom").to_target_usize(ecx)?; + let sys_futex = ecx.eval_libc("SYS_futex").to_target_usize(ecx)?; + let sys_eventfd2 = ecx.eval_libc("SYS_eventfd2").to_target_usize(ecx)?; + + let [op] = check_min_arg_count("syscall", args)?; + match ecx.read_target_usize(op)? { + // `libc::syscall(NR_GETRANDOM, buf.as_mut_ptr(), buf.len(), GRND_NONBLOCK)` + // is called if a `HashMap` is created the regular way (e.g. HashMap). + num if num == sys_getrandom => { + // Used by getrandom 0.1 + // The first argument is the syscall id, so skip over it. + let [_, ptr, len, flags] = check_min_arg_count("syscall(SYS_getrandom, ...)", args)?; + + let ptr = ecx.read_pointer(ptr)?; + let len = ecx.read_target_usize(len)?; + // The only supported flags are GRND_RANDOM and GRND_NONBLOCK, + // neither of which have any effect on our current PRNG. + // See for a discussion of argument sizes. + let _flags = ecx.read_scalar(flags)?.to_i32()?; + + ecx.gen_random(ptr, len)?; + ecx.write_scalar(Scalar::from_target_usize(len, ecx), dest)?; + } + // `futex` is used by some synchronization primitives. + num if num == sys_futex => { + futex(ecx, args, dest)?; + } + num if num == sys_eventfd2 => { + let [_, initval, flags] = check_min_arg_count("syscall(SYS_evetfd2, ...)", args)?; + + let result = ecx.eventfd(initval, flags)?; + ecx.write_int(result.to_i32()?, dest)?; + } + num => { + throw_unsup_format!("syscall: unsupported syscall number {num}"); + } + }; + + interp_ok(()) +} diff --git a/src/tools/miri/src/shims/unix/mod.rs b/src/tools/miri/src/shims/unix/mod.rs index c8c25c636ee..0620b57753a 100644 --- a/src/tools/miri/src/shims/unix/mod.rs +++ b/src/tools/miri/src/shims/unix/mod.rs @@ -11,6 +11,7 @@ mod unnamed_socket; mod android; mod freebsd; mod linux; +mod linux_like; mod macos; mod solarish; @@ -18,7 +19,7 @@ mod solarish; pub use self::env::{EvalContextExt as _, UnixEnvVars}; pub use self::fd::{EvalContextExt as _, FdTable, FileDescription}; pub use self::fs::{DirTable, EvalContextExt as _}; -pub use self::linux::epoll::EpollInterestTable; +pub use self::linux_like::epoll::EpollInterestTable; pub use self::mem::EvalContextExt as _; pub use self::sync::EvalContextExt as _; pub use self::thread::{EvalContextExt as _, ThreadNameResult}; diff --git a/src/tools/miri/src/shims/unix/unnamed_socket.rs b/src/tools/miri/src/shims/unix/unnamed_socket.rs index 36575f4b5fb..232f4500dba 100644 --- a/src/tools/miri/src/shims/unix/unnamed_socket.rs +++ b/src/tools/miri/src/shims/unix/unnamed_socket.rs @@ -11,7 +11,7 @@ use rustc_abi::Size; use crate::concurrency::VClock; use crate::shims::unix::fd::{FileDescriptionRef, WeakFileDescriptionRef}; -use crate::shims::unix::linux::epoll::{EpollReadyEvents, EvalContextExt as _}; +use crate::shims::unix::linux_like::epoll::{EpollReadyEvents, EvalContextExt as _}; use crate::shims::unix::*; use crate::*; -- cgit 1.4.1-3-g733a5