diff options
Diffstat (limited to 'library/std/src')
| -rw-r--r-- | library/std/src/collections/hash/map.rs | 6 | ||||
| -rw-r--r-- | library/std/src/error.rs | 4 | ||||
| -rw-r--r-- | library/std/src/fs.rs | 2 | ||||
| -rw-r--r-- | library/std/src/io/copy.rs | 88 | ||||
| -rw-r--r-- | library/std/src/io/impls.rs | 14 | ||||
| -rw-r--r-- | library/std/src/io/mod.rs | 13 | ||||
| -rw-r--r-- | library/std/src/io/stdio.rs | 185 | ||||
| -rw-r--r-- | library/std/src/io/tests.rs | 2 | ||||
| -rw-r--r-- | library/std/src/io/util.rs | 73 | ||||
| -rw-r--r-- | library/std/src/lib.rs | 4 | ||||
| -rw-r--r-- | library/std/src/panicking.rs | 12 | ||||
| -rw-r--r-- | library/std/src/sys/unix/fs.rs | 85 | ||||
| -rw-r--r-- | library/std/src/sys/unix/kernel_copy.rs | 603 | ||||
| -rw-r--r-- | library/std/src/sys/unix/kernel_copy/tests.rs | 213 | ||||
| -rw-r--r-- | library/std/src/sys/unix/mod.rs | 2 | ||||
| -rw-r--r-- | library/std/src/sys/wasm/mutex_atomics.rs | 2 | ||||
| -rw-r--r-- | library/std/src/thread/mod.rs | 6 |
17 files changed, 993 insertions, 321 deletions
diff --git a/library/std/src/collections/hash/map.rs b/library/std/src/collections/hash/map.rs index fa229251703..27d90e66137 100644 --- a/library/std/src/collections/hash/map.rs +++ b/library/std/src/collections/hash/map.rs @@ -34,8 +34,8 @@ use crate::sys; /// attacks such as HashDoS. /// /// The hashing algorithm can be replaced on a per-`HashMap` basis using the -/// [`default`], [`with_hasher`], and [`with_capacity_and_hasher`] methods. Many -/// alternative algorithms are available on crates.io, such as the [`fnv`] crate. +/// [`default`], [`with_hasher`], and [`with_capacity_and_hasher`] methods. +/// There are many alternative [hashing algorithms available on crates.io]. /// /// It is required that the keys implement the [`Eq`] and [`Hash`] traits, although /// this can frequently be achieved by using `#[derive(PartialEq, Eq, Hash)]`. @@ -57,6 +57,7 @@ use crate::sys; /// The original C++ version of SwissTable can be found [here], and this /// [CppCon talk] gives an overview of how the algorithm works. /// +/// [hashing algorithms available on crates.io]: https://crates.io/keywords/hasher /// [SwissTable]: https://abseil.io/blog/20180927-swisstables /// [here]: https://github.com/abseil/abseil-cpp/blob/master/absl/container/internal/raw_hash_set.h /// [CppCon talk]: https://www.youtube.com/watch?v=ncHmEUmJZf4 @@ -154,7 +155,6 @@ use crate::sys; /// [`default`]: Default::default /// [`with_hasher`]: Self::with_hasher /// [`with_capacity_and_hasher`]: Self::with_capacity_and_hasher -/// [`fnv`]: https://crates.io/crates/fnv /// /// ``` /// use std::collections::HashMap; diff --git a/library/std/src/error.rs b/library/std/src/error.rs index 5771ca758af..0044e59d697 100644 --- a/library/std/src/error.rs +++ b/library/std/src/error.rs @@ -19,7 +19,7 @@ mod tests; use core::array; use core::convert::Infallible; -use crate::alloc::{AllocError, LayoutErr}; +use crate::alloc::{AllocError, LayoutError}; use crate::any::TypeId; use crate::backtrace::Backtrace; use crate::borrow::Cow; @@ -390,7 +390,7 @@ impl Error for ! {} impl Error for AllocError {} #[stable(feature = "alloc_layout", since = "1.28.0")] -impl Error for LayoutErr {} +impl Error for LayoutError {} #[stable(feature = "rust1", since = "1.0.0")] impl Error for str::ParseBoolError { diff --git a/library/std/src/fs.rs b/library/std/src/fs.rs index c256f556b3c..a4123cc15b8 100644 --- a/library/std/src/fs.rs +++ b/library/std/src/fs.rs @@ -1656,7 +1656,7 @@ pub fn rename<P: AsRef<Path>, Q: AsRef<Path>>(from: P, to: Q) -> io::Result<()> /// the length of the `to` file as reported by `metadata`. /// /// If you’re wanting to copy the contents of one file to another and you’re -/// working with [`File`]s, see the [`io::copy`] function. +/// working with [`File`]s, see the [`io::copy()`] function. /// /// # Platform-specific behavior /// diff --git a/library/std/src/io/copy.rs b/library/std/src/io/copy.rs new file mode 100644 index 00000000000..b88bca2f2b4 --- /dev/null +++ b/library/std/src/io/copy.rs @@ -0,0 +1,88 @@ +use crate::io::{self, ErrorKind, Read, Write}; +use crate::mem::MaybeUninit; + +/// Copies the entire contents of a reader into a writer. +/// +/// This function will continuously read data from `reader` and then +/// write it into `writer` in a streaming fashion until `reader` +/// returns EOF. +/// +/// On success, the total number of bytes that were copied from +/// `reader` to `writer` is returned. +/// +/// If you’re wanting to copy the contents of one file to another and you’re +/// working with filesystem paths, see the [`fs::copy`] function. +/// +/// [`fs::copy`]: crate::fs::copy +/// +/// # Errors +/// +/// This function will return an error immediately if any call to [`read`] or +/// [`write`] returns an error. All instances of [`ErrorKind::Interrupted`] are +/// handled by this function and the underlying operation is retried. +/// +/// [`read`]: Read::read +/// [`write`]: Write::write +/// +/// # Examples +/// +/// ``` +/// use std::io; +/// +/// fn main() -> io::Result<()> { +/// let mut reader: &[u8] = b"hello"; +/// let mut writer: Vec<u8> = vec![]; +/// +/// io::copy(&mut reader, &mut writer)?; +/// +/// assert_eq!(&b"hello"[..], &writer[..]); +/// Ok(()) +/// } +/// ``` +#[stable(feature = "rust1", since = "1.0.0")] +pub fn copy<R: ?Sized, W: ?Sized>(reader: &mut R, writer: &mut W) -> io::Result<u64> +where + R: Read, + W: Write, +{ + cfg_if::cfg_if! { + if #[cfg(any(target_os = "linux", target_os = "android"))] { + crate::sys::kernel_copy::copy_spec(reader, writer) + } else { + generic_copy(reader, writer) + } + } +} + +/// The general read-write-loop implementation of +/// `io::copy` that is used when specializations are not available or not applicable. +pub(crate) fn generic_copy<R: ?Sized, W: ?Sized>(reader: &mut R, writer: &mut W) -> io::Result<u64> +where + R: Read, + W: Write, +{ + let mut buf = MaybeUninit::<[u8; super::DEFAULT_BUF_SIZE]>::uninit(); + // FIXME: #42788 + // + // - This creates a (mut) reference to a slice of + // _uninitialized_ integers, which is **undefined behavior** + // + // - Only the standard library gets to soundly "ignore" this, + // based on its privileged knowledge of unstable rustc + // internals; + unsafe { + reader.initializer().initialize(buf.assume_init_mut()); + } + + let mut written = 0; + loop { + let len = match reader.read(unsafe { buf.assume_init_mut() }) { + Ok(0) => return Ok(written), + Ok(len) => len, + Err(ref e) if e.kind() == ErrorKind::Interrupted => continue, + Err(e) => return Err(e), + }; + writer.write_all(unsafe { &buf.assume_init_ref()[..len] })?; + written += len as u64; + } +} diff --git a/library/std/src/io/impls.rs b/library/std/src/io/impls.rs index 66426101d27..6b3c86cb0df 100644 --- a/library/std/src/io/impls.rs +++ b/library/std/src/io/impls.rs @@ -209,20 +209,6 @@ impl<B: BufRead + ?Sized> BufRead for Box<B> { } } -// Used by panicking::default_hook -#[cfg(test)] -/// This impl is only used by printing logic, so any error returned is always -/// of kind `Other`, and should be ignored. -impl Write for dyn ::realstd::io::LocalOutput { - fn write(&mut self, buf: &[u8]) -> io::Result<usize> { - (*self).write(buf).map_err(|_| ErrorKind::Other.into()) - } - - fn flush(&mut self) -> io::Result<()> { - (*self).flush().map_err(|_| ErrorKind::Other.into()) - } -} - // ============================================================================= // In-memory buffer implementations diff --git a/library/std/src/io/mod.rs b/library/std/src/io/mod.rs index e6efe6ec57e..703c3755b63 100644 --- a/library/std/src/io/mod.rs +++ b/library/std/src/io/mod.rs @@ -266,24 +266,25 @@ pub use self::buffered::IntoInnerError; #[stable(feature = "rust1", since = "1.0.0")] pub use self::buffered::{BufReader, BufWriter, LineWriter}; #[stable(feature = "rust1", since = "1.0.0")] +pub use self::copy::copy; +#[stable(feature = "rust1", since = "1.0.0")] pub use self::cursor::Cursor; #[stable(feature = "rust1", since = "1.0.0")] pub use self::error::{Error, ErrorKind, Result}; +#[unstable(feature = "internal_output_capture", issue = "none")] +#[doc(no_inline, hidden)] +pub use self::stdio::set_output_capture; #[stable(feature = "rust1", since = "1.0.0")] pub use self::stdio::{stderr, stdin, stdout, Stderr, Stdin, Stdout}; #[stable(feature = "rust1", since = "1.0.0")] pub use self::stdio::{StderrLock, StdinLock, StdoutLock}; #[unstable(feature = "print_internals", issue = "none")] pub use self::stdio::{_eprint, _print}; -#[unstable(feature = "libstd_io_internals", issue = "42788")] -#[doc(no_inline, hidden)] -pub use self::stdio::{set_panic, set_print, LocalOutput}; #[stable(feature = "rust1", since = "1.0.0")] -pub use self::util::{copy, empty, repeat, sink, Empty, Repeat, Sink}; - -pub(crate) use self::stdio::clone_io; +pub use self::util::{empty, repeat, sink, Empty, Repeat, Sink}; mod buffered; +pub(crate) mod copy; mod cursor; mod error; mod impls; diff --git a/library/std/src/io/stdio.rs b/library/std/src/io/stdio.rs index 2eb5fb45286..6ea7704d422 100644 --- a/library/std/src/io/stdio.rs +++ b/library/std/src/io/stdio.rs @@ -5,44 +5,38 @@ mod tests; use crate::io::prelude::*; -use crate::cell::RefCell; +use crate::cell::{Cell, RefCell}; use crate::fmt; use crate::io::{self, BufReader, Initializer, IoSlice, IoSliceMut, LineWriter}; use crate::lazy::SyncOnceCell; use crate::sync::atomic::{AtomicBool, Ordering}; -use crate::sync::{Mutex, MutexGuard}; +use crate::sync::{Arc, Mutex, MutexGuard}; use crate::sys::stdio; use crate::sys_common; use crate::sys_common::remutex::{ReentrantMutex, ReentrantMutexGuard}; -use crate::thread::LocalKey; -thread_local! { - /// Used by the test crate to capture the output of the print! and println! macros. - static LOCAL_STDOUT: RefCell<Option<Box<dyn LocalOutput>>> = { - RefCell::new(None) - } -} +type LocalStream = Arc<Mutex<Vec<u8>>>; thread_local! { - /// Used by the test crate to capture the output of the eprint! and eprintln! macros, and panics. - static LOCAL_STDERR: RefCell<Option<Box<dyn LocalOutput>>> = { - RefCell::new(None) + /// Used by the test crate to capture the output of the print macros and panics. + static OUTPUT_CAPTURE: Cell<Option<LocalStream>> = { + Cell::new(None) } } -/// Flag to indicate LOCAL_STDOUT and/or LOCAL_STDERR is used. +/// Flag to indicate OUTPUT_CAPTURE is used. /// -/// If both are None and were never set on any thread, this flag is set to -/// false, and both LOCAL_STDOUT and LOCAL_STDOUT can be safely ignored on all -/// threads, saving some time and memory registering an unused thread local. +/// If it is None and was never set on any thread, this flag is set to false, +/// and OUTPUT_CAPTURE can be safely ignored on all threads, saving some time +/// and memory registering an unused thread local. /// -/// Note about memory ordering: This contains information about whether two -/// thread local variables might be in use. Although this is a global flag, the +/// Note about memory ordering: This contains information about whether a +/// thread local variable might be in use. Although this is a global flag, the /// memory ordering between threads does not matter: we only want this flag to -/// have a consistent order between set_print/set_panic and print_to *within +/// have a consistent order between set_output_capture and print_to *within /// the same thread*. Within the same thread, things always have a perfectly /// consistent order. So Ordering::Relaxed is fine. -static LOCAL_STREAMS: AtomicBool = AtomicBool::new(false); +static OUTPUT_CAPTURE_USED: AtomicBool = AtomicBool::new(false); /// A handle to a raw instance of the standard input stream of this process. /// @@ -409,6 +403,14 @@ impl Read for Stdin { } } +// only used by platform-dependent io::copy specializations, i.e. unused on some platforms +#[cfg(any(target_os = "linux", target_os = "android"))] +impl StdinLock<'_> { + pub(crate) fn as_mut_buf(&mut self) -> &mut BufReader<impl Read> { + &mut self.inner + } +} + #[stable(feature = "rust1", since = "1.0.0")] impl Read for StdinLock<'_> { fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> { @@ -888,97 +890,24 @@ impl fmt::Debug for StderrLock<'_> { } } -/// A writer than can be cloned to new threads. +/// Sets the thread-local output capture buffer and returns the old one. #[unstable( - feature = "set_stdio", - reason = "this trait may disappear completely or be replaced \ - with a more general mechanism", + feature = "internal_output_capture", + reason = "this function is meant for use in the test crate \ + and may disappear in the future", issue = "none" )] #[doc(hidden)] -pub trait LocalOutput: Write + Send { - fn clone_box(&self) -> Box<dyn LocalOutput>; -} - -/// Resets the thread-local stderr handle to the specified writer -/// -/// This will replace the current thread's stderr handle, returning the old -/// handle. All future calls to `panic!` and friends will emit their output to -/// this specified handle. -/// -/// Note that this does not need to be called for all new threads; the default -/// output handle is to the process's stderr stream. -#[unstable( - feature = "set_stdio", - reason = "this function may disappear completely or be replaced \ - with a more general mechanism", - issue = "none" -)] -#[doc(hidden)] -pub fn set_panic(sink: Option<Box<dyn LocalOutput>>) -> Option<Box<dyn LocalOutput>> { - use crate::mem; - if sink.is_none() && !LOCAL_STREAMS.load(Ordering::Relaxed) { - // LOCAL_STDERR is definitely None since LOCAL_STREAMS is false. +pub fn set_output_capture(sink: Option<LocalStream>) -> Option<LocalStream> { + if sink.is_none() && !OUTPUT_CAPTURE_USED.load(Ordering::Relaxed) { + // OUTPUT_CAPTURE is definitely None since OUTPUT_CAPTURE_USED is false. return None; } - let s = LOCAL_STDERR.with(move |slot| mem::replace(&mut *slot.borrow_mut(), sink)).and_then( - |mut s| { - let _ = s.flush(); - Some(s) - }, - ); - LOCAL_STREAMS.store(true, Ordering::Relaxed); - s + OUTPUT_CAPTURE_USED.store(true, Ordering::Relaxed); + OUTPUT_CAPTURE.with(move |slot| slot.replace(sink)) } -/// Resets the thread-local stdout handle to the specified writer -/// -/// This will replace the current thread's stdout handle, returning the old -/// handle. All future calls to `print!` and friends will emit their output to -/// this specified handle. -/// -/// Note that this does not need to be called for all new threads; the default -/// output handle is to the process's stdout stream. -#[unstable( - feature = "set_stdio", - reason = "this function may disappear completely or be replaced \ - with a more general mechanism", - issue = "none" -)] -#[doc(hidden)] -pub fn set_print(sink: Option<Box<dyn LocalOutput>>) -> Option<Box<dyn LocalOutput>> { - use crate::mem; - if sink.is_none() && !LOCAL_STREAMS.load(Ordering::Relaxed) { - // LOCAL_STDOUT is definitely None since LOCAL_STREAMS is false. - return None; - } - let s = LOCAL_STDOUT.with(move |slot| mem::replace(&mut *slot.borrow_mut(), sink)).and_then( - |mut s| { - let _ = s.flush(); - Some(s) - }, - ); - LOCAL_STREAMS.store(true, Ordering::Relaxed); - s -} - -pub(crate) fn clone_io() -> (Option<Box<dyn LocalOutput>>, Option<Box<dyn LocalOutput>>) { - // Don't waste time when LOCAL_{STDOUT,STDERR} are definitely None. - if !LOCAL_STREAMS.load(Ordering::Relaxed) { - return (None, None); - } - - LOCAL_STDOUT.with(|stdout| { - LOCAL_STDERR.with(|stderr| { - ( - stdout.borrow().as_ref().map(|o| o.clone_box()), - stderr.borrow().as_ref().map(|o| o.clone_box()), - ) - }) - }) -} - -/// Write `args` to output stream `local_s` if possible, `global_s` +/// Write `args` to the capture buffer if enabled and possible, or `global_s` /// otherwise. `label` identifies the stream in a panic message. /// /// This function is used to print error messages, so it takes extra @@ -988,36 +917,26 @@ pub(crate) fn clone_io() -> (Option<Box<dyn LocalOutput>>, Option<Box<dyn LocalO /// thread, it will just fall back to the global stream. /// /// However, if the actual I/O causes an error, this function does panic. -fn print_to<T>( - args: fmt::Arguments<'_>, - local_s: &'static LocalKey<RefCell<Option<Box<dyn LocalOutput>>>>, - global_s: fn() -> T, - label: &str, -) where +fn print_to<T>(args: fmt::Arguments<'_>, global_s: fn() -> T, label: &str) +where T: Write, { - let result = LOCAL_STREAMS - .load(Ordering::Relaxed) - .then(|| { - local_s - .try_with(|s| { - // Note that we completely remove a local sink to write to in case - // our printing recursively panics/prints, so the recursive - // panic/print goes to the global sink instead of our local sink. - let prev = s.borrow_mut().take(); - if let Some(mut w) = prev { - let result = w.write_fmt(args); - *s.borrow_mut() = Some(w); - return result; - } - global_s().write_fmt(args) - }) - .ok() - }) - .flatten() - .unwrap_or_else(|| global_s().write_fmt(args)); - - if let Err(e) = result { + if OUTPUT_CAPTURE_USED.load(Ordering::Relaxed) + && OUTPUT_CAPTURE.try_with(|s| { + // Note that we completely remove a local sink to write to in case + // our printing recursively panics/prints, so the recursive + // panic/print goes to the global sink instead of our local sink. + s.take().map(|w| { + let _ = w.lock().unwrap_or_else(|e| e.into_inner()).write_fmt(args); + s.set(Some(w)); + }) + }) == Ok(Some(())) + { + // Succesfully wrote to capture buffer. + return; + } + + if let Err(e) = global_s().write_fmt(args) { panic!("failed printing to {}: {}", label, e); } } @@ -1030,7 +949,7 @@ fn print_to<T>( #[doc(hidden)] #[cfg(not(test))] pub fn _print(args: fmt::Arguments<'_>) { - print_to(args, &LOCAL_STDOUT, stdout, "stdout"); + print_to(args, stdout, "stdout"); } #[unstable( @@ -1041,7 +960,7 @@ pub fn _print(args: fmt::Arguments<'_>) { #[doc(hidden)] #[cfg(not(test))] pub fn _eprint(args: fmt::Arguments<'_>) { - print_to(args, &LOCAL_STDERR, stderr, "stderr"); + print_to(args, stderr, "stderr"); } #[cfg(test)] diff --git a/library/std/src/io/tests.rs b/library/std/src/io/tests.rs index 913b28538b7..f176c2f088c 100644 --- a/library/std/src/io/tests.rs +++ b/library/std/src/io/tests.rs @@ -1,7 +1,7 @@ use super::{repeat, Cursor, SeekFrom}; use crate::cmp::{self, min}; -use crate::io::prelude::*; use crate::io::{self, IoSlice, IoSliceMut}; +use crate::io::{BufRead, Read, Seek, Write}; use crate::ops::Deref; #[test] diff --git a/library/std/src/io/util.rs b/library/std/src/io/util.rs index 2b1f371129e..db845457c96 100644 --- a/library/std/src/io/util.rs +++ b/library/std/src/io/util.rs @@ -4,78 +4,7 @@ mod tests; use crate::fmt; -use crate::io::{self, BufRead, ErrorKind, Initializer, IoSlice, IoSliceMut, Read, Write}; -use crate::mem::MaybeUninit; - -/// Copies the entire contents of a reader into a writer. -/// -/// This function will continuously read data from `reader` and then -/// write it into `writer` in a streaming fashion until `reader` -/// returns EOF. -/// -/// On success, the total number of bytes that were copied from -/// `reader` to `writer` is returned. -/// -/// If you’re wanting to copy the contents of one file to another and you’re -/// working with filesystem paths, see the [`fs::copy`] function. -/// -/// [`fs::copy`]: crate::fs::copy -/// -/// # Errors -/// -/// This function will return an error immediately if any call to [`read`] or -/// [`write`] returns an error. All instances of [`ErrorKind::Interrupted`] are -/// handled by this function and the underlying operation is retried. -/// -/// [`read`]: Read::read -/// [`write`]: Write::write -/// -/// # Examples -/// -/// ``` -/// use std::io; -/// -/// fn main() -> io::Result<()> { -/// let mut reader: &[u8] = b"hello"; -/// let mut writer: Vec<u8> = vec![]; -/// -/// io::copy(&mut reader, &mut writer)?; -/// -/// assert_eq!(&b"hello"[..], &writer[..]); -/// Ok(()) -/// } -/// ``` -#[stable(feature = "rust1", since = "1.0.0")] -pub fn copy<R: ?Sized, W: ?Sized>(reader: &mut R, writer: &mut W) -> io::Result<u64> -where - R: Read, - W: Write, -{ - let mut buf = MaybeUninit::<[u8; super::DEFAULT_BUF_SIZE]>::uninit(); - // FIXME: #42788 - // - // - This creates a (mut) reference to a slice of - // _uninitialized_ integers, which is **undefined behavior** - // - // - Only the standard library gets to soundly "ignore" this, - // based on its privileged knowledge of unstable rustc - // internals; - unsafe { - reader.initializer().initialize(buf.assume_init_mut()); - } - - let mut written = 0; - loop { - let len = match reader.read(unsafe { buf.assume_init_mut() }) { - Ok(0) => return Ok(written), - Ok(len) => len, - Err(ref e) if e.kind() == ErrorKind::Interrupted => continue, - Err(e) => return Err(e), - }; - writer.write_all(unsafe { &buf.assume_init_ref()[..len] })?; - written += len as u64; - } -} +use crate::io::{self, BufRead, Initializer, IoSlice, IoSliceMut, Read, Write}; /// A reader which is always at EOF. /// diff --git a/library/std/src/lib.rs b/library/std/src/lib.rs index fa2608e6fb6..ac11fcf7329 100644 --- a/library/std/src/lib.rs +++ b/library/std/src/lib.rs @@ -207,7 +207,7 @@ // std may use features in a platform-specific way #![allow(unused_features)] #![cfg_attr(not(bootstrap), feature(rustc_allow_const_fn_unstable))] -#![cfg_attr(test, feature(print_internals, set_stdio, update_panic_count))] +#![cfg_attr(test, feature(internal_output_capture, print_internals, update_panic_count))] #![cfg_attr( all(target_vendor = "fortanix", target_env = "sgx"), feature(slice_index_methods, coerce_unsized, sgx_platform) @@ -298,6 +298,7 @@ #![feature(raw)] #![feature(raw_ref_macros)] #![feature(ready_macro)] +#![feature(refcell_take)] #![feature(rustc_attrs)] #![feature(rustc_private)] #![feature(shrink_to)] @@ -317,6 +318,7 @@ #![feature(toowned_clone_into)] #![feature(total_cmp)] #![feature(trace_macros)] +#![feature(try_blocks)] #![feature(try_reserve)] #![feature(unboxed_closures)] #![feature(unsafe_block_in_unsafe_fn)] diff --git a/library/std/src/panicking.rs b/library/std/src/panicking.rs index fbbc61f4e60..8ba3feccb6b 100644 --- a/library/std/src/panicking.rs +++ b/library/std/src/panicking.rs @@ -24,11 +24,11 @@ use crate::sys_common::{thread_info, util}; use crate::thread; #[cfg(not(test))] -use crate::io::set_panic; +use crate::io::set_output_capture; // make sure to use the stderr output configured // by libtest in the real copy of std #[cfg(test)] -use realstd::io::set_panic; +use realstd::io::set_output_capture; // Binary interface to the panic runtime that the standard library depends on. // @@ -218,11 +218,9 @@ fn default_hook(info: &PanicInfo<'_>) { } }; - if let Some(mut local) = set_panic(None) { - // NB. In `cfg(test)` this uses the forwarding impl - // for `dyn ::realstd::io::LocalOutput`. - write(&mut local); - set_panic(Some(local)); + if let Some(local) = set_output_capture(None) { + write(&mut *local.lock().unwrap_or_else(|e| e.into_inner())); + set_output_capture(Some(local)); } else if let Some(mut out) = panic_output() { write(&mut out); } diff --git a/library/std/src/sys/unix/fs.rs b/library/std/src/sys/unix/fs.rs index 96594095cc3..13cf930379c 100644 --- a/library/std/src/sys/unix/fs.rs +++ b/library/std/src/sys/unix/fs.rs @@ -1204,88 +1204,19 @@ pub fn copy(from: &Path, to: &Path) -> io::Result<u64> { #[cfg(any(target_os = "linux", target_os = "android"))] pub fn copy(from: &Path, to: &Path) -> io::Result<u64> { - use crate::cmp; - use crate::sync::atomic::{AtomicBool, Ordering}; - - // Kernel prior to 4.5 don't have copy_file_range - // We store the availability in a global to avoid unnecessary syscalls - static HAS_COPY_FILE_RANGE: AtomicBool = AtomicBool::new(true); - - unsafe fn copy_file_range( - fd_in: libc::c_int, - off_in: *mut libc::loff_t, - fd_out: libc::c_int, - off_out: *mut libc::loff_t, - len: libc::size_t, - flags: libc::c_uint, - ) -> libc::c_long { - libc::syscall(libc::SYS_copy_file_range, fd_in, off_in, fd_out, off_out, len, flags) - } - let (mut reader, reader_metadata) = open_from(from)?; let max_len = u64::MAX; let (mut writer, _) = open_to_and_set_permissions(to, reader_metadata)?; - let has_copy_file_range = HAS_COPY_FILE_RANGE.load(Ordering::Relaxed); - let mut written = 0u64; - while written < max_len { - let copy_result = if has_copy_file_range { - let bytes_to_copy = cmp::min(max_len - written, usize::MAX as u64) as usize; - let copy_result = unsafe { - // We actually don't have to adjust the offsets, - // because copy_file_range adjusts the file offset automatically - cvt(copy_file_range( - reader.as_raw_fd(), - ptr::null_mut(), - writer.as_raw_fd(), - ptr::null_mut(), - bytes_to_copy, - 0, - )) - }; - if let Err(ref copy_err) = copy_result { - match copy_err.raw_os_error() { - Some(libc::ENOSYS | libc::EPERM | libc::EOPNOTSUPP) => { - HAS_COPY_FILE_RANGE.store(false, Ordering::Relaxed); - } - _ => {} - } - } - copy_result - } else { - Err(io::Error::from_raw_os_error(libc::ENOSYS)) - }; - match copy_result { - Ok(0) if written == 0 => { - // fallback to work around several kernel bugs where copy_file_range will fail to - // copy any bytes and return 0 instead of an error if - // - reading virtual files from the proc filesystem which appear to have 0 size - // but are not empty. noted in coreutils to affect kernels at least up to 5.6.19. - // - copying from an overlay filesystem in docker. reported to occur on fedora 32. - return io::copy(&mut reader, &mut writer); - } - Ok(0) => return Ok(written), // reached EOF - Ok(ret) => written += ret as u64, - Err(err) => { - match err.raw_os_error() { - Some( - libc::ENOSYS | libc::EXDEV | libc::EINVAL | libc::EPERM | libc::EOPNOTSUPP, - ) => { - // Try fallback io::copy if either: - // - Kernel version is < 4.5 (ENOSYS) - // - Files are mounted on different fs (EXDEV) - // - copy_file_range is broken in various ways on RHEL/CentOS 7 (EOPNOTSUPP) - // - copy_file_range is disallowed, for example by seccomp (EPERM) - // - copy_file_range cannot be used with pipes or device nodes (EINVAL) - assert_eq!(written, 0); - return io::copy(&mut reader, &mut writer); - } - _ => return Err(err), - } - } - } + use super::kernel_copy::{copy_regular_files, CopyResult}; + + match copy_regular_files(reader.as_raw_fd(), writer.as_raw_fd(), max_len) { + CopyResult::Ended(result) => result, + CopyResult::Fallback(written) => match io::copy::generic_copy(&mut reader, &mut writer) { + Ok(bytes) => Ok(bytes + written), + Err(e) => Err(e), + }, } - Ok(written) } #[cfg(any(target_os = "macos", target_os = "ios"))] diff --git a/library/std/src/sys/unix/kernel_copy.rs b/library/std/src/sys/unix/kernel_copy.rs new file mode 100644 index 00000000000..ac2fcfcb53f --- /dev/null +++ b/library/std/src/sys/unix/kernel_copy.rs @@ -0,0 +1,603 @@ +//! This module contains specializations that can offload `io::copy()` operations on file descriptor +//! containing types (`File`, `TcpStream`, etc.) to more efficient syscalls than `read(2)` and `write(2)`. +//! +//! Specialization is only applied to wholly std-owned types so that user code can't observe +//! that the `Read` and `Write` traits are not used. +//! +//! Since a copy operation involves a reader and writer side where each can consist of different types +//! and also involve generic wrappers (e.g. `Take`, `BufReader`) it is not practical to specialize +//! a single method on all possible combinations. +//! +//! Instead readers and writers are handled separately by the `CopyRead` and `CopyWrite` specialization +//! traits and then specialized on by the `Copier::copy` method. +//! +//! `Copier` uses the specialization traits to unpack the underlying file descriptors and +//! additional prerequisites and constraints imposed by the wrapper types. +//! +//! Once it has obtained all necessary pieces and brought any wrapper types into a state where they +//! can be safely bypassed it will attempt to use the `copy_file_range(2)`, +//! `sendfile(2)` or `splice(2)` syscalls to move data directly between file descriptors. +//! Since those syscalls have requirements that cannot be fully checked in advance and +//! gathering additional information about file descriptors would require additional syscalls +//! anyway it simply attempts to use them one after another (guided by inaccurate hints) to +//! figure out which one works and and falls back to the generic read-write copy loop if none of them +//! does. +//! Once a working syscall is found for a pair of file descriptors it will be called in a loop +//! until the copy operation is completed. +//! +//! Advantages of using these syscalls: +//! +//! * fewer context switches since reads and writes are coalesced into a single syscall +//! and more bytes are transferred per syscall. This translates to higher throughput +//! and fewer CPU cycles, at least for sufficiently large transfers to amortize the initial probing. +//! * `copy_file_range` creates reflink copies on CoW filesystems, thus moving less data and +//! consuming less disk space +//! * `sendfile` and `splice` can perform zero-copy IO under some circumstances while +//! a naive copy loop would move every byte through the CPU. +//! +//! Drawbacks: +//! +//! * copy operations smaller than the default buffer size can under some circumstances, especially +//! on older kernels, incur more syscalls than the naive approach would. As mentioned above +//! the syscall selection is guided by hints to minimize this possibility but they are not perfect. +//! * optimizations only apply to std types. If a user adds a custom wrapper type, e.g. to report +//! progress, they can hit a performance cliff. +//! * complexity + +use crate::cmp::min; +use crate::convert::TryInto; +use crate::fs::{File, Metadata}; +use crate::io::copy::generic_copy; +use crate::io::{ + BufRead, BufReader, BufWriter, Error, Read, Result, StderrLock, StdinLock, StdoutLock, Take, + Write, +}; +use crate::mem::ManuallyDrop; +use crate::net::TcpStream; +use crate::os::unix::fs::FileTypeExt; +use crate::os::unix::io::{AsRawFd, FromRawFd, RawFd}; +use crate::process::{ChildStderr, ChildStdin, ChildStdout}; +use crate::ptr; +use crate::sync::atomic::{AtomicBool, Ordering}; +use crate::sys::cvt; + +#[cfg(test)] +mod tests; + +pub(crate) fn copy_spec<R: Read + ?Sized, W: Write + ?Sized>( + read: &mut R, + write: &mut W, +) -> Result<u64> { + let copier = Copier { read, write }; + SpecCopy::copy(copier) +} + +/// This type represents either the inferred `FileType` of a `RawFd` based on the source +/// type from which it was extracted or the actual metadata +/// +/// The methods on this type only provide hints, due to `AsRawFd` and `FromRawFd` the inferred +/// type may be wrong. +enum FdMeta { + /// We obtained the FD from a type that can contain any type of `FileType` and queried the metadata + /// because it is cheaper than probing all possible syscalls (reader side) + Metadata(Metadata), + Socket, + Pipe, + /// We don't have any metadata, e.g. because the original type was `File` which can represent + /// any `FileType` and we did not query the metadata either since it did not seem beneficial + /// (writer side) + NoneObtained, +} + +impl FdMeta { + fn maybe_fifo(&self) -> bool { + match self { + FdMeta::Metadata(meta) => meta.file_type().is_fifo(), + FdMeta::Socket => false, + FdMeta::Pipe => true, + FdMeta::NoneObtained => true, + } + } + + fn potential_sendfile_source(&self) -> bool { + match self { + // procfs erronously shows 0 length on non-empty readable files. + // and if a file is truly empty then a `read` syscall will determine that and skip the write syscall + // thus there would be benefit from attempting sendfile + FdMeta::Metadata(meta) + if meta.file_type().is_file() && meta.len() > 0 + || meta.file_type().is_block_device() => + { + true + } + _ => false, + } + } + + fn copy_file_range_candidate(&self) -> bool { + match self { + // copy_file_range will fail on empty procfs files. `read` can determine whether EOF has been reached + // without extra cost and skip the write, thus there is no benefit in attempting copy_file_range + FdMeta::Metadata(meta) if meta.is_file() && meta.len() > 0 => true, + FdMeta::NoneObtained => true, + _ => false, + } + } +} + +struct CopyParams(FdMeta, Option<RawFd>); + +struct Copier<'a, 'b, R: Read + ?Sized, W: Write + ?Sized> { + read: &'a mut R, + write: &'b mut W, +} + +trait SpecCopy { + fn copy(self) -> Result<u64>; +} + +impl<R: Read + ?Sized, W: Write + ?Sized> SpecCopy for Copier<'_, '_, R, W> { + default fn copy(self) -> Result<u64> { + generic_copy(self.read, self.write) + } +} + +impl<R: CopyRead, W: CopyWrite> SpecCopy for Copier<'_, '_, R, W> { + fn copy(self) -> Result<u64> { + let (reader, writer) = (self.read, self.write); + let r_cfg = reader.properties(); + let w_cfg = writer.properties(); + + // before direct operations on file descriptors ensure that all source and sink buffers are empty + let mut flush = || -> crate::io::Result<u64> { + let bytes = reader.drain_to(writer, u64::MAX)?; + // BufWriter buffered bytes have already been accounted for in earlier write() calls + writer.flush()?; + Ok(bytes) + }; + + let mut written = 0u64; + + if let (CopyParams(input_meta, Some(readfd)), CopyParams(output_meta, Some(writefd))) = + (r_cfg, w_cfg) + { + written += flush()?; + let max_write = reader.min_limit(); + + if input_meta.copy_file_range_candidate() && output_meta.copy_file_range_candidate() { + let result = copy_regular_files(readfd, writefd, max_write); + + match result { + CopyResult::Ended(Ok(bytes_copied)) => return Ok(bytes_copied + written), + CopyResult::Ended(err) => return err, + CopyResult::Fallback(bytes) => written += bytes, + } + } + + // on modern kernels sendfile can copy from any mmapable type (some but not all regular files and block devices) + // to any writable file descriptor. On older kernels the writer side can only be a socket. + // So we just try and fallback if needed. + // If current file offsets + write sizes overflow it may also fail, we do not try to fix that and instead + // fall back to the generic copy loop. + if input_meta.potential_sendfile_source() { + let result = sendfile_splice(SpliceMode::Sendfile, readfd, writefd, max_write); + + match result { + CopyResult::Ended(Ok(bytes_copied)) => return Ok(bytes_copied + written), + CopyResult::Ended(err) => return err, + CopyResult::Fallback(bytes) => written += bytes, + } + } + + if input_meta.maybe_fifo() || output_meta.maybe_fifo() { + let result = sendfile_splice(SpliceMode::Splice, readfd, writefd, max_write); + + match result { + CopyResult::Ended(Ok(bytes_copied)) => return Ok(bytes_copied + written), + CopyResult::Ended(err) => return err, + CopyResult::Fallback(0) => { /* use the fallback below */ } + CopyResult::Fallback(_) => { + unreachable!("splice should not return > 0 bytes on the fallback path") + } + } + } + } + + // fallback if none of the more specialized syscalls wants to work with these file descriptors + match generic_copy(reader, writer) { + Ok(bytes) => Ok(bytes + written), + err => err, + } + } +} + +#[rustc_specialization_trait] +trait CopyRead: Read { + /// Implementations that contain buffers (i.e. `BufReader`) must transfer data from their internal + /// buffers into `writer` until either the buffers are emptied or `limit` bytes have been + /// transferred, whichever occurs sooner. + /// If nested buffers are present the outer buffers must be drained first. + /// + /// This is necessary to directly bypass the wrapper types while preserving the data order + /// when operating directly on the underlying file descriptors. + fn drain_to<W: Write>(&mut self, _writer: &mut W, _limit: u64) -> Result<u64> { + Ok(0) + } + + /// The minimum of the limit of all `Take<_>` wrappers, `u64::MAX` otherwise. + /// This method does not account for data `BufReader` buffers and would underreport + /// the limit of a `Take<BufReader<Take<_>>>` type. Thus its result is only valid + /// after draining the buffers via `drain_to`. + fn min_limit(&self) -> u64 { + u64::MAX + } + + /// Extracts the file descriptor and hints/metadata, delegating through wrappers if necessary. + fn properties(&self) -> CopyParams; +} + +#[rustc_specialization_trait] +trait CopyWrite: Write { + /// Extracts the file descriptor and hints/metadata, delegating through wrappers if necessary. + fn properties(&self) -> CopyParams; +} + +impl<T> CopyRead for &mut T +where + T: CopyRead, +{ + fn drain_to<W: Write>(&mut self, writer: &mut W, limit: u64) -> Result<u64> { + (**self).drain_to(writer, limit) + } + + fn min_limit(&self) -> u64 { + (**self).min_limit() + } + + fn properties(&self) -> CopyParams { + (**self).properties() + } +} + +impl<T> CopyWrite for &mut T +where + T: CopyWrite, +{ + fn properties(&self) -> CopyParams { + (**self).properties() + } +} + +impl CopyRead for File { + fn properties(&self) -> CopyParams { + CopyParams(fd_to_meta(self), Some(self.as_raw_fd())) + } +} + +impl CopyRead for &File { + fn properties(&self) -> CopyParams { + CopyParams(fd_to_meta(*self), Some(self.as_raw_fd())) + } +} + +impl CopyWrite for File { + fn properties(&self) -> CopyParams { + CopyParams(FdMeta::NoneObtained, Some(self.as_raw_fd())) + } +} + +impl CopyWrite for &File { + fn properties(&self) -> CopyParams { + CopyParams(FdMeta::NoneObtained, Some(self.as_raw_fd())) + } +} + +impl CopyRead for TcpStream { + fn properties(&self) -> CopyParams { + // avoid the stat syscall since we can be fairly sure it's a socket + CopyParams(FdMeta::Socket, Some(self.as_raw_fd())) + } +} + +impl CopyRead for &TcpStream { + fn properties(&self) -> CopyParams { + // avoid the stat syscall since we can be fairly sure it's a socket + CopyParams(FdMeta::Socket, Some(self.as_raw_fd())) + } +} + +impl CopyWrite for TcpStream { + fn properties(&self) -> CopyParams { + // avoid the stat syscall since we can be fairly sure it's a socket + CopyParams(FdMeta::Socket, Some(self.as_raw_fd())) + } +} + +impl CopyWrite for &TcpStream { + fn properties(&self) -> CopyParams { + // avoid the stat syscall since we can be fairly sure it's a socket + CopyParams(FdMeta::Socket, Some(self.as_raw_fd())) + } +} + +impl CopyWrite for ChildStdin { + fn properties(&self) -> CopyParams { + CopyParams(FdMeta::Pipe, Some(self.as_raw_fd())) + } +} + +impl CopyRead for ChildStdout { + fn properties(&self) -> CopyParams { + CopyParams(FdMeta::Pipe, Some(self.as_raw_fd())) + } +} + +impl CopyRead for ChildStderr { + fn properties(&self) -> CopyParams { + CopyParams(FdMeta::Pipe, Some(self.as_raw_fd())) + } +} + +impl CopyRead for StdinLock<'_> { + fn drain_to<W: Write>(&mut self, writer: &mut W, outer_limit: u64) -> Result<u64> { + let buf_reader = self.as_mut_buf(); + let buf = buf_reader.buffer(); + let buf = &buf[0..min(buf.len(), outer_limit.try_into().unwrap_or(usize::MAX))]; + let bytes_drained = buf.len(); + writer.write_all(buf)?; + buf_reader.consume(bytes_drained); + + Ok(bytes_drained as u64) + } + + fn properties(&self) -> CopyParams { + CopyParams(fd_to_meta(self), Some(self.as_raw_fd())) + } +} + +impl CopyWrite for StdoutLock<'_> { + fn properties(&self) -> CopyParams { + CopyParams(FdMeta::NoneObtained, Some(self.as_raw_fd())) + } +} + +impl CopyWrite for StderrLock<'_> { + fn properties(&self) -> CopyParams { + CopyParams(FdMeta::NoneObtained, Some(self.as_raw_fd())) + } +} + +impl<T: CopyRead> CopyRead for Take<T> { + fn drain_to<W: Write>(&mut self, writer: &mut W, outer_limit: u64) -> Result<u64> { + let local_limit = self.limit(); + let combined_limit = min(outer_limit, local_limit); + let bytes_drained = self.get_mut().drain_to(writer, combined_limit)?; + // update limit since read() was bypassed + self.set_limit(local_limit - bytes_drained); + + Ok(bytes_drained) + } + + fn min_limit(&self) -> u64 { + min(Take::limit(self), self.get_ref().min_limit()) + } + + fn properties(&self) -> CopyParams { + self.get_ref().properties() + } +} + +impl<T: CopyRead> CopyRead for BufReader<T> { + fn drain_to<W: Write>(&mut self, writer: &mut W, outer_limit: u64) -> Result<u64> { + let buf = self.buffer(); + let buf = &buf[0..min(buf.len(), outer_limit.try_into().unwrap_or(usize::MAX))]; + let bytes = buf.len(); + writer.write_all(buf)?; + self.consume(bytes); + + let remaining = outer_limit - bytes as u64; + + // in case of nested bufreaders we also need to drain the ones closer to the source + let inner_bytes = self.get_mut().drain_to(writer, remaining)?; + + Ok(bytes as u64 + inner_bytes) + } + + fn min_limit(&self) -> u64 { + self.get_ref().min_limit() + } + + fn properties(&self) -> CopyParams { + self.get_ref().properties() + } +} + +impl<T: CopyWrite> CopyWrite for BufWriter<T> { + fn properties(&self) -> CopyParams { + self.get_ref().properties() + } +} + +fn fd_to_meta<T: AsRawFd>(fd: &T) -> FdMeta { + let fd = fd.as_raw_fd(); + let file: ManuallyDrop<File> = ManuallyDrop::new(unsafe { File::from_raw_fd(fd) }); + match file.metadata() { + Ok(meta) => FdMeta::Metadata(meta), + Err(_) => FdMeta::NoneObtained, + } +} + +pub(super) enum CopyResult { + Ended(Result<u64>), + Fallback(u64), +} + +/// linux-specific implementation that will attempt to use copy_file_range for copy offloading +/// as the name says, it only works on regular files +/// +/// Callers must handle fallback to a generic copy loop. +/// `Fallback` may indicate non-zero number of bytes already written +/// if one of the files' cursor +`max_len` would exceed u64::MAX (`EOVERFLOW`). +pub(super) fn copy_regular_files(reader: RawFd, writer: RawFd, max_len: u64) -> CopyResult { + use crate::cmp; + + // Kernel prior to 4.5 don't have copy_file_range + // We store the availability in a global to avoid unnecessary syscalls + static HAS_COPY_FILE_RANGE: AtomicBool = AtomicBool::new(true); + + unsafe fn copy_file_range( + fd_in: libc::c_int, + off_in: *mut libc::loff_t, + fd_out: libc::c_int, + off_out: *mut libc::loff_t, + len: libc::size_t, + flags: libc::c_uint, + ) -> libc::c_long { + libc::syscall(libc::SYS_copy_file_range, fd_in, off_in, fd_out, off_out, len, flags) + } + + let has_copy_file_range = HAS_COPY_FILE_RANGE.load(Ordering::Relaxed); + let mut written = 0u64; + while written < max_len { + let copy_result = if has_copy_file_range { + let bytes_to_copy = cmp::min(max_len - written, usize::MAX as u64); + // cap to 1GB chunks in case u64::MAX is passed as max_len and the file has a non-zero seek position + // this allows us to copy large chunks without hitting EOVERFLOW, + // unless someone sets a file offset close to u64::MAX - 1GB, in which case a fallback would be required + let bytes_to_copy = cmp::min(bytes_to_copy as usize, 0x4000_0000usize); + let copy_result = unsafe { + // We actually don't have to adjust the offsets, + // because copy_file_range adjusts the file offset automatically + cvt(copy_file_range( + reader, + ptr::null_mut(), + writer, + ptr::null_mut(), + bytes_to_copy, + 0, + )) + }; + if let Err(ref copy_err) = copy_result { + match copy_err.raw_os_error() { + Some(libc::ENOSYS | libc::EPERM | libc::EOPNOTSUPP) => { + HAS_COPY_FILE_RANGE.store(false, Ordering::Relaxed); + } + _ => {} + } + } + copy_result + } else { + Err(Error::from_raw_os_error(libc::ENOSYS)) + }; + match copy_result { + Ok(0) if written == 0 => { + // fallback to work around several kernel bugs where copy_file_range will fail to + // copy any bytes and return 0 instead of an error if + // - reading virtual files from the proc filesystem which appear to have 0 size + // but are not empty. noted in coreutils to affect kernels at least up to 5.6.19. + // - copying from an overlay filesystem in docker. reported to occur on fedora 32. + return CopyResult::Fallback(0); + } + Ok(0) => return CopyResult::Ended(Ok(written)), // reached EOF + Ok(ret) => written += ret as u64, + Err(err) => { + return match err.raw_os_error() { + // when file offset + max_length > u64::MAX + Some(libc::EOVERFLOW) => CopyResult::Fallback(written), + Some( + libc::ENOSYS | libc::EXDEV | libc::EINVAL | libc::EPERM | libc::EOPNOTSUPP, + ) => { + // Try fallback io::copy if either: + // - Kernel version is < 4.5 (ENOSYS) + // - Files are mounted on different fs (EXDEV) + // - copy_file_range is broken in various ways on RHEL/CentOS 7 (EOPNOTSUPP) + // - copy_file_range is disallowed, for example by seccomp (EPERM) + // - copy_file_range cannot be used with pipes or device nodes (EINVAL) + assert_eq!(written, 0); + CopyResult::Fallback(0) + } + _ => CopyResult::Ended(Err(err)), + }; + } + } + } + CopyResult::Ended(Ok(written)) +} + +#[derive(PartialEq)] +enum SpliceMode { + Sendfile, + Splice, +} + +/// performs splice or sendfile between file descriptors +/// Does _not_ fall back to a generic copy loop. +fn sendfile_splice(mode: SpliceMode, reader: RawFd, writer: RawFd, len: u64) -> CopyResult { + static HAS_SENDFILE: AtomicBool = AtomicBool::new(true); + static HAS_SPLICE: AtomicBool = AtomicBool::new(true); + + syscall! { + fn splice( + srcfd: libc::c_int, + src_offset: *const i64, + dstfd: libc::c_int, + dst_offset: *const i64, + len: libc::size_t, + flags: libc::c_int + ) -> libc::ssize_t + } + + match mode { + SpliceMode::Sendfile if !HAS_SENDFILE.load(Ordering::Relaxed) => { + return CopyResult::Fallback(0); + } + SpliceMode::Splice if !HAS_SPLICE.load(Ordering::Relaxed) => { + return CopyResult::Fallback(0); + } + _ => (), + } + + let mut written = 0u64; + while written < len { + // according to its manpage that's the maximum size sendfile() will copy per invocation + let chunk_size = crate::cmp::min(len - written, 0x7ffff000_u64) as usize; + + let result = match mode { + SpliceMode::Sendfile => { + cvt(unsafe { libc::sendfile(writer, reader, ptr::null_mut(), chunk_size) }) + } + SpliceMode::Splice => cvt(unsafe { + splice(reader, ptr::null_mut(), writer, ptr::null_mut(), chunk_size, 0) + }), + }; + + match result { + Ok(0) => break, // EOF + Ok(ret) => written += ret as u64, + Err(err) => { + return match err.raw_os_error() { + Some(libc::ENOSYS | libc::EPERM) => { + // syscall not supported (ENOSYS) + // syscall is disallowed, e.g. by seccomp (EPERM) + match mode { + SpliceMode::Sendfile => HAS_SENDFILE.store(false, Ordering::Relaxed), + SpliceMode::Splice => HAS_SPLICE.store(false, Ordering::Relaxed), + } + assert_eq!(written, 0); + CopyResult::Fallback(0) + } + Some(libc::EINVAL) => { + // splice/sendfile do not support this particular file descriptor (EINVAL) + assert_eq!(written, 0); + CopyResult::Fallback(0) + } + Some(os_err) if mode == SpliceMode::Sendfile && os_err == libc::EOVERFLOW => { + CopyResult::Fallback(written) + } + _ => CopyResult::Ended(Err(err)), + }; + } + } + } + CopyResult::Ended(Ok(written)) +} diff --git a/library/std/src/sys/unix/kernel_copy/tests.rs b/library/std/src/sys/unix/kernel_copy/tests.rs new file mode 100644 index 00000000000..21b121c26ff --- /dev/null +++ b/library/std/src/sys/unix/kernel_copy/tests.rs @@ -0,0 +1,213 @@ +use crate::env::temp_dir; +use crate::fs::OpenOptions; +use crate::io; +use crate::io::Result; +use crate::io::SeekFrom; +use crate::io::{BufRead, Read, Seek, Write}; +use crate::os::unix::io::AsRawFd; + +#[test] +fn copy_specialization() -> Result<()> { + use crate::io::{BufReader, BufWriter}; + + let path = crate::env::temp_dir(); + let source_path = path.join("copy-spec.source"); + let sink_path = path.join("copy-spec.sink"); + + let result: Result<()> = try { + let mut source = crate::fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(true) + .open(&source_path)?; + source.write_all(b"abcdefghiklmnopqr")?; + source.seek(SeekFrom::Start(8))?; + let mut source = BufReader::with_capacity(8, source.take(5)); + source.fill_buf()?; + assert_eq!(source.buffer(), b"iklmn"); + source.get_mut().set_limit(6); + source.get_mut().get_mut().seek(SeekFrom::Start(1))?; // "bcdefg" + let mut source = source.take(10); // "iklmnbcdef" + + let mut sink = crate::fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(true) + .open(&sink_path)?; + sink.write_all(b"000000")?; + let mut sink = BufWriter::with_capacity(5, sink); + sink.write_all(b"wxyz")?; + assert_eq!(sink.buffer(), b"wxyz"); + + let copied = crate::io::copy(&mut source, &mut sink)?; + assert_eq!(copied, 10); + assert_eq!(sink.buffer().len(), 0); + + let mut sink = sink.into_inner()?; + sink.seek(SeekFrom::Start(0))?; + let mut copied = Vec::new(); + sink.read_to_end(&mut copied)?; + assert_eq!(&copied, b"000000wxyziklmnbcdef"); + }; + + let rm1 = crate::fs::remove_file(source_path); + let rm2 = crate::fs::remove_file(sink_path); + + result.and(rm1).and(rm2) +} + +#[bench] +fn bench_file_to_file_copy(b: &mut test::Bencher) { + const BYTES: usize = 128 * 1024; + let src_path = temp_dir().join("file-copy-bench-src"); + let mut src = crate::fs::OpenOptions::new() + .create(true) + .truncate(true) + .read(true) + .write(true) + .open(src_path) + .unwrap(); + src.write(&vec![0u8; BYTES]).unwrap(); + + let sink_path = temp_dir().join("file-copy-bench-sink"); + let mut sink = crate::fs::OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(sink_path) + .unwrap(); + + b.bytes = BYTES as u64; + b.iter(|| { + src.seek(SeekFrom::Start(0)).unwrap(); + sink.seek(SeekFrom::Start(0)).unwrap(); + assert_eq!(BYTES as u64, io::copy(&mut src, &mut sink).unwrap()); + }); +} + +#[bench] +fn bench_file_to_socket_copy(b: &mut test::Bencher) { + const BYTES: usize = 128 * 1024; + let src_path = temp_dir().join("pipe-copy-bench-src"); + let mut src = OpenOptions::new() + .create(true) + .truncate(true) + .read(true) + .write(true) + .open(src_path) + .unwrap(); + src.write(&vec![0u8; BYTES]).unwrap(); + + let sink_drainer = crate::net::TcpListener::bind("localhost:0").unwrap(); + let mut sink = crate::net::TcpStream::connect(sink_drainer.local_addr().unwrap()).unwrap(); + let mut sink_drainer = sink_drainer.accept().unwrap().0; + + crate::thread::spawn(move || { + let mut sink_buf = vec![0u8; 1024 * 1024]; + loop { + sink_drainer.read(&mut sink_buf[..]).unwrap(); + } + }); + + b.bytes = BYTES as u64; + b.iter(|| { + src.seek(SeekFrom::Start(0)).unwrap(); + assert_eq!(BYTES as u64, io::copy(&mut src, &mut sink).unwrap()); + }); +} + +#[cfg(any(target_os = "linux", target_os = "android"))] +#[bench] +fn bench_socket_pipe_socket_copy(b: &mut test::Bencher) { + use super::CopyResult; + use crate::io::ErrorKind; + use crate::process::{ChildStdin, ChildStdout}; + use crate::sys_common::FromInner; + + let (read_end, write_end) = crate::sys::pipe::anon_pipe().unwrap(); + + let mut read_end = ChildStdout::from_inner(read_end); + let write_end = ChildStdin::from_inner(write_end); + + let acceptor = crate::net::TcpListener::bind("localhost:0").unwrap(); + let mut remote_end = crate::net::TcpStream::connect(acceptor.local_addr().unwrap()).unwrap(); + + let local_end = crate::sync::Arc::new(acceptor.accept().unwrap().0); + + // the data flow in this benchmark: + // + // socket(tx) local_source + // remote_end (write) +--------> (splice to) + // write_end + // + + // | + // | pipe + // v + // read_end + // remote_end (read) <---------+ (splice to) * + // socket(rx) local_end + // + // * benchmark loop using io::copy + + crate::thread::spawn(move || { + let mut sink_buf = vec![0u8; 1024 * 1024]; + remote_end.set_nonblocking(true).unwrap(); + loop { + match remote_end.write(&mut sink_buf[..]) { + Err(err) if err.kind() == ErrorKind::WouldBlock => {} + Ok(_) => {} + err => { + err.expect("write failed"); + } + }; + match remote_end.read(&mut sink_buf[..]) { + Err(err) if err.kind() == ErrorKind::WouldBlock => {} + Ok(_) => {} + err => { + err.expect("read failed"); + } + }; + } + }); + + // check that splice works, otherwise the benchmark would hang + let probe = super::sendfile_splice( + super::SpliceMode::Splice, + local_end.as_raw_fd(), + write_end.as_raw_fd(), + 1, + ); + + match probe { + CopyResult::Ended(Ok(1)) => { + // splice works + } + _ => { + eprintln!("splice failed, skipping benchmark"); + return; + } + } + + let local_source = local_end.clone(); + crate::thread::spawn(move || { + loop { + super::sendfile_splice( + super::SpliceMode::Splice, + local_source.as_raw_fd(), + write_end.as_raw_fd(), + u64::MAX, + ); + } + }); + + const BYTES: usize = 128 * 1024; + b.bytes = BYTES as u64; + b.iter(|| { + assert_eq!( + BYTES as u64, + io::copy(&mut (&mut read_end).take(BYTES as u64), &mut &*local_end).unwrap() + ); + }); +} diff --git a/library/std/src/sys/unix/mod.rs b/library/std/src/sys/unix/mod.rs index b28c6d85b7c..7609afbdd76 100644 --- a/library/std/src/sys/unix/mod.rs +++ b/library/std/src/sys/unix/mod.rs @@ -51,6 +51,8 @@ pub mod fd; pub mod fs; pub mod futex; pub mod io; +#[cfg(any(target_os = "linux", target_os = "android"))] +pub mod kernel_copy; #[cfg(target_os = "l4re")] mod l4re; pub mod memchr; diff --git a/library/std/src/sys/wasm/mutex_atomics.rs b/library/std/src/sys/wasm/mutex_atomics.rs index 479182ffa44..5ff0ec052b6 100644 --- a/library/std/src/sys/wasm/mutex_atomics.rs +++ b/library/std/src/sys/wasm/mutex_atomics.rs @@ -138,7 +138,7 @@ impl ReentrantMutex { self.owner.swap(0, SeqCst); // SAFETY: the caller must gurantee that `self.ptr()` is valid i32. unsafe { - wasm32::atomic_notify(self.ptr() as *mut i32, 1); + wasm32::memory_atomic_notify(self.ptr() as *mut i32, 1); } // wake up one waiter, if any } ref mut n => *n -= 1, diff --git a/library/std/src/thread/mod.rs b/library/std/src/thread/mod.rs index fefaa77a2a1..5d65f960fcd 100644 --- a/library/std/src/thread/mod.rs +++ b/library/std/src/thread/mod.rs @@ -456,15 +456,15 @@ impl Builder { let my_packet: Arc<UnsafeCell<Option<Result<T>>>> = Arc::new(UnsafeCell::new(None)); let their_packet = my_packet.clone(); - let (stdout, stderr) = crate::io::clone_io(); + let output_capture = crate::io::set_output_capture(None); + crate::io::set_output_capture(output_capture.clone()); let main = move || { if let Some(name) = their_thread.cname() { imp::Thread::set_name(name); } - crate::io::set_print(stdout); - crate::io::set_panic(stderr); + crate::io::set_output_capture(output_capture); // SAFETY: the stack guard passed is the one for the current thread. // This means the current thread's stack and the new thread's stack |