std: Clamp max read/write sizes on Unix

Turns out that even though all these functions take a `size_t` they don't
actually work that well with anything larger than the maximum value of
`ssize_t`, the return value. Furthermore it looks like OSX rejects any
read/write requests larger than `INT_MAX - 1`. Handle all these cases by just
clamping the maximum size of a read/write on Unix to a platform-specific value.

Closes #38590

2 files changed, 24 insertions, 12 deletions

diff --git a/src/libstd/sys/unix/fd.rs b/src/libstd/sys/unix/fd.rs
index 2384d959881..dcab30aad83 100644
--- a/src/libstd/sys/unix/fd.rs
+++ b/src/libstd/sys/unix/fd.rs
@@ -10,8 +10,9 @@
 
 #![unstable(reason = "not public", issue = "0", feature = "fd")]
 
+use cmp;
 use io::{self, Read};
-use libc::{self, c_int, c_void};
+use libc::{self, c_int, c_void, ssize_t};
 use mem;
 use sync::atomic::{AtomicBool, Ordering};
 use sys::cvt;
@@ -23,6 +24,22 @@ pub struct FileDesc {
     fd: c_int,
 }
 
+fn max_len() -> usize {
+    // The maximum read limit on most posix-like systems is `SSIZE_MAX`,
+    // with the man page quoting that if the count of bytes to read is
+    // greater than `SSIZE_MAX` the result is "unspecified".
+    //
+    // On OSX, however, apparently the 64-bit libc is either buggy or
+    // intentionally showing odd behavior by rejecting any read with a size
+    // larger than or equal to INT_MAX. To handle both of these the read
+    // size is capped on both platforms.
+    if cfg!(target_os = "macos") {
+        <c_int>::max_value() as usize - 1
+    } else {
+        <ssize_t>::max_value() as usize
+    }
+}
+
 impl FileDesc {
     pub fn new(fd: c_int) -> FileDesc {
         FileDesc { fd: fd }
@@ -41,7 +58,7 @@ impl FileDesc {
         let ret = cvt(unsafe {
             libc::read(self.fd,
                        buf.as_mut_ptr() as *mut c_void,
-                       buf.len())
+                       cmp::min(buf.len(), max_len()))
         })?;
         Ok(ret as usize)
     }
@@ -69,7 +86,7 @@ impl FileDesc {
         unsafe {
             cvt_pread64(self.fd,
                         buf.as_mut_ptr() as *mut c_void,
-                        buf.len(),
+                        cmp::min(buf.len(), max_len()),
                         offset as i64)
                 .map(|n| n as usize)
         }
@@ -79,7 +96,7 @@ impl FileDesc {
         let ret = cvt(unsafe {
             libc::write(self.fd,
                         buf.as_ptr() as *const c_void,
-                        buf.len())
+                        cmp::min(buf.len(), max_len()))
         })?;
         Ok(ret as usize)
     }
@@ -102,7 +119,7 @@ impl FileDesc {
         unsafe {
             cvt_pwrite64(self.fd,
                          buf.as_ptr() as *const c_void,
-                         buf.len(),
+                         cmp::min(buf.len(), max_len()),
                          offset as i64)
                 .map(|n| n as usize)
         }
diff --git a/src/libstd/sys/windows/handle.rs b/src/libstd/sys/windows/handle.rs
index 10b86ba44bc..fdb9483fe1c 100644
--- a/src/libstd/sys/windows/handle.rs
+++ b/src/libstd/sys/windows/handle.rs
@@ -19,7 +19,6 @@ use ptr;
 use sys::c;
 use sys::cvt;
 use sys_common::io::read_to_end_uninitialized;
-use u32;
 
 /// An owned container for `HANDLE` object, closing them on Drop.
 ///
@@ -83,9 +82,7 @@ impl RawHandle {
 
     pub fn read(&self, buf: &mut [u8]) -> io::Result<usize> {
         let mut read = 0;
-        // ReadFile takes a DWORD (u32) for the length so it only supports
-        // reading u32::MAX bytes at a time.
-        let len = cmp::min(buf.len(), u32::MAX as usize) as c::DWORD;
+        let len = cmp::min(buf.len(), <c::DWORD>::max_value() as usize) as c::DWORD;
         let res = cvt(unsafe {
             c::ReadFile(self.0, buf.as_mut_ptr() as c::LPVOID,
                         len, &mut read, ptr::null_mut())
@@ -181,9 +178,7 @@ impl RawHandle {
 
     pub fn write(&self, buf: &[u8]) -> io::Result<usize> {
         let mut amt = 0;
-        // WriteFile takes a DWORD (u32) for the length so it only supports
-        // writing u32::MAX bytes at a time.
-        let len = cmp::min(buf.len(), u32::MAX as usize) as c::DWORD;
+        let len = cmp::min(buf.len(), <c::DWORD>::max_value() as usize) as c::DWORD;
         cvt(unsafe {
             c::WriteFile(self.0, buf.as_ptr() as c::LPVOID,
                          len, &mut amt, ptr::null_mut())