Add `IsTerminal` trait to determine if a descriptor or handle is a terminal

The UNIX and WASI implementations use `isatty`. The Windows
implementation uses the same logic the `atty` crate uses, including the
hack needed to detect msys terminals.

Implement this trait for `File` and for `Stdin`/`Stdout`/`Stderr` and
their locked counterparts on all platforms. On UNIX and WASI, implement
it for `BorrowedFd`/`OwnedFd`. On Windows, implement it for
`BorrowedHandle`/`OwnedHandle`.

Based on https://github.com/rust-lang/rust/pull/91121

Co-authored-by: Matt Wilkinson <mattwilki17@gmail.com>

5 files changed, 83 insertions, 0 deletions

diff --git a/library/std/src/sys/unix/io.rs b/library/std/src/sys/unix/io.rs
index deb5ee76bd0..29c340dd349 100644
--- a/library/std/src/sys/unix/io.rs
+++ b/library/std/src/sys/unix/io.rs
@@ -1,4 +1,5 @@
 use crate::marker::PhantomData;
+use crate::os::fd::{AsFd, AsRawFd};
 use crate::slice;
 
 use libc::{c_void, iovec};
@@ -74,3 +75,8 @@ impl<'a> IoSliceMut<'a> {
         unsafe { slice::from_raw_parts_mut(self.vec.iov_base as *mut u8, self.vec.iov_len) }
     }
 }
+
+pub fn is_terminal(fd: &impl AsFd) -> bool {
+    let fd = fd.as_fd();
+    unsafe { libc::isatty(fd.as_raw_fd()) != 0 }
+}
diff --git a/library/std/src/sys/unsupported/io.rs b/library/std/src/sys/unsupported/io.rs
index d5f475b4310..82610ffab7e 100644
--- a/library/std/src/sys/unsupported/io.rs
+++ b/library/std/src/sys/unsupported/io.rs
@@ -45,3 +45,7 @@ impl<'a> IoSliceMut<'a> {
         self.0
     }
 }
+
+pub fn is_terminal<T>(_: &T) -> bool {
+    false
+}
diff --git a/library/std/src/sys/wasi/io.rs b/library/std/src/sys/wasi/io.rs
index ee017d13a4c..2cd45df88fa 100644
--- a/library/std/src/sys/wasi/io.rs
+++ b/library/std/src/sys/wasi/io.rs
@@ -1,6 +1,7 @@
 #![deny(unsafe_op_in_unsafe_fn)]
 
 use crate::marker::PhantomData;
+use crate::os::fd::{AsFd, AsRawFd};
 use crate::slice;
 
 #[derive(Copy, Clone)]
@@ -71,3 +72,8 @@ impl<'a> IoSliceMut<'a> {
         unsafe { slice::from_raw_parts_mut(self.vec.buf as *mut u8, self.vec.buf_len) }
     }
 }
+
+pub fn is_terminal(fd: &impl AsFd) -> bool {
+    let fd = fd.as_fd();
+    unsafe { libc::isatty(fd.as_raw_fd()) != 0 }
+}
diff --git a/library/std/src/sys/windows/c.rs b/library/std/src/sys/windows/c.rs
index 732e227d7e2..917fc8e4995 100644
--- a/library/std/src/sys/windows/c.rs
+++ b/library/std/src/sys/windows/c.rs
@@ -127,6 +127,8 @@ pub const SECURITY_SQOS_PRESENT: DWORD = 0x00100000;
 
 pub const FIONBIO: c_ulong = 0x8004667e;
 
+pub const MAX_PATH: usize = 260;
+
 #[repr(C)]
 #[derive(Copy)]
 pub struct WIN32_FIND_DATAW {
@@ -539,6 +541,12 @@ pub struct SYMBOLIC_LINK_REPARSE_BUFFER {
 /// NB: Use carefully! In general using this as a reference is likely to get the
 /// provenance wrong for the `PathBuffer` field!
 #[repr(C)]
+pub struct FILE_NAME_INFO {
+    pub FileNameLength: DWORD,
+    pub FileName: [WCHAR; 1],
+}
+
+#[repr(C)]
 pub struct MOUNT_POINT_REPARSE_BUFFER {
     pub SubstituteNameOffset: c_ushort,
     pub SubstituteNameLength: c_ushort,
diff --git a/library/std/src/sys/windows/io.rs b/library/std/src/sys/windows/io.rs
index fb06df1f80c..489d66b0671 100644
--- a/library/std/src/sys/windows/io.rs
+++ b/library/std/src/sys/windows/io.rs
@@ -1,6 +1,10 @@
 use crate::marker::PhantomData;
+use crate::mem::size_of;
+use crate::os::windows::io::{AsHandle, AsRawHandle, BorrowedHandle};
 use crate::slice;
 use crate::sys::c;
+use core;
+use libc;
 
 #[derive(Copy, Clone)]
 #[repr(transparent)]
@@ -78,3 +82,58 @@ impl<'a> IoSliceMut<'a> {
         unsafe { slice::from_raw_parts_mut(self.vec.buf as *mut u8, self.vec.len as usize) }
     }
 }
+
+pub fn is_terminal(h: &impl AsHandle) -> bool {
+    unsafe { handle_is_console(h.as_handle()) }
+}
+
+unsafe fn handle_is_console(handle: BorrowedHandle<'_>) -> bool {
+    let handle = handle.as_raw_handle();
+
+    let mut out = 0;
+    if c::GetConsoleMode(handle, &mut out) != 0 {
+        // False positives aren't possible. If we got a console then we definitely have a console.
+        return true;
+    }
+
+    // At this point, we *could* have a false negative. We can determine that this is a true
+    // negative if we can detect the presence of a console on any of the standard I/O streams. If
+    // another stream has a console, then we know we're in a Windows console and can therefore
+    // trust the negative.
+    for std_handle in [c::STD_INPUT_HANDLE, c::STD_OUTPUT_HANDLE, c::STD_ERROR_HANDLE] {
+        let std_handle = c::GetStdHandle(std_handle);
+        if std_handle != handle && c::GetConsoleMode(std_handle, &mut out) != 0 {
+            return false;
+        }
+    }
+
+    // Otherwise, we fall back to an msys hack to see if we can detect the presence of a pty.
+    msys_tty_on(handle)
+}
+
+unsafe fn msys_tty_on(handle: c::HANDLE) -> bool {
+    let size = size_of::<c::FILE_NAME_INFO>() + c::MAX_PATH * size_of::<c::WCHAR>();
+    let mut name_info_bytes = vec![0u8; size];
+    let res = c::GetFileInformationByHandleEx(
+        handle,
+        c::FileNameInfo,
+        name_info_bytes.as_mut_ptr() as *mut libc::c_void,
+        size as u32,
+    );
+    if res == 0 {
+        return false;
+    }
+    let name_info: &c::FILE_NAME_INFO = &*(name_info_bytes.as_ptr() as *const c::FILE_NAME_INFO);
+    let s = core::slice::from_raw_parts(
+        name_info.FileName.as_ptr(),
+        name_info.FileNameLength as usize / 2,
+    );
+    let name = String::from_utf16_lossy(s);
+    // This checks whether 'pty' exists in the file name, which indicates that
+    // a pseudo-terminal is attached. To mitigate against false positives
+    // (e.g., an actual file name that contains 'pty'), we also require that
+    // either the strings 'msys-' or 'cygwin-' are in the file name as well.)
+    let is_msys = name.contains("msys-") || name.contains("cygwin-");
+    let is_pty = name.contains("-pty");
+    is_msys && is_pty
+}