diff options
Diffstat (limited to 'library/alloc/src')
| -rw-r--r-- | library/alloc/src/boxed.rs | 5 | ||||
| -rw-r--r-- | library/alloc/src/boxed/convert.rs | 7 | ||||
| -rw-r--r-- | library/alloc/src/collections/btree/map.rs | 76 | ||||
| -rw-r--r-- | library/alloc/src/collections/btree/map/entry.rs | 4 | ||||
| -rw-r--r-- | library/alloc/src/ffi/c_str.rs | 21 | ||||
| -rw-r--r-- | library/alloc/src/lib.rs | 3 | ||||
| -rw-r--r-- | library/alloc/src/raw_vec/mod.rs | 2 | ||||
| -rw-r--r-- | library/alloc/src/string.rs | 87 | ||||
| -rw-r--r-- | library/alloc/src/sync.rs | 5 | ||||
| -rw-r--r-- | library/alloc/src/wtf8/mod.rs | 562 | ||||
| -rw-r--r-- | library/alloc/src/wtf8/tests.rs | 732 |
11 files changed, 1371 insertions, 133 deletions
diff --git a/library/alloc/src/boxed.rs b/library/alloc/src/boxed.rs index fa12d379c8c..98c9f6b51ab 100644 --- a/library/alloc/src/boxed.rs +++ b/library/alloc/src/boxed.rs @@ -2128,11 +2128,6 @@ impl<F: ?Sized + Future + Unpin, A: Allocator> Future for Box<F, A> { #[stable(feature = "box_error", since = "1.8.0")] impl<E: Error> Error for Box<E> { - #[allow(deprecated, deprecated_in_future)] - fn description(&self) -> &str { - Error::description(&**self) - } - #[allow(deprecated)] fn cause(&self) -> Option<&dyn Error> { Error::cause(&**self) diff --git a/library/alloc/src/boxed/convert.rs b/library/alloc/src/boxed/convert.rs index 80626580202..45c46fb5263 100644 --- a/library/alloc/src/boxed/convert.rs +++ b/library/alloc/src/boxed/convert.rs @@ -608,12 +608,7 @@ impl<'a> From<String> for Box<dyn Error + Send + Sync + 'a> { fn from(err: String) -> Box<dyn Error + Send + Sync + 'a> { struct StringError(String); - impl Error for StringError { - #[allow(deprecated)] - fn description(&self) -> &str { - &self.0 - } - } + impl Error for StringError {} impl fmt::Display for StringError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { diff --git a/library/alloc/src/collections/btree/map.rs b/library/alloc/src/collections/btree/map.rs index c4e599222e5..8b6d86a2888 100644 --- a/library/alloc/src/collections/btree/map.rs +++ b/library/alloc/src/collections/btree/map.rs @@ -40,30 +40,15 @@ pub(super) const MIN_LEN: usize = node::MIN_LEN_AFTER_SPLIT; /// An ordered map based on a [B-Tree]. /// -/// B-Trees represent a fundamental compromise between cache-efficiency and actually minimizing -/// the amount of work performed in a search. In theory, a binary search tree (BST) is the optimal -/// choice for a sorted map, as a perfectly balanced BST performs the theoretical minimum amount of -/// comparisons necessary to find an element (log<sub>2</sub>n). However, in practice the way this -/// is done is *very* inefficient for modern computer architectures. In particular, every element -/// is stored in its own individually heap-allocated node. This means that every single insertion -/// triggers a heap-allocation, and every single comparison should be a cache-miss. Since these -/// are both notably expensive things to do in practice, we are forced to, at the very least, -/// reconsider the BST strategy. -/// -/// A B-Tree instead makes each node contain B-1 to 2B-1 elements in a contiguous array. By doing -/// this, we reduce the number of allocations by a factor of B, and improve cache efficiency in -/// searches. However, this does mean that searches will have to do *more* comparisons on average. -/// The precise number of comparisons depends on the node search strategy used. For optimal cache -/// efficiency, one could search the nodes linearly. For optimal comparisons, one could search -/// the node using binary search. As a compromise, one could also perform a linear search -/// that initially only checks every i<sup>th</sup> element for some choice of i. +/// Given a key type with a [total order], an ordered map stores its entries in key order. +/// That means that keys must be of a type that implements the [`Ord`] trait, +/// such that two keys can always be compared to determine their [`Ordering`]. +/// Examples of keys with a total order are strings with lexicographical order, +/// and numbers with their natural order. /// -/// Currently, our implementation simply performs naive linear search. This provides excellent -/// performance on *small* nodes of elements which are cheap to compare. However in the future we -/// would like to further explore choosing the optimal search strategy based on the choice of B, -/// and possibly other factors. Using linear search, searching for a random element is expected -/// to take B * log(n) comparisons, which is generally worse than a BST. In practice, -/// however, performance is excellent. +/// Iterators obtained from functions such as [`BTreeMap::iter`], [`BTreeMap::into_iter`], [`BTreeMap::values`], or +/// [`BTreeMap::keys`] produce their items in key order, and take worst-case logarithmic and +/// amortized constant time per item returned. /// /// It is a logic error for a key to be modified in such a way that the key's ordering relative to /// any other key, as determined by the [`Ord`] trait, changes while it is in the map. This is @@ -72,14 +57,6 @@ pub(super) const MIN_LEN: usize = node::MIN_LEN_AFTER_SPLIT; /// `BTreeMap` that observed the logic error and not result in undefined behavior. This could /// include panics, incorrect results, aborts, memory leaks, and non-termination. /// -/// Iterators obtained from functions such as [`BTreeMap::iter`], [`BTreeMap::into_iter`], [`BTreeMap::values`], or -/// [`BTreeMap::keys`] produce their items in order by key, and take worst-case logarithmic and -/// amortized constant time per item returned. -/// -/// [B-Tree]: https://en.wikipedia.org/wiki/B-tree -/// [`Cell`]: core::cell::Cell -/// [`RefCell`]: core::cell::RefCell -/// /// # Examples /// /// ``` @@ -169,6 +146,43 @@ pub(super) const MIN_LEN: usize = node::MIN_LEN_AFTER_SPLIT; /// // modify an entry before an insert with in-place mutation /// player_stats.entry("mana").and_modify(|mana| *mana += 200).or_insert(100); /// ``` +/// +/// # Background +/// +/// A B-tree is (like) a [binary search tree], but adapted to the natural granularity that modern +/// machines like to consume data at. This means that each node contains an entire array of elements, +/// instead of just a single element. +/// +/// B-Trees represent a fundamental compromise between cache-efficiency and actually minimizing +/// the amount of work performed in a search. In theory, a binary search tree (BST) is the optimal +/// choice for a sorted map, as a perfectly balanced BST performs the theoretical minimum number of +/// comparisons necessary to find an element (log<sub>2</sub>n). However, in practice the way this +/// is done is *very* inefficient for modern computer architectures. In particular, every element +/// is stored in its own individually heap-allocated node. This means that every single insertion +/// triggers a heap-allocation, and every comparison is a potential cache-miss due to the indirection. +/// Since both heap-allocations and cache-misses are notably expensive in practice, we are forced to, +/// at the very least, reconsider the BST strategy. +/// +/// A B-Tree instead makes each node contain B-1 to 2B-1 elements in a contiguous array. By doing +/// this, we reduce the number of allocations by a factor of B, and improve cache efficiency in +/// searches. However, this does mean that searches will have to do *more* comparisons on average. +/// The precise number of comparisons depends on the node search strategy used. For optimal cache +/// efficiency, one could search the nodes linearly. For optimal comparisons, one could search +/// the node using binary search. As a compromise, one could also perform a linear search +/// that initially only checks every i<sup>th</sup> element for some choice of i. +/// +/// Currently, our implementation simply performs naive linear search. This provides excellent +/// performance on *small* nodes of elements which are cheap to compare. However in the future we +/// would like to further explore choosing the optimal search strategy based on the choice of B, +/// and possibly other factors. Using linear search, searching for a random element is expected +/// to take B * log(n) comparisons, which is generally worse than a BST. In practice, +/// however, performance is excellent. +/// +/// [B-Tree]: https://en.wikipedia.org/wiki/B-tree +/// [binary search tree]: https://en.wikipedia.org/wiki/Binary_search_tree +/// [total order]: https://en.wikipedia.org/wiki/Total_order +/// [`Cell`]: core::cell::Cell +/// [`RefCell`]: core::cell::RefCell #[stable(feature = "rust1", since = "1.0.0")] #[cfg_attr(not(test), rustc_diagnostic_item = "BTreeMap")] #[rustc_insignificant_dtor] diff --git a/library/alloc/src/collections/btree/map/entry.rs b/library/alloc/src/collections/btree/map/entry.rs index ea8fa363c38..ec9b774c308 100644 --- a/library/alloc/src/collections/btree/map/entry.rs +++ b/library/alloc/src/collections/btree/map/entry.rs @@ -136,10 +136,6 @@ impl<'a, K: Debug + Ord, V: Debug, A: Allocator + Clone> fmt::Display impl<'a, K: core::fmt::Debug + Ord, V: core::fmt::Debug> core::error::Error for crate::collections::btree_map::OccupiedError<'a, K, V> { - #[allow(deprecated)] - fn description(&self) -> &str { - "key already exists" - } } impl<'a, K: Ord, V, A: Allocator + Clone> Entry<'a, K, V, A> { diff --git a/library/alloc/src/ffi/c_str.rs b/library/alloc/src/ffi/c_str.rs index fe6c89a3094..b0c8c4b1ca4 100644 --- a/library/alloc/src/ffi/c_str.rs +++ b/library/alloc/src/ffi/c_str.rs @@ -1061,17 +1061,10 @@ impl IntoStringError { } } -impl IntoStringError { - fn description(&self) -> &str { - "C string contained non-utf8 bytes" - } -} - #[stable(feature = "cstring_into", since = "1.7.0")] impl fmt::Display for IntoStringError { - #[allow(deprecated, deprecated_in_future)] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.description().fmt(f) + "C string contained non-utf8 bytes".fmt(f) } } @@ -1291,23 +1284,13 @@ impl PartialEq<CString> for Cow<'_, CStr> { } #[stable(feature = "rust1", since = "1.0.0")] -impl core::error::Error for NulError { - #[allow(deprecated)] - fn description(&self) -> &str { - "nul byte found in data" - } -} +impl core::error::Error for NulError {} #[stable(feature = "cstring_from_vec_with_nul", since = "1.58.0")] impl core::error::Error for FromVecWithNulError {} #[stable(feature = "cstring_into", since = "1.7.0")] impl core::error::Error for IntoStringError { - #[allow(deprecated)] - fn description(&self) -> &str { - "C string contained non-utf8 bytes" - } - fn source(&self) -> Option<&(dyn core::error::Error + 'static)> { Some(&self.error) } diff --git a/library/alloc/src/lib.rs b/library/alloc/src/lib.rs index 639c5d4c930..711092ae8eb 100644 --- a/library/alloc/src/lib.rs +++ b/library/alloc/src/lib.rs @@ -159,6 +159,7 @@ #![feature(unicode_internals)] #![feature(unsize)] #![feature(unwrap_infallible)] +#![feature(wtf8_internals)] // tidy-alphabetical-end // // Language features: @@ -232,6 +233,8 @@ pub mod sync; #[cfg(all(not(no_global_oom_handling), not(no_rc), not(no_sync)))] pub mod task; pub mod vec; +#[cfg(all(not(no_rc), not(no_sync), not(no_global_oom_handling)))] +pub mod wtf8; #[doc(hidden)] #[unstable(feature = "liballoc_internals", issue = "none", reason = "implementation detail")] diff --git a/library/alloc/src/raw_vec/mod.rs b/library/alloc/src/raw_vec/mod.rs index 40716755aad..fd05f9ca464 100644 --- a/library/alloc/src/raw_vec/mod.rs +++ b/library/alloc/src/raw_vec/mod.rs @@ -155,7 +155,7 @@ impl RawVecInner<Global> { } // Tiny Vecs are dumb. Skip to: -// - 8 if the element size is 1, because any heap allocators is likely +// - 8 if the element size is 1, because any heap allocator is likely // to round up a request of less than 8 bytes to at least 8 bytes. // - 4 if elements are moderate-sized (<= 1 KiB). // - 1 otherwise, to avoid wasting too much space for very short Vecs. diff --git a/library/alloc/src/string.rs b/library/alloc/src/string.rs index 9eacbf00e42..1d0dd4be1b6 100644 --- a/library/alloc/src/string.rs +++ b/library/alloc/src/string.rs @@ -2285,20 +2285,10 @@ impl fmt::Display for FromUtf16Error { } #[stable(feature = "rust1", since = "1.0.0")] -impl Error for FromUtf8Error { - #[allow(deprecated)] - fn description(&self) -> &str { - "invalid utf-8" - } -} +impl Error for FromUtf8Error {} #[stable(feature = "rust1", since = "1.0.0")] -impl Error for FromUtf16Error { - #[allow(deprecated)] - fn description(&self) -> &str { - "invalid utf-16" - } -} +impl Error for FromUtf16Error {} #[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] @@ -2949,68 +2939,41 @@ impl SpecToString for i8 { } } -// Generic/generated code can sometimes have multiple, nested references -// for strings, including `&&&str`s that would never be written -// by hand. This macro generates twelve layers of nested `&`-impl -// for primitive strings. -#[cfg(not(no_global_oom_handling))] -macro_rules! to_string_str_wrap_in_ref { - {x $($x:ident)*} => { - &to_string_str_wrap_in_ref! { $($x)* } - }; - {} => { str }; -} -#[cfg(not(no_global_oom_handling))] -macro_rules! to_string_expr_wrap_in_deref { - {$self:expr ; x $($x:ident)*} => { - *(to_string_expr_wrap_in_deref! { $self ; $($x)* }) - }; - {$self:expr ;} => { $self }; -} #[cfg(not(no_global_oom_handling))] macro_rules! to_string_str { - {$($($x:ident)*),+} => { + {$($type:ty,)*} => { $( - impl SpecToString for to_string_str_wrap_in_ref!($($x)*) { + impl SpecToString for $type { #[inline] fn spec_to_string(&self) -> String { - String::from(to_string_expr_wrap_in_deref!(self ; $($x)*)) + let s: &str = self; + String::from(s) } } - )+ + )* }; } #[cfg(not(no_global_oom_handling))] to_string_str! { - x x x x x x x x x x x x, - x x x x x x x x x x x, - x x x x x x x x x x, - x x x x x x x x x, - x x x x x x x x, - x x x x x x x, - x x x x x x, - x x x x x, - x x x x, - x x x, - x x, - x, -} - -#[cfg(not(no_global_oom_handling))] -impl SpecToString for Cow<'_, str> { - #[inline] - fn spec_to_string(&self) -> String { - self[..].to_owned() - } -} - -#[cfg(not(no_global_oom_handling))] -impl SpecToString for String { - #[inline] - fn spec_to_string(&self) -> String { - self.to_owned() - } + Cow<'_, str>, + String, + // Generic/generated code can sometimes have multiple, nested references + // for strings, including `&&&str`s that would never be written + // by hand. + &&&&&&&&&&&&str, + &&&&&&&&&&&str, + &&&&&&&&&&str, + &&&&&&&&&str, + &&&&&&&&str, + &&&&&&&str, + &&&&&&str, + &&&&&str, + &&&&str, + &&&str, + &&str, + &str, + str, } #[cfg(not(no_global_oom_handling))] diff --git a/library/alloc/src/sync.rs b/library/alloc/src/sync.rs index 29caa7bc539..a21b6880674 100644 --- a/library/alloc/src/sync.rs +++ b/library/alloc/src/sync.rs @@ -4113,11 +4113,6 @@ impl<T: ?Sized, A: Allocator> Drop for UniqueArcUninit<T, A> { #[stable(feature = "arc_error", since = "1.52.0")] impl<T: core::error::Error + ?Sized> core::error::Error for Arc<T> { - #[allow(deprecated, deprecated_in_future)] - fn description(&self) -> &str { - core::error::Error::description(&**self) - } - #[allow(deprecated)] fn cause(&self) -> Option<&dyn core::error::Error> { core::error::Error::cause(&**self) diff --git a/library/alloc/src/wtf8/mod.rs b/library/alloc/src/wtf8/mod.rs new file mode 100644 index 00000000000..047994adc44 --- /dev/null +++ b/library/alloc/src/wtf8/mod.rs @@ -0,0 +1,562 @@ +//! Heap-allocated counterpart to core `wtf8` module. +#![unstable( + feature = "wtf8_internals", + issue = "none", + reason = "this is internal code for representing OsStr on some platforms and not a public API" +)] +// rustdoc bug: doc(hidden) on the module won't stop types in the module from showing up in trait +// implementations, so, we'll have to add more doc(hidden)s anyway +#![doc(hidden)] + +// Note: This module is also included in the alloctests crate using #[path] to +// run the tests. See the comment there for an explanation why this is the case. + +#[cfg(test)] +mod tests; + +use core::char::{MAX_LEN_UTF8, encode_utf8_raw}; +use core::hash::{Hash, Hasher}; +pub use core::wtf8::{CodePoint, Wtf8}; +#[cfg(not(test))] +pub use core::wtf8::{EncodeWide, Wtf8CodePoints}; +use core::{fmt, mem, ops, str}; + +use crate::borrow::{Cow, ToOwned}; +use crate::boxed::Box; +use crate::collections::TryReserveError; +#[cfg(not(test))] +use crate::rc::Rc; +use crate::string::String; +#[cfg(all(not(test), target_has_atomic = "ptr"))] +use crate::sync::Arc; +use crate::vec::Vec; + +/// An owned, growable string of well-formed WTF-8 data. +/// +/// Similar to `String`, but can additionally contain surrogate code points +/// if they’re not in a surrogate pair. +#[derive(Eq, PartialEq, Ord, PartialOrd, Clone)] +#[doc(hidden)] +pub struct Wtf8Buf { + bytes: Vec<u8>, + + /// Do we know that `bytes` holds a valid UTF-8 encoding? We can easily + /// know this if we're constructed from a `String` or `&str`. + /// + /// It is possible for `bytes` to have valid UTF-8 without this being + /// set, such as when we're concatenating `&Wtf8`'s and surrogates become + /// paired, as we don't bother to rescan the entire string. + is_known_utf8: bool, +} + +impl ops::Deref for Wtf8Buf { + type Target = Wtf8; + + fn deref(&self) -> &Wtf8 { + self.as_slice() + } +} + +impl ops::DerefMut for Wtf8Buf { + fn deref_mut(&mut self) -> &mut Wtf8 { + self.as_mut_slice() + } +} + +/// Formats the string in double quotes, with characters escaped according to +/// [`char::escape_debug`] and unpaired surrogates represented as `\u{xxxx}`, +/// where each `x` is a hexadecimal digit. +/// +/// For example, the code units [U+0061, U+D800, U+000A] are formatted as +/// `"a\u{D800}\n"`. +impl fmt::Debug for Wtf8Buf { + #[inline] + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&**self, formatter) + } +} + +/// Formats the string with unpaired surrogates substituted with the replacement +/// character, U+FFFD. +impl fmt::Display for Wtf8Buf { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + if let Some(s) = self.as_known_utf8() { + fmt::Display::fmt(s, formatter) + } else { + fmt::Display::fmt(&**self, formatter) + } + } +} + +#[cfg_attr(test, allow(dead_code))] +impl Wtf8Buf { + /// Creates a new, empty WTF-8 string. + #[inline] + pub fn new() -> Wtf8Buf { + Wtf8Buf { bytes: Vec::new(), is_known_utf8: true } + } + + /// Creates a new, empty WTF-8 string with pre-allocated capacity for `capacity` bytes. + #[inline] + pub fn with_capacity(capacity: usize) -> Wtf8Buf { + Wtf8Buf { bytes: Vec::with_capacity(capacity), is_known_utf8: true } + } + + /// Creates a WTF-8 string from a WTF-8 byte vec. + /// + /// Since the byte vec is not checked for valid WTF-8, this function is + /// marked unsafe. + #[inline] + pub unsafe fn from_bytes_unchecked(value: Vec<u8>) -> Wtf8Buf { + Wtf8Buf { bytes: value, is_known_utf8: false } + } + + /// Creates a WTF-8 string from a UTF-8 `String`. + /// + /// This takes ownership of the `String` and does not copy. + /// + /// Since WTF-8 is a superset of UTF-8, this always succeeds. + #[inline] + pub const fn from_string(string: String) -> Wtf8Buf { + Wtf8Buf { bytes: string.into_bytes(), is_known_utf8: true } + } + + /// Creates a WTF-8 string from a UTF-8 `&str` slice. + /// + /// This copies the content of the slice. + /// + /// Since WTF-8 is a superset of UTF-8, this always succeeds. + #[inline] + pub fn from_str(s: &str) -> Wtf8Buf { + Wtf8Buf { bytes: s.as_bytes().to_vec(), is_known_utf8: true } + } + + pub fn clear(&mut self) { + self.bytes.clear(); + self.is_known_utf8 = true; + } + + /// Creates a WTF-8 string from a potentially ill-formed UTF-16 slice of 16-bit code units. + /// + /// This is lossless: calling `.encode_wide()` on the resulting string + /// will always return the original code units. + pub fn from_wide(v: &[u16]) -> Wtf8Buf { + let mut string = Wtf8Buf::with_capacity(v.len()); + for item in char::decode_utf16(v.iter().cloned()) { + match item { + Ok(ch) => string.push_char(ch), + Err(surrogate) => { + let surrogate = surrogate.unpaired_surrogate(); + // Surrogates are known to be in the code point range. + let code_point = unsafe { CodePoint::from_u32_unchecked(surrogate as u32) }; + // The string will now contain an unpaired surrogate. + string.is_known_utf8 = false; + // Skip the WTF-8 concatenation check, + // surrogate pairs are already decoded by decode_utf16 + unsafe { + string.push_code_point_unchecked(code_point); + } + } + } + } + string + } + + /// Appends the given `char` to the end of this string. + /// This does **not** include the WTF-8 concatenation check or `is_known_utf8` check. + /// Copied from String::push. + unsafe fn push_code_point_unchecked(&mut self, code_point: CodePoint) { + let mut bytes = [0; MAX_LEN_UTF8]; + let bytes = encode_utf8_raw(code_point.to_u32(), &mut bytes); + self.bytes.extend_from_slice(bytes) + } + + #[inline] + pub fn as_slice(&self) -> &Wtf8 { + unsafe { Wtf8::from_bytes_unchecked(&self.bytes) } + } + + #[inline] + pub fn as_mut_slice(&mut self) -> &mut Wtf8 { + // Safety: `Wtf8` doesn't expose any way to mutate the bytes that would + // cause them to change from well-formed UTF-8 to ill-formed UTF-8, + // which would break the assumptions of the `is_known_utf8` field. + unsafe { Wtf8::from_mut_bytes_unchecked(&mut self.bytes) } + } + + /// Converts the string to UTF-8 without validation, if it was created from + /// valid UTF-8. + #[inline] + fn as_known_utf8(&self) -> Option<&str> { + if self.is_known_utf8 { + // SAFETY: The buffer is known to be valid UTF-8. + Some(unsafe { str::from_utf8_unchecked(self.as_bytes()) }) + } else { + None + } + } + + /// Reserves capacity for at least `additional` more bytes to be inserted + /// in the given `Wtf8Buf`. + /// The collection may reserve more space to avoid frequent reallocations. + /// + /// # Panics + /// + /// Panics if the new capacity exceeds `isize::MAX` bytes. + #[inline] + pub fn reserve(&mut self, additional: usize) { + self.bytes.reserve(additional) + } + + /// Tries to reserve capacity for at least `additional` more bytes to be + /// inserted in the given `Wtf8Buf`. The `Wtf8Buf` may reserve more space to + /// avoid frequent reallocations. After calling `try_reserve`, capacity will + /// be greater than or equal to `self.len() + additional`. Does nothing if + /// capacity is already sufficient. This method preserves the contents even + /// if an error occurs. + /// + /// # Errors + /// + /// If the capacity overflows, or the allocator reports a failure, then an error + /// is returned. + #[inline] + pub fn try_reserve(&mut self, additional: usize) -> Result<(), TryReserveError> { + self.bytes.try_reserve(additional) + } + + #[inline] + pub fn reserve_exact(&mut self, additional: usize) { + self.bytes.reserve_exact(additional) + } + + /// Tries to reserve the minimum capacity for exactly `additional` more + /// bytes to be inserted in the given `Wtf8Buf`. After calling + /// `try_reserve_exact`, capacity will be greater than or equal to + /// `self.len() + additional` if it returns `Ok(())`. + /// Does nothing if the capacity is already sufficient. + /// + /// Note that the allocator may give the `Wtf8Buf` more space than it + /// requests. Therefore, capacity can not be relied upon to be precisely + /// minimal. Prefer [`try_reserve`] if future insertions are expected. + /// + /// [`try_reserve`]: Wtf8Buf::try_reserve + /// + /// # Errors + /// + /// If the capacity overflows, or the allocator reports a failure, then an error + /// is returned. + #[inline] + pub fn try_reserve_exact(&mut self, additional: usize) -> Result<(), TryReserveError> { + self.bytes.try_reserve_exact(additional) + } + + #[inline] + pub fn shrink_to_fit(&mut self) { + self.bytes.shrink_to_fit() + } + + #[inline] + pub fn shrink_to(&mut self, min_capacity: usize) { + self.bytes.shrink_to(min_capacity) + } + + #[inline] + pub fn leak<'a>(self) -> &'a mut Wtf8 { + unsafe { Wtf8::from_mut_bytes_unchecked(self.bytes.leak()) } + } + + /// Returns the number of bytes that this string buffer can hold without reallocating. + #[inline] + pub fn capacity(&self) -> usize { + self.bytes.capacity() + } + + /// Append a UTF-8 slice at the end of the string. + #[inline] + pub fn push_str(&mut self, other: &str) { + self.bytes.extend_from_slice(other.as_bytes()) + } + + /// Append a WTF-8 slice at the end of the string. + /// + /// This replaces newly paired surrogates at the boundary + /// with a supplementary code point, + /// like concatenating ill-formed UTF-16 strings effectively would. + #[inline] + pub fn push_wtf8(&mut self, other: &Wtf8) { + match ((&*self).final_lead_surrogate(), other.initial_trail_surrogate()) { + // Replace newly paired surrogates by a supplementary code point. + (Some(lead), Some(trail)) => { + let len_without_lead_surrogate = self.len() - 3; + self.bytes.truncate(len_without_lead_surrogate); + let other_without_trail_surrogate = &other.as_bytes()[3..]; + // 4 bytes for the supplementary code point + self.bytes.reserve(4 + other_without_trail_surrogate.len()); + self.push_char(decode_surrogate_pair(lead, trail)); + self.bytes.extend_from_slice(other_without_trail_surrogate); + } + _ => { + // If we'll be pushing a string containing a surrogate, we may + // no longer have UTF-8. + if self.is_known_utf8 && other.next_surrogate(0).is_some() { + self.is_known_utf8 = false; + } + + self.bytes.extend_from_slice(other.as_bytes()); + } + } + } + + /// Append a Unicode scalar value at the end of the string. + #[inline] + pub fn push_char(&mut self, c: char) { + // SAFETY: It's always safe to push a char. + unsafe { self.push_code_point_unchecked(CodePoint::from_char(c)) } + } + + /// Append a code point at the end of the string. + /// + /// This replaces newly paired surrogates at the boundary + /// with a supplementary code point, + /// like concatenating ill-formed UTF-16 strings effectively would. + #[inline] + pub fn push(&mut self, code_point: CodePoint) { + if let Some(trail) = code_point.to_trail_surrogate() { + if let Some(lead) = (&*self).final_lead_surrogate() { + let len_without_lead_surrogate = self.len() - 3; + self.bytes.truncate(len_without_lead_surrogate); + self.push_char(decode_surrogate_pair(lead, trail)); + return; + } + + // We're pushing a trailing surrogate. + self.is_known_utf8 = false; + } else if code_point.to_lead_surrogate().is_some() { + // We're pushing a leading surrogate. + self.is_known_utf8 = false; + } + + // No newly paired surrogates at the boundary. + unsafe { self.push_code_point_unchecked(code_point) } + } + + /// Shortens a string to the specified length. + /// + /// # Panics + /// + /// Panics if `new_len` > current length, + /// or if `new_len` is not a code point boundary. + #[inline] + pub fn truncate(&mut self, new_len: usize) { + assert!(self.is_code_point_boundary(new_len)); + self.bytes.truncate(new_len) + } + + /// Consumes the WTF-8 string and tries to convert it to a vec of bytes. + #[inline] + pub fn into_bytes(self) -> Vec<u8> { + self.bytes + } + + /// Consumes the WTF-8 string and tries to convert it to UTF-8. + /// + /// This does not copy the data. + /// + /// If the contents are not well-formed UTF-8 + /// (that is, if the string contains surrogates), + /// the original WTF-8 string is returned instead. + pub fn into_string(self) -> Result<String, Wtf8Buf> { + if self.is_known_utf8 || self.next_surrogate(0).is_none() { + Ok(unsafe { String::from_utf8_unchecked(self.bytes) }) + } else { + Err(self) + } + } + + /// Consumes the WTF-8 string and converts it lossily to UTF-8. + /// + /// This does not copy the data (but may overwrite parts of it in place). + /// + /// Surrogates are replaced with `"\u{FFFD}"` (the replacement character “�”) + pub fn into_string_lossy(mut self) -> String { + if !self.is_known_utf8 { + let mut pos = 0; + while let Some((surrogate_pos, _)) = self.next_surrogate(pos) { + pos = surrogate_pos + 3; + // Surrogates and the replacement character are all 3 bytes, so + // they can substituted in-place. + self.bytes[surrogate_pos..pos].copy_from_slice("\u{FFFD}".as_bytes()); + } + } + unsafe { String::from_utf8_unchecked(self.bytes) } + } + + /// Converts this `Wtf8Buf` into a boxed `Wtf8`. + #[inline] + pub fn into_box(self) -> Box<Wtf8> { + // SAFETY: relies on `Wtf8` being `repr(transparent)`. + unsafe { mem::transmute(self.bytes.into_boxed_slice()) } + } + + /// Converts a `Box<Wtf8>` into a `Wtf8Buf`. + pub fn from_box(boxed: Box<Wtf8>) -> Wtf8Buf { + let bytes: Box<[u8]> = unsafe { mem::transmute(boxed) }; + Wtf8Buf { bytes: bytes.into_vec(), is_known_utf8: false } + } + + /// Provides plumbing to core `Vec::extend_from_slice`. + /// More well behaving alternative to allowing outer types + /// full mutable access to the core `Vec`. + #[inline] + pub unsafe fn extend_from_slice_unchecked(&mut self, other: &[u8]) { + self.bytes.extend_from_slice(other); + self.is_known_utf8 = false; + } +} + +/// Creates a new WTF-8 string from an iterator of code points. +/// +/// This replaces surrogate code point pairs with supplementary code points, +/// like concatenating ill-formed UTF-16 strings effectively would. +impl FromIterator<CodePoint> for Wtf8Buf { + fn from_iter<T: IntoIterator<Item = CodePoint>>(iter: T) -> Wtf8Buf { + let mut string = Wtf8Buf::new(); + string.extend(iter); + string + } +} + +/// Append code points from an iterator to the string. +/// +/// This replaces surrogate code point pairs with supplementary code points, +/// like concatenating ill-formed UTF-16 strings effectively would. +impl Extend<CodePoint> for Wtf8Buf { + fn extend<T: IntoIterator<Item = CodePoint>>(&mut self, iter: T) { + let iterator = iter.into_iter(); + let (low, _high) = iterator.size_hint(); + // Lower bound of one byte per code point (ASCII only) + self.bytes.reserve(low); + iterator.for_each(move |code_point| self.push(code_point)); + } + + #[inline] + fn extend_one(&mut self, code_point: CodePoint) { + self.push(code_point); + } + + #[inline] + fn extend_reserve(&mut self, additional: usize) { + // Lower bound of one byte per code point (ASCII only) + self.bytes.reserve(additional); + } +} + +/// Creates an owned `Wtf8Buf` from a borrowed `Wtf8`. +pub(super) fn to_owned(slice: &Wtf8) -> Wtf8Buf { + Wtf8Buf { bytes: slice.as_bytes().to_vec(), is_known_utf8: false } +} + +/// Lossily converts the string to UTF-8. +/// Returns a UTF-8 `&str` slice if the contents are well-formed in UTF-8. +/// +/// Surrogates are replaced with `"\u{FFFD}"` (the replacement character “�”). +/// +/// This only copies the data if necessary (if it contains any surrogate). +pub(super) fn to_string_lossy(slice: &Wtf8) -> Cow<'_, str> { + let Some((surrogate_pos, _)) = slice.next_surrogate(0) else { + return Cow::Borrowed(unsafe { str::from_utf8_unchecked(slice.as_bytes()) }); + }; + let wtf8_bytes = slice.as_bytes(); + let mut utf8_bytes = Vec::with_capacity(slice.len()); + utf8_bytes.extend_from_slice(&wtf8_bytes[..surrogate_pos]); + utf8_bytes.extend_from_slice("\u{FFFD}".as_bytes()); + let mut pos = surrogate_pos + 3; + loop { + match slice.next_surrogate(pos) { + Some((surrogate_pos, _)) => { + utf8_bytes.extend_from_slice(&wtf8_bytes[pos..surrogate_pos]); + utf8_bytes.extend_from_slice("\u{FFFD}".as_bytes()); + pos = surrogate_pos + 3; + } + None => { + utf8_bytes.extend_from_slice(&wtf8_bytes[pos..]); + return Cow::Owned(unsafe { String::from_utf8_unchecked(utf8_bytes) }); + } + } + } +} + +#[inline] +pub(super) fn clone_into(slice: &Wtf8, buf: &mut Wtf8Buf) { + buf.is_known_utf8 = false; + slice.as_bytes().clone_into(&mut buf.bytes); +} + +#[cfg(not(test))] +impl Wtf8 { + #[rustc_allow_incoherent_impl] + pub fn to_owned(&self) -> Wtf8Buf { + to_owned(self) + } + + #[rustc_allow_incoherent_impl] + pub fn clone_into(&self, buf: &mut Wtf8Buf) { + clone_into(self, buf) + } + + #[rustc_allow_incoherent_impl] + pub fn to_string_lossy(&self) -> Cow<'_, str> { + to_string_lossy(self) + } + + #[rustc_allow_incoherent_impl] + pub fn into_box(&self) -> Box<Wtf8> { + let boxed: Box<[u8]> = self.as_bytes().into(); + unsafe { mem::transmute(boxed) } + } + + #[rustc_allow_incoherent_impl] + pub fn empty_box() -> Box<Wtf8> { + let boxed: Box<[u8]> = Default::default(); + unsafe { mem::transmute(boxed) } + } + + #[cfg(target_has_atomic = "ptr")] + #[rustc_allow_incoherent_impl] + pub fn into_arc(&self) -> Arc<Wtf8> { + let arc: Arc<[u8]> = Arc::from(self.as_bytes()); + unsafe { Arc::from_raw(Arc::into_raw(arc) as *const Wtf8) } + } + + #[rustc_allow_incoherent_impl] + pub fn into_rc(&self) -> Rc<Wtf8> { + let rc: Rc<[u8]> = Rc::from(self.as_bytes()); + unsafe { Rc::from_raw(Rc::into_raw(rc) as *const Wtf8) } + } + + #[inline] + #[rustc_allow_incoherent_impl] + pub fn to_ascii_lowercase(&self) -> Wtf8Buf { + Wtf8Buf { bytes: self.as_bytes().to_ascii_lowercase(), is_known_utf8: false } + } + + #[inline] + #[rustc_allow_incoherent_impl] + pub fn to_ascii_uppercase(&self) -> Wtf8Buf { + Wtf8Buf { bytes: self.as_bytes().to_ascii_uppercase(), is_known_utf8: false } + } +} + +#[inline] +fn decode_surrogate_pair(lead: u16, trail: u16) -> char { + let code_point = 0x10000 + ((((lead - 0xD800) as u32) << 10) | (trail - 0xDC00) as u32); + unsafe { char::from_u32_unchecked(code_point) } +} + +impl Hash for Wtf8Buf { + #[inline] + fn hash<H: Hasher>(&self, state: &mut H) { + state.write(&self.bytes); + 0xfeu8.hash(state) + } +} diff --git a/library/alloc/src/wtf8/tests.rs b/library/alloc/src/wtf8/tests.rs new file mode 100644 index 00000000000..291f63f9f9e --- /dev/null +++ b/library/alloc/src/wtf8/tests.rs @@ -0,0 +1,732 @@ +use realalloc::string::ToString; + +use super::*; + +#[test] +fn code_point_from_u32() { + assert!(CodePoint::from_u32(0).is_some()); + assert!(CodePoint::from_u32(0xD800).is_some()); + assert!(CodePoint::from_u32(0x10FFFF).is_some()); + assert!(CodePoint::from_u32(0x110000).is_none()); +} + +#[test] +fn code_point_to_u32() { + fn c(value: u32) -> CodePoint { + CodePoint::from_u32(value).unwrap() + } + assert_eq!(c(0).to_u32(), 0); + assert_eq!(c(0xD800).to_u32(), 0xD800); + assert_eq!(c(0x10FFFF).to_u32(), 0x10FFFF); +} + +#[test] +fn code_point_to_lead_surrogate() { + fn c(value: u32) -> CodePoint { + CodePoint::from_u32(value).unwrap() + } + assert_eq!(c(0).to_lead_surrogate(), None); + assert_eq!(c(0xE9).to_lead_surrogate(), None); + assert_eq!(c(0xD800).to_lead_surrogate(), Some(0xD800)); + assert_eq!(c(0xDBFF).to_lead_surrogate(), Some(0xDBFF)); + assert_eq!(c(0xDC00).to_lead_surrogate(), None); + assert_eq!(c(0xDFFF).to_lead_surrogate(), None); + assert_eq!(c(0x1F4A9).to_lead_surrogate(), None); + assert_eq!(c(0x10FFFF).to_lead_surrogate(), None); +} + +#[test] +fn code_point_to_trail_surrogate() { + fn c(value: u32) -> CodePoint { + CodePoint::from_u32(value).unwrap() + } + assert_eq!(c(0).to_trail_surrogate(), None); + assert_eq!(c(0xE9).to_trail_surrogate(), None); + assert_eq!(c(0xD800).to_trail_surrogate(), None); + assert_eq!(c(0xDBFF).to_trail_surrogate(), None); + assert_eq!(c(0xDC00).to_trail_surrogate(), Some(0xDC00)); + assert_eq!(c(0xDFFF).to_trail_surrogate(), Some(0xDFFF)); + assert_eq!(c(0x1F4A9).to_trail_surrogate(), None); + assert_eq!(c(0x10FFFF).to_trail_surrogate(), None); +} + +#[test] +fn code_point_from_char() { + assert_eq!(CodePoint::from_char('a').to_u32(), 0x61); + assert_eq!(CodePoint::from_char('💩').to_u32(), 0x1F4A9); +} + +#[test] +fn code_point_to_string() { + assert_eq!(format!("{:?}", CodePoint::from_char('a')), "U+0061"); + assert_eq!(format!("{:?}", CodePoint::from_char('💩')), "U+1F4A9"); +} + +#[test] +fn code_point_to_char() { + fn c(value: u32) -> CodePoint { + CodePoint::from_u32(value).unwrap() + } + assert_eq!(c(0x61).to_char(), Some('a')); + assert_eq!(c(0x1F4A9).to_char(), Some('💩')); + assert_eq!(c(0xD800).to_char(), None); +} + +#[test] +fn code_point_to_char_lossy() { + fn c(value: u32) -> CodePoint { + CodePoint::from_u32(value).unwrap() + } + assert_eq!(c(0x61).to_char_lossy(), 'a'); + assert_eq!(c(0x1F4A9).to_char_lossy(), '💩'); + assert_eq!(c(0xD800).to_char_lossy(), '\u{FFFD}'); +} + +#[test] +fn wtf8buf_new() { + assert_eq!(Wtf8Buf::new().as_bytes(), b""); +} + +#[test] +fn wtf8buf_from_str() { + assert_eq!(Wtf8Buf::from_str("").as_bytes(), b""); + assert_eq!(Wtf8Buf::from_str("aé 💩").as_bytes(), b"a\xC3\xA9 \xF0\x9F\x92\xA9"); +} + +#[test] +fn wtf8buf_from_string() { + assert_eq!(Wtf8Buf::from_string(String::from("")).as_bytes(), b""); + assert_eq!( + Wtf8Buf::from_string(String::from("aé 💩")).as_bytes(), + b"a\xC3\xA9 \xF0\x9F\x92\xA9" + ); +} + +#[test] +fn wtf8buf_from_wide() { + let buf = Wtf8Buf::from_wide(&[]); + assert_eq!(buf.as_bytes(), b""); + assert!(buf.is_known_utf8); + + let buf = Wtf8Buf::from_wide(&[0x61, 0xE9, 0x20, 0xD83D, 0xDCA9]); + assert_eq!(buf.as_bytes(), b"a\xC3\xA9 \xF0\x9F\x92\xA9"); + assert!(buf.is_known_utf8); + + let buf = Wtf8Buf::from_wide(&[0x61, 0xE9, 0x20, 0xD83D, 0xD83D, 0xDCA9]); + assert_eq!(buf.as_bytes(), b"a\xC3\xA9 \xED\xA0\xBD\xF0\x9F\x92\xA9"); + assert!(!buf.is_known_utf8); + + let buf = Wtf8Buf::from_wide(&[0xD800]); + assert_eq!(buf.as_bytes(), b"\xED\xA0\x80"); + assert!(!buf.is_known_utf8); + + let buf = Wtf8Buf::from_wide(&[0xDBFF]); + assert_eq!(buf.as_bytes(), b"\xED\xAF\xBF"); + assert!(!buf.is_known_utf8); + + let buf = Wtf8Buf::from_wide(&[0xDC00]); + assert_eq!(buf.as_bytes(), b"\xED\xB0\x80"); + assert!(!buf.is_known_utf8); + + let buf = Wtf8Buf::from_wide(&[0xDFFF]); + assert_eq!(buf.as_bytes(), b"\xED\xBF\xBF"); + assert!(!buf.is_known_utf8); +} + +#[test] +fn wtf8buf_push_str() { + let mut string = Wtf8Buf::new(); + assert_eq!(string.as_bytes(), b""); + assert!(string.is_known_utf8); + + string.push_str("aé 💩"); + assert_eq!(string.as_bytes(), b"a\xC3\xA9 \xF0\x9F\x92\xA9"); + assert!(string.is_known_utf8); +} + +#[test] +fn wtf8buf_push_char() { + let mut string = Wtf8Buf::from_str("aé "); + assert_eq!(string.as_bytes(), b"a\xC3\xA9 "); + assert!(string.is_known_utf8); + + string.push_char('💩'); + assert_eq!(string.as_bytes(), b"a\xC3\xA9 \xF0\x9F\x92\xA9"); + assert!(string.is_known_utf8); +} + +#[test] +fn wtf8buf_push() { + let mut string = Wtf8Buf::from_str("aé "); + assert_eq!(string.as_bytes(), b"a\xC3\xA9 "); + assert!(string.is_known_utf8); + + string.push(CodePoint::from_char('💩')); + assert_eq!(string.as_bytes(), b"a\xC3\xA9 \xF0\x9F\x92\xA9"); + assert!(string.is_known_utf8); + + fn c(value: u32) -> CodePoint { + CodePoint::from_u32(value).unwrap() + } + + let mut string = Wtf8Buf::new(); + string.push(c(0xD83D)); // lead + assert!(!string.is_known_utf8); + string.push(c(0xDCA9)); // trail + assert_eq!(string.as_bytes(), b"\xF0\x9F\x92\xA9"); // Magic! + + let mut string = Wtf8Buf::new(); + string.push(c(0xD83D)); // lead + assert!(!string.is_known_utf8); + string.push(c(0x20)); // not surrogate + string.push(c(0xDCA9)); // trail + assert_eq!(string.as_bytes(), b"\xED\xA0\xBD \xED\xB2\xA9"); + + let mut string = Wtf8Buf::new(); + string.push(c(0xD800)); // lead + assert!(!string.is_known_utf8); + string.push(c(0xDBFF)); // lead + assert_eq!(string.as_bytes(), b"\xED\xA0\x80\xED\xAF\xBF"); + + let mut string = Wtf8Buf::new(); + string.push(c(0xD800)); // lead + assert!(!string.is_known_utf8); + string.push(c(0xE000)); // not surrogate + assert_eq!(string.as_bytes(), b"\xED\xA0\x80\xEE\x80\x80"); + + let mut string = Wtf8Buf::new(); + string.push(c(0xD7FF)); // not surrogate + assert!(string.is_known_utf8); + string.push(c(0xDC00)); // trail + assert!(!string.is_known_utf8); + assert_eq!(string.as_bytes(), b"\xED\x9F\xBF\xED\xB0\x80"); + + let mut string = Wtf8Buf::new(); + string.push(c(0x61)); // not surrogate, < 3 bytes + assert!(string.is_known_utf8); + string.push(c(0xDC00)); // trail + assert!(!string.is_known_utf8); + assert_eq!(string.as_bytes(), b"\x61\xED\xB0\x80"); + + let mut string = Wtf8Buf::new(); + string.push(c(0xDC00)); // trail + assert!(!string.is_known_utf8); + assert_eq!(string.as_bytes(), b"\xED\xB0\x80"); +} + +#[test] +fn wtf8buf_push_wtf8() { + let mut string = Wtf8Buf::from_str("aé"); + assert_eq!(string.as_bytes(), b"a\xC3\xA9"); + string.push_wtf8(Wtf8::from_str(" 💩")); + assert_eq!(string.as_bytes(), b"a\xC3\xA9 \xF0\x9F\x92\xA9"); + assert!(string.is_known_utf8); + + fn w(v: &[u8]) -> &Wtf8 { + unsafe { Wtf8::from_bytes_unchecked(v) } + } + + let mut string = Wtf8Buf::new(); + string.push_wtf8(w(b"\xED\xA0\xBD")); // lead + string.push_wtf8(w(b"\xED\xB2\xA9")); // trail + assert_eq!(string.as_bytes(), b"\xF0\x9F\x92\xA9"); // Magic! + + let mut string = Wtf8Buf::new(); + string.push_wtf8(w(b"\xED\xA0\xBD")); // lead + string.push_wtf8(w(b" ")); // not surrogate + string.push_wtf8(w(b"\xED\xB2\xA9")); // trail + assert_eq!(string.as_bytes(), b"\xED\xA0\xBD \xED\xB2\xA9"); + assert!(!string.is_known_utf8); + + let mut string = Wtf8Buf::new(); + string.push_wtf8(w(b"\xED\xA0\x80")); // lead + string.push_wtf8(w(b"\xED\xAF\xBF")); // lead + assert_eq!(string.as_bytes(), b"\xED\xA0\x80\xED\xAF\xBF"); + assert!(!string.is_known_utf8); + + let mut string = Wtf8Buf::new(); + string.push_wtf8(w(b"\xED\xA0\x80")); // lead + string.push_wtf8(w(b"\xEE\x80\x80")); // not surrogate + assert_eq!(string.as_bytes(), b"\xED\xA0\x80\xEE\x80\x80"); + assert!(!string.is_known_utf8); + + let mut string = Wtf8Buf::new(); + string.push_wtf8(w(b"\xED\x9F\xBF")); // not surrogate + string.push_wtf8(w(b"\xED\xB0\x80")); // trail + assert_eq!(string.as_bytes(), b"\xED\x9F\xBF\xED\xB0\x80"); + assert!(!string.is_known_utf8); + + let mut string = Wtf8Buf::new(); + string.push_wtf8(w(b"a")); // not surrogate, < 3 bytes + string.push_wtf8(w(b"\xED\xB0\x80")); // trail + assert_eq!(string.as_bytes(), b"\x61\xED\xB0\x80"); + assert!(!string.is_known_utf8); + + let mut string = Wtf8Buf::new(); + string.push_wtf8(w(b"\xED\xB0\x80")); // trail + assert_eq!(string.as_bytes(), b"\xED\xB0\x80"); + assert!(!string.is_known_utf8); +} + +#[test] +fn wtf8buf_truncate() { + let mut string = Wtf8Buf::from_str("aé"); + assert!(string.is_known_utf8); + + string.truncate(3); + assert_eq!(string.as_bytes(), b"a\xC3\xA9"); + assert!(string.is_known_utf8); + + string.truncate(1); + assert_eq!(string.as_bytes(), b"a"); + assert!(string.is_known_utf8); + + string.truncate(0); + assert_eq!(string.as_bytes(), b""); + assert!(string.is_known_utf8); +} + +#[test] +fn wtf8buf_truncate_around_non_bmp() { + let mut string = Wtf8Buf::from_str("💩"); + assert!(string.is_known_utf8); + + string.truncate(4); + assert_eq!(string.as_bytes(), b"\xF0\x9F\x92\xA9"); + assert!(string.is_known_utf8); + + string.truncate(0); + assert_eq!(string.as_bytes(), b""); + assert!(string.is_known_utf8); +} + +#[test] +#[should_panic] +fn wtf8buf_truncate_fail_code_point_boundary() { + let mut string = Wtf8Buf::from_str("aé"); + string.truncate(2); +} + +#[test] +#[should_panic] +fn wtf8buf_truncate_fail_longer() { + let mut string = Wtf8Buf::from_str("aé"); + string.truncate(4); +} + +#[test] +#[should_panic] +fn wtf8buf_truncate_splitting_non_bmp3() { + let mut string = Wtf8Buf::from_str("💩"); + assert!(string.is_known_utf8); + string.truncate(3); +} + +#[test] +#[should_panic] +fn wtf8buf_truncate_splitting_non_bmp2() { + let mut string = Wtf8Buf::from_str("💩"); + assert!(string.is_known_utf8); + string.truncate(2); +} + +#[test] +#[should_panic] +fn wtf8buf_truncate_splitting_non_bmp1() { + let mut string = Wtf8Buf::from_str("💩"); + assert!(string.is_known_utf8); + string.truncate(1); +} + +#[test] +fn wtf8buf_into_string() { + let mut string = Wtf8Buf::from_str("aé 💩"); + assert!(string.is_known_utf8); + assert_eq!(string.clone().into_string(), Ok(String::from("aé 💩"))); + string.push(CodePoint::from_u32(0xD800).unwrap()); + assert!(!string.is_known_utf8); + assert_eq!(string.clone().into_string(), Err(string)); +} + +#[test] +fn wtf8buf_into_string_lossy() { + let mut string = Wtf8Buf::from_str("aé 💩"); + assert_eq!(string.clone().into_string_lossy(), String::from("aé 💩")); + string.push(CodePoint::from_u32(0xD800).unwrap()); + assert_eq!(string.clone().into_string_lossy(), String::from("aé 💩�")); +} + +#[test] +fn wtf8buf_from_iterator() { + fn f(values: &[u32]) -> Wtf8Buf { + values.iter().map(|&c| CodePoint::from_u32(c).unwrap()).collect::<Wtf8Buf>() + } + assert_eq!( + f(&[0x61, 0xE9, 0x20, 0x1F4A9]), + Wtf8Buf { bytes: b"a\xC3\xA9 \xF0\x9F\x92\xA9".to_vec(), is_known_utf8: true } + ); + + assert_eq!(f(&[0xD83D, 0xDCA9]).as_bytes(), b"\xF0\x9F\x92\xA9"); // Magic! + assert_eq!( + f(&[0xD83D, 0x20, 0xDCA9]), + Wtf8Buf { bytes: b"\xED\xA0\xBD \xED\xB2\xA9".to_vec(), is_known_utf8: false } + ); + assert_eq!( + f(&[0xD800, 0xDBFF]), + Wtf8Buf { bytes: b"\xED\xA0\x80\xED\xAF\xBF".to_vec(), is_known_utf8: false } + ); + assert_eq!( + f(&[0xD800, 0xE000]), + Wtf8Buf { bytes: b"\xED\xA0\x80\xEE\x80\x80".to_vec(), is_known_utf8: false } + ); + assert_eq!( + f(&[0xD7FF, 0xDC00]), + Wtf8Buf { bytes: b"\xED\x9F\xBF\xED\xB0\x80".to_vec(), is_known_utf8: false } + ); + assert_eq!( + f(&[0x61, 0xDC00]), + Wtf8Buf { bytes: b"\x61\xED\xB0\x80".to_vec(), is_known_utf8: false } + ); + assert_eq!(f(&[0xDC00]), Wtf8Buf { bytes: b"\xED\xB0\x80".to_vec(), is_known_utf8: false }); +} + +#[test] +fn wtf8buf_extend() { + fn e(initial: &[u32], extended: &[u32]) -> Wtf8Buf { + fn c(value: &u32) -> CodePoint { + CodePoint::from_u32(*value).unwrap() + } + let mut string = initial.iter().map(c).collect::<Wtf8Buf>(); + string.extend(extended.iter().map(c)); + string + } + + assert_eq!( + e(&[0x61, 0xE9], &[0x20, 0x1F4A9]), + Wtf8Buf { bytes: b"a\xC3\xA9 \xF0\x9F\x92\xA9".to_vec(), is_known_utf8: true } + ); + + assert_eq!(e(&[0xD83D], &[0xDCA9]).as_bytes(), b"\xF0\x9F\x92\xA9"); // Magic! + assert_eq!( + e(&[0xD83D, 0x20], &[0xDCA9]), + Wtf8Buf { bytes: b"\xED\xA0\xBD \xED\xB2\xA9".to_vec(), is_known_utf8: false } + ); + assert_eq!( + e(&[0xD800], &[0xDBFF]), + Wtf8Buf { bytes: b"\xED\xA0\x80\xED\xAF\xBF".to_vec(), is_known_utf8: false } + ); + assert_eq!( + e(&[0xD800], &[0xE000]), + Wtf8Buf { bytes: b"\xED\xA0\x80\xEE\x80\x80".to_vec(), is_known_utf8: false } + ); + assert_eq!( + e(&[0xD7FF], &[0xDC00]), + Wtf8Buf { bytes: b"\xED\x9F\xBF\xED\xB0\x80".to_vec(), is_known_utf8: false } + ); + assert_eq!( + e(&[0x61], &[0xDC00]), + Wtf8Buf { bytes: b"\x61\xED\xB0\x80".to_vec(), is_known_utf8: false } + ); + assert_eq!( + e(&[], &[0xDC00]), + Wtf8Buf { bytes: b"\xED\xB0\x80".to_vec(), is_known_utf8: false } + ); +} + +#[test] +fn wtf8buf_show() { + let mut string = Wtf8Buf::from_str("a\té \u{7f}💩\r"); + string.push(CodePoint::from_u32(0xD800).unwrap()); + assert_eq!(format!("{string:?}"), "\"a\\té \\u{7f}\u{1f4a9}\\r\\u{d800}\""); +} + +#[test] +fn wtf8buf_as_slice() { + assert_eq!(Wtf8Buf::from_str("aé").as_slice(), Wtf8::from_str("aé")); +} + +#[test] +fn wtf8buf_show_str() { + let text = "a\té 💩\r"; + let string = Wtf8Buf::from_str(text); + assert_eq!(format!("{text:?}"), format!("{string:?}")); +} + +#[test] +fn wtf8_from_str() { + assert_eq!(&Wtf8::from_str("").as_bytes(), b""); + assert_eq!(&Wtf8::from_str("aé 💩").as_bytes(), b"a\xC3\xA9 \xF0\x9F\x92\xA9"); +} + +#[test] +fn wtf8_len() { + assert_eq!(Wtf8::from_str("").len(), 0); + assert_eq!(Wtf8::from_str("aé 💩").len(), 8); +} + +#[test] +fn wtf8_slice() { + assert_eq!(&Wtf8::from_str("aé 💩")[1..4].as_bytes(), b"\xC3\xA9 "); +} + +#[test] +#[should_panic] +fn wtf8_slice_not_code_point_boundary() { + let _ = &Wtf8::from_str("aé 💩")[2..4]; +} + +#[test] +fn wtf8_slice_from() { + assert_eq!(&Wtf8::from_str("aé 💩")[1..].as_bytes(), b"\xC3\xA9 \xF0\x9F\x92\xA9"); +} + +#[test] +#[should_panic] +fn wtf8_slice_from_not_code_point_boundary() { + let _ = &Wtf8::from_str("aé 💩")[2..]; +} + +#[test] +fn wtf8_slice_to() { + assert_eq!(&Wtf8::from_str("aé 💩")[..4].as_bytes(), b"a\xC3\xA9 "); +} + +#[test] +#[should_panic] +fn wtf8_slice_to_not_code_point_boundary() { + let _ = &Wtf8::from_str("aé 💩")[5..]; +} + +#[test] +fn wtf8_ascii_byte_at() { + let slice = Wtf8::from_str("aé 💩"); + assert_eq!(slice.ascii_byte_at(0), b'a'); + assert_eq!(slice.ascii_byte_at(1), b'\xFF'); + assert_eq!(slice.ascii_byte_at(2), b'\xFF'); + assert_eq!(slice.ascii_byte_at(3), b' '); + assert_eq!(slice.ascii_byte_at(4), b'\xFF'); +} + +#[test] +fn wtf8_code_points() { + fn c(value: u32) -> CodePoint { + CodePoint::from_u32(value).unwrap() + } + fn cp(string: &Wtf8Buf) -> Vec<Option<char>> { + string.code_points().map(|c| c.to_char()).collect::<Vec<_>>() + } + let mut string = Wtf8Buf::from_str("é "); + assert_eq!(cp(&string), [Some('é'), Some(' ')]); + string.push(c(0xD83D)); + assert_eq!(cp(&string), [Some('é'), Some(' '), None]); + string.push(c(0xDCA9)); + assert_eq!(cp(&string), [Some('é'), Some(' '), Some('💩')]); +} + +#[test] +fn wtf8_as_str() { + assert_eq!(Wtf8::from_str("").as_str(), Ok("")); + assert_eq!(Wtf8::from_str("aé 💩").as_str(), Ok("aé 💩")); + let mut string = Wtf8Buf::new(); + string.push(CodePoint::from_u32(0xD800).unwrap()); + assert!(string.as_str().is_err()); +} + +#[test] +fn wtf8_to_string_lossy() { + assert_eq!(to_string_lossy(Wtf8::from_str("")), Cow::Borrowed("")); + assert_eq!(to_string_lossy(Wtf8::from_str("aé 💩")), Cow::Borrowed("aé 💩")); + let mut string = Wtf8Buf::from_str("aé 💩"); + string.push(CodePoint::from_u32(0xD800).unwrap()); + let expected: Cow<'_, str> = Cow::Owned(String::from("aé 💩�")); + assert_eq!(to_string_lossy(&string), expected); +} + +#[test] +fn wtf8_display() { + fn d(b: &[u8]) -> String { + (&unsafe { Wtf8::from_bytes_unchecked(b) }).to_string() + } + + assert_eq!("", d("".as_bytes())); + assert_eq!("aé 💩", d("aé 💩".as_bytes())); + + let mut string = Wtf8Buf::from_str("aé 💩"); + string.push(CodePoint::from_u32(0xD800).unwrap()); + assert_eq!("aé 💩�", d(string.as_ref())); +} + +#[test] +fn wtf8_encode_wide() { + let mut string = Wtf8Buf::from_str("aé "); + string.push(CodePoint::from_u32(0xD83D).unwrap()); + string.push_char('💩'); + assert_eq!( + string.encode_wide().collect::<Vec<_>>(), + vec![0x61, 0xE9, 0x20, 0xD83D, 0xD83D, 0xDCA9] + ); +} + +#[test] +fn wtf8_encode_wide_size_hint() { + let string = Wtf8Buf::from_str("\u{12345}"); + let mut iter = string.encode_wide(); + assert_eq!((1, Some(8)), iter.size_hint()); + iter.next().unwrap(); + assert_eq!((1, Some(1)), iter.size_hint()); + iter.next().unwrap(); + assert_eq!((0, Some(0)), iter.size_hint()); + assert!(iter.next().is_none()); +} + +#[test] +fn wtf8_clone_into() { + let mut string = Wtf8Buf::new(); + clone_into(Wtf8::from_str("green"), &mut string); + assert_eq!(string.as_bytes(), b"green"); + + let mut string = Wtf8Buf::from_str("green"); + clone_into(Wtf8::from_str(""), &mut string); + assert_eq!(string.as_bytes(), b""); + + let mut string = Wtf8Buf::from_str("red"); + clone_into(Wtf8::from_str("green"), &mut string); + assert_eq!(string.as_bytes(), b"green"); + + let mut string = Wtf8Buf::from_str("green"); + clone_into(Wtf8::from_str("red"), &mut string); + assert_eq!(string.as_bytes(), b"red"); + + let mut string = Wtf8Buf::from_str("green"); + assert!(string.is_known_utf8); + clone_into(unsafe { Wtf8::from_bytes_unchecked(b"\xED\xA0\x80") }, &mut string); + assert_eq!(string.as_bytes(), b"\xED\xA0\x80"); + assert!(!string.is_known_utf8); +} + +#[test] +fn wtf8_make_ascii_lowercase() { + let mut lowercase = Wtf8Buf::from_str(""); + lowercase.make_ascii_lowercase(); + assert_eq!(lowercase.as_bytes(), b""); + + let mut lowercase = Wtf8Buf::from_str("GrEeN gRaPeS! 🍇"); + lowercase.make_ascii_lowercase(); + assert_eq!(lowercase.as_bytes(), b"green grapes! \xf0\x9f\x8d\x87"); + + let mut lowercase = to_owned(unsafe { Wtf8::from_bytes_unchecked(b"\xED\xA0\x80") }); + lowercase.make_ascii_lowercase(); + assert_eq!(lowercase.as_bytes(), b"\xED\xA0\x80"); + assert!(!lowercase.is_known_utf8); +} + +#[test] +fn wtf8_make_ascii_uppercase() { + let mut uppercase = Wtf8Buf::from_str(""); + uppercase.make_ascii_uppercase(); + assert_eq!(uppercase.as_bytes(), b""); + + let mut uppercase = Wtf8Buf::from_str("GrEeN gRaPeS! 🍇"); + uppercase.make_ascii_uppercase(); + assert_eq!(uppercase.as_bytes(), b"GREEN GRAPES! \xf0\x9f\x8d\x87"); + + let mut uppercase = to_owned(unsafe { Wtf8::from_bytes_unchecked(b"\xED\xA0\x80") }); + uppercase.make_ascii_uppercase(); + assert_eq!(uppercase.as_bytes(), b"\xED\xA0\x80"); + assert!(!uppercase.is_known_utf8); +} + +#[test] +fn wtf8_to_owned() { + let string = to_owned(unsafe { Wtf8::from_bytes_unchecked(b"\xED\xA0\x80") }); + assert_eq!(string.as_bytes(), b"\xED\xA0\x80"); + assert!(!string.is_known_utf8); +} + +#[test] +fn wtf8_valid_utf8_boundaries() { + let mut string = Wtf8Buf::from_str("aé 💩"); + string.push(CodePoint::from_u32(0xD800).unwrap()); + string.push(CodePoint::from_u32(0xD800).unwrap()); + string.check_utf8_boundary(0); + string.check_utf8_boundary(1); + string.check_utf8_boundary(3); + string.check_utf8_boundary(4); + string.check_utf8_boundary(8); + string.check_utf8_boundary(14); + assert_eq!(string.len(), 14); + + string.push_char('a'); + string.check_utf8_boundary(14); + string.check_utf8_boundary(15); + + let mut string = Wtf8Buf::from_str("a"); + string.push(CodePoint::from_u32(0xD800).unwrap()); + string.check_utf8_boundary(1); + + let mut string = Wtf8Buf::from_str("\u{D7FF}"); + string.push(CodePoint::from_u32(0xD800).unwrap()); + string.check_utf8_boundary(3); + + let mut string = Wtf8Buf::new(); + string.push(CodePoint::from_u32(0xD800).unwrap()); + string.push_char('\u{D7FF}'); + string.check_utf8_boundary(3); +} + +#[test] +#[should_panic(expected = "byte index 4 is out of bounds")] +fn wtf8_utf8_boundary_out_of_bounds() { + let string = Wtf8::from_str("aé"); + string.check_utf8_boundary(4); +} + +#[test] +#[should_panic(expected = "byte index 1 is not a codepoint boundary")] +fn wtf8_utf8_boundary_inside_codepoint() { + let string = Wtf8::from_str("é"); + string.check_utf8_boundary(1); +} + +#[test] +#[should_panic(expected = "byte index 1 is not a codepoint boundary")] +fn wtf8_utf8_boundary_inside_surrogate() { + let mut string = Wtf8Buf::new(); + string.push(CodePoint::from_u32(0xD800).unwrap()); + string.check_utf8_boundary(1); +} + +#[test] +#[should_panic(expected = "byte index 3 lies between surrogate codepoints")] +fn wtf8_utf8_boundary_between_surrogates() { + let mut string = Wtf8Buf::new(); + string.push(CodePoint::from_u32(0xD800).unwrap()); + string.push(CodePoint::from_u32(0xD800).unwrap()); + string.check_utf8_boundary(3); +} + +#[test] +fn wobbled_wtf8_plus_bytes_isnt_utf8() { + let mut string: Wtf8Buf = to_owned(unsafe { Wtf8::from_bytes_unchecked(b"\xED\xA0\x80") }); + assert!(!string.is_known_utf8); + unsafe { + string.extend_from_slice_unchecked(b"some utf-8"); + } + assert!(!string.is_known_utf8); +} + +#[test] +fn wobbled_wtf8_plus_str_isnt_utf8() { + let mut string: Wtf8Buf = to_owned(unsafe { Wtf8::from_bytes_unchecked(b"\xED\xA0\x80") }); + assert!(!string.is_known_utf8); + string.push_str("some utf-8"); + assert!(!string.is_known_utf8); +} + +#[test] +fn unwobbly_wtf8_plus_utf8_is_utf8() { + let mut string: Wtf8Buf = Wtf8Buf::from_str("hello world"); + assert!(string.is_known_utf8); + string.push_str("some utf-8"); + assert!(string.is_known_utf8); +} |