diff options
| author | bors <bors@rust-lang.org> | 2018-04-12 00:35:33 +0000 |
|---|---|---|
| committer | bors <bors@rust-lang.org> | 2018-04-12 00:35:33 +0000 |
| commit | d26f9e42df76603fd21b0fe5dd1e8001caf9bf81 (patch) | |
| tree | ce66073c04b872fdc4bc51d3a7f51fe32bf8a680 /src | |
| parent | e28ef22ae5ef22a48e8582210bb8be9aba116b81 (diff) | |
| parent | ef41788cf37074e44f70257508c97efd539a7f29 (diff) | |
Auto merge of #49698 - SimonSapin:unicode-for-everyone, r=alexcrichton
Merge the std_unicode crate into the core crate [The standard library facade](https://github.com/rust-lang/rust/issues/27783) has historically contained a number of crates with different roles, but that number has decreased over time. `rand` and `libc` have moved to crates.io, and [`collections` was merged into `alloc`](https://github.com/rust-lang/rust/pull/42648). Today we have `core` that applies everywhere, `std` that expects a full operating system, and `alloc` in-between that only requires a memory allocator (which can be provided by users)… and `std_unicode`, which doesn’t really have a reason to be separate anymore. It contains functionality based on Unicode data tables that can be large, but as long as relevant functions are not called the tables should be removed from binaries by linkers. This deprecates the unstable `std_unicode` crate and moves all of its contents into `core`, replacing them with `pub use` reexports. The crate can be removed later. This also removes the `CharExt` trait (replaced with inherent methods in libcore) and `UnicodeStr` trait (merged into `StrExt`). There traits were both unstable and not intended to be used or named directly. A number of new items are newly-available in libcore and instantly stable there, but only if they were already stable in libstd. Fixes #49319.
Diffstat (limited to 'src')
42 files changed, 1398 insertions, 1537 deletions
diff --git a/src/Cargo.lock b/src/Cargo.lock index a60679e417a..6e7c4b67acf 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -13,7 +13,6 @@ dependencies = [ "compiler_builtins 0.0.0", "core 0.0.0", "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "std_unicode 0.0.0", ] [[package]] diff --git a/src/ci/docker/wasm32-unknown/Dockerfile b/src/ci/docker/wasm32-unknown/Dockerfile index 6c0ec1ad9d4..853923ad947 100644 --- a/src/ci/docker/wasm32-unknown/Dockerfile +++ b/src/ci/docker/wasm32-unknown/Dockerfile @@ -34,4 +34,3 @@ ENV SCRIPT python2.7 /checkout/x.py test --target $TARGETS \ src/test/mir-opt \ src/test/codegen-units \ src/libcore \ - src/libstd_unicode/ \ diff --git a/src/doc/unstable-book/src/language-features/lang-items.md b/src/doc/unstable-book/src/language-features/lang-items.md index c5167418614..6a7aea7f1c2 100644 --- a/src/doc/unstable-book/src/language-features/lang-items.md +++ b/src/doc/unstable-book/src/language-features/lang-items.md @@ -243,7 +243,7 @@ the source code. - `usize`: `libcore/num/mod.rs` - `f32`: `libstd/f32.rs` - `f64`: `libstd/f64.rs` - - `char`: `libstd_unicode/char.rs` + - `char`: `libcore/char.rs` - `slice`: `liballoc/slice.rs` - `str`: `liballoc/str.rs` - `const_ptr`: `libcore/ptr.rs` diff --git a/src/liballoc/Cargo.toml b/src/liballoc/Cargo.toml index 2eb8ea12604..6383bd1e941 100644 --- a/src/liballoc/Cargo.toml +++ b/src/liballoc/Cargo.toml @@ -9,7 +9,6 @@ path = "lib.rs" [dependencies] core = { path = "../libcore" } -std_unicode = { path = "../libstd_unicode" } compiler_builtins = { path = "../rustc/compiler_builtins_shim" } [dev-dependencies] diff --git a/src/liballoc/lib.rs b/src/liballoc/lib.rs index b08bd66b47c..69fc007ab7c 100644 --- a/src/liballoc/lib.rs +++ b/src/liballoc/lib.rs @@ -113,7 +113,7 @@ #![feature(trusted_len)] #![feature(try_reserve)] #![feature(unboxed_closures)] -#![feature(unicode)] +#![feature(unicode_internals)] #![feature(unsize)] #![feature(allocator_internals)] #![feature(on_unimplemented)] @@ -135,8 +135,6 @@ extern crate test; #[cfg(test)] extern crate rand; -extern crate std_unicode; - // Module with internal macros used by other modules (needs to be included before other modules). #[macro_use] mod macros; diff --git a/src/liballoc/str.rs b/src/liballoc/str.rs index d5ef41df0d8..65df93bd3bb 100644 --- a/src/liballoc/str.rs +++ b/src/liballoc/str.rs @@ -45,12 +45,10 @@ use core::str::pattern::{Searcher, ReverseSearcher, DoubleEndedSearcher}; use core::mem; use core::ptr; use core::iter::FusedIterator; -use std_unicode::str::{UnicodeStr, Utf16Encoder}; use vec_deque::VecDeque; use borrow::{Borrow, ToOwned}; use string::String; -use std_unicode; use vec::Vec; use slice::{SliceConcatExt, SliceIndex}; use boxed::Box; @@ -75,7 +73,7 @@ pub use core::str::{from_utf8, from_utf8_mut, Chars, CharIndices, Bytes}; #[stable(feature = "rust1", since = "1.0.0")] pub use core::str::{from_utf8_unchecked, from_utf8_unchecked_mut, ParseBoolError}; #[stable(feature = "rust1", since = "1.0.0")] -pub use std_unicode::str::SplitWhitespace; +pub use core::str::SplitWhitespace; #[stable(feature = "rust1", since = "1.0.0")] pub use core::str::pattern; @@ -147,7 +145,8 @@ impl<S: Borrow<str>> SliceConcatExt<str> for [S] { #[derive(Clone)] #[stable(feature = "encode_utf16", since = "1.8.0")] pub struct EncodeUtf16<'a> { - encoder: Utf16Encoder<Chars<'a>>, + chars: Chars<'a>, + extra: u16, } #[stable(feature = "collection_debug", since = "1.17.0")] @@ -163,12 +162,29 @@ impl<'a> Iterator for EncodeUtf16<'a> { #[inline] fn next(&mut self) -> Option<u16> { - self.encoder.next() + if self.extra != 0 { + let tmp = self.extra; + self.extra = 0; + return Some(tmp); + } + + let mut buf = [0; 2]; + self.chars.next().map(|ch| { + let n = ch.encode_utf16(&mut buf).len(); + if n == 2 { + self.extra = buf[1]; + } + buf[0] + }) } #[inline] fn size_hint(&self) -> (usize, Option<usize>) { - self.encoder.size_hint() + let (low, high) = self.chars.size_hint(); + // every char gets either one u16 or two u16, + // so this iterator is between 1 or 2 times as + // long as the underlying iterator. + (low, high.and_then(|n| n.checked_mul(2))) } } @@ -801,7 +817,7 @@ impl str { #[stable(feature = "split_whitespace", since = "1.1.0")] #[inline] pub fn split_whitespace(&self) -> SplitWhitespace { - UnicodeStr::split_whitespace(self) + StrExt::split_whitespace(self) } /// An iterator over the lines of a string, as string slices. @@ -871,7 +887,7 @@ impl str { /// ``` #[stable(feature = "encode_utf16", since = "1.8.0")] pub fn encode_utf16(&self) -> EncodeUtf16 { - EncodeUtf16 { encoder: Utf16Encoder::new(self[..].chars()) } + EncodeUtf16 { chars: self[..].chars(), extra: 0 } } /// Returns `true` if the given pattern matches a sub-slice of @@ -1571,7 +1587,7 @@ impl str { /// ``` #[stable(feature = "rust1", since = "1.0.0")] pub fn trim(&self) -> &str { - UnicodeStr::trim(self) + StrExt::trim(self) } /// Returns a string slice with leading whitespace removed. @@ -1607,7 +1623,7 @@ impl str { /// ``` #[stable(feature = "rust1", since = "1.0.0")] pub fn trim_left(&self) -> &str { - UnicodeStr::trim_left(self) + StrExt::trim_left(self) } /// Returns a string slice with trailing whitespace removed. @@ -1643,7 +1659,7 @@ impl str { /// ``` #[stable(feature = "rust1", since = "1.0.0")] pub fn trim_right(&self) -> &str { - UnicodeStr::trim_right(self) + StrExt::trim_right(self) } /// Returns a string slice with all prefixes and suffixes that match a @@ -1960,7 +1976,7 @@ impl str { } fn case_ignoreable_then_cased<I: Iterator<Item = char>>(iter: I) -> bool { - use std_unicode::derived_property::{Cased, Case_Ignorable}; + use core::unicode::derived_property::{Cased, Case_Ignorable}; match iter.skip_while(|&c| Case_Ignorable(c)).next() { Some(c) => Cased(c), None => false, @@ -2142,7 +2158,7 @@ impl str { #[stable(feature = "unicode_methods_on_intrinsics", since = "1.27.0")] #[inline] pub fn is_whitespace(&self) -> bool { - UnicodeStr::is_whitespace(self) + StrExt::is_whitespace(self) } /// Returns true if this `str` is entirely alphanumeric, and false otherwise. @@ -2161,7 +2177,7 @@ impl str { #[stable(feature = "unicode_methods_on_intrinsics", since = "1.27.0")] #[inline] pub fn is_alphanumeric(&self) -> bool { - UnicodeStr::is_alphanumeric(self) + StrExt::is_alphanumeric(self) } /// Checks if all characters in this string are within the ASCII range. diff --git a/src/liballoc/string.rs b/src/liballoc/string.rs index b95aae02894..29d759b1f00 100644 --- a/src/liballoc/string.rs +++ b/src/liballoc/string.rs @@ -56,6 +56,7 @@ #![stable(feature = "rust1", since = "1.0.0")] +use core::char::{decode_utf16, REPLACEMENT_CHARACTER}; use core::fmt; use core::hash; use core::iter::{FromIterator, FusedIterator}; @@ -63,8 +64,7 @@ use core::ops::Bound::{Excluded, Included, Unbounded}; use core::ops::{self, Add, AddAssign, Index, IndexMut, RangeBounds}; use core::ptr; use core::str::pattern::Pattern; -use std_unicode::lossy; -use std_unicode::char::{decode_utf16, REPLACEMENT_CHARACTER}; +use core::str::lossy; use borrow::{Cow, ToOwned}; use str::{self, from_boxed_utf8_unchecked, FromStr, Utf8Error, Chars}; diff --git a/src/liballoc/tests/lib.rs b/src/liballoc/tests/lib.rs index 17f1d0464a5..32272169307 100644 --- a/src/liballoc/tests/lib.rs +++ b/src/liballoc/tests/lib.rs @@ -24,12 +24,11 @@ #![feature(string_retain)] #![feature(try_reserve)] #![feature(unboxed_closures)] -#![feature(unicode)] #![feature(exact_chunks)] #![feature(inclusive_range_fields)] extern crate alloc_system; -extern crate std_unicode; +extern crate core; extern crate rand; use std::hash::{Hash, Hasher}; diff --git a/src/liballoc/tests/str.rs b/src/liballoc/tests/str.rs index a14a5d32738..a3f4c385fe2 100644 --- a/src/liballoc/tests/str.rs +++ b/src/liballoc/tests/str.rs @@ -1204,8 +1204,7 @@ fn test_rev_split_char_iterator_no_trailing() { #[test] fn test_utf16_code_units() { - use std_unicode::str::Utf16Encoder; - assert_eq!(Utf16Encoder::new(vec!['é', '\u{1F4A9}'].into_iter()).collect::<Vec<u16>>(), + assert_eq!("é\u{1F4A9}".encode_utf16().collect::<Vec<u16>>(), [0xE9, 0xD83D, 0xDCA9]) } diff --git a/src/liballoc/tests/string.rs b/src/liballoc/tests/string.rs index cb4a17a22d8..17d53e4cf3e 100644 --- a/src/liballoc/tests/string.rs +++ b/src/liballoc/tests/string.rs @@ -132,7 +132,7 @@ fn test_from_utf16() { let s_as_utf16 = s.encode_utf16().collect::<Vec<u16>>(); let u_as_string = String::from_utf16(&u).unwrap(); - assert!(::std_unicode::char::decode_utf16(u.iter().cloned()).all(|r| r.is_ok())); + assert!(::core::char::decode_utf16(u.iter().cloned()).all(|r| r.is_ok())); assert_eq!(s_as_utf16, u); assert_eq!(u_as_string, s); diff --git a/src/libcore/char.rs b/src/libcore/char.rs deleted file mode 100644 index 718c6b893ed..00000000000 --- a/src/libcore/char.rs +++ /dev/null @@ -1,918 +0,0 @@ -// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT -// file at the top-level directory of this distribution and at -// http://rust-lang.org/COPYRIGHT. -// -// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or -// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license -// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -//! Character manipulation. -//! -//! For more details, see ::std_unicode::char (a.k.a. std::char) - -#![allow(non_snake_case)] -#![stable(feature = "core_char", since = "1.2.0")] - -use char_private::is_printable; -use convert::TryFrom; -use fmt::{self, Write}; -use slice; -use str::{from_utf8_unchecked_mut, FromStr}; -use iter::FusedIterator; -use mem::transmute; - -// UTF-8 ranges and tags for encoding characters -const TAG_CONT: u8 = 0b1000_0000; -const TAG_TWO_B: u8 = 0b1100_0000; -const TAG_THREE_B: u8 = 0b1110_0000; -const TAG_FOUR_B: u8 = 0b1111_0000; -const MAX_ONE_B: u32 = 0x80; -const MAX_TWO_B: u32 = 0x800; -const MAX_THREE_B: u32 = 0x10000; - -/* - Lu Uppercase_Letter an uppercase letter - Ll Lowercase_Letter a lowercase letter - Lt Titlecase_Letter a digraphic character, with first part uppercase - Lm Modifier_Letter a modifier letter - Lo Other_Letter other letters, including syllables and ideographs - Mn Nonspacing_Mark a nonspacing combining mark (zero advance width) - Mc Spacing_Mark a spacing combining mark (positive advance width) - Me Enclosing_Mark an enclosing combining mark - Nd Decimal_Number a decimal digit - Nl Letter_Number a letterlike numeric character - No Other_Number a numeric character of other type - Pc Connector_Punctuation a connecting punctuation mark, like a tie - Pd Dash_Punctuation a dash or hyphen punctuation mark - Ps Open_Punctuation an opening punctuation mark (of a pair) - Pe Close_Punctuation a closing punctuation mark (of a pair) - Pi Initial_Punctuation an initial quotation mark - Pf Final_Punctuation a final quotation mark - Po Other_Punctuation a punctuation mark of other type - Sm Math_Symbol a symbol of primarily mathematical use - Sc Currency_Symbol a currency sign - Sk Modifier_Symbol a non-letterlike modifier symbol - So Other_Symbol a symbol of other type - Zs Space_Separator a space character (of various non-zero widths) - Zl Line_Separator U+2028 LINE SEPARATOR only - Zp Paragraph_Separator U+2029 PARAGRAPH SEPARATOR only - Cc Control a C0 or C1 control code - Cf Format a format control character - Cs Surrogate a surrogate code point - Co Private_Use a private-use character - Cn Unassigned a reserved unassigned code point or a noncharacter -*/ - -/// The highest valid code point a `char` can have. -/// -/// A [`char`] is a [Unicode Scalar Value], which means that it is a [Code -/// Point], but only ones within a certain range. `MAX` is the highest valid -/// code point that's a valid [Unicode Scalar Value]. -/// -/// [`char`]: ../../std/primitive.char.html -/// [Unicode Scalar Value]: http://www.unicode.org/glossary/#unicode_scalar_value -/// [Code Point]: http://www.unicode.org/glossary/#code_point -#[stable(feature = "rust1", since = "1.0.0")] -pub const MAX: char = '\u{10ffff}'; - -/// Converts a `u32` to a `char`. -/// -/// Note that all [`char`]s are valid [`u32`]s, and can be cast to one with -/// [`as`]: -/// -/// ``` -/// let c = '💯'; -/// let i = c as u32; -/// -/// assert_eq!(128175, i); -/// ``` -/// -/// However, the reverse is not true: not all valid [`u32`]s are valid -/// [`char`]s. `from_u32()` will return `None` if the input is not a valid value -/// for a [`char`]. -/// -/// [`char`]: ../../std/primitive.char.html -/// [`u32`]: ../../std/primitive.u32.html -/// [`as`]: ../../book/first-edition/casting-between-types.html#as -/// -/// For an unsafe version of this function which ignores these checks, see -/// [`from_u32_unchecked`]. -/// -/// [`from_u32_unchecked`]: fn.from_u32_unchecked.html -/// -/// # Examples -/// -/// Basic usage: -/// -/// ``` -/// use std::char; -/// -/// let c = char::from_u32(0x2764); -/// -/// assert_eq!(Some('❤'), c); -/// ``` -/// -/// Returning `None` when the input is not a valid [`char`]: -/// -/// ``` -/// use std::char; -/// -/// let c = char::from_u32(0x110000); -/// -/// assert_eq!(None, c); -/// ``` -#[inline] -#[stable(feature = "rust1", since = "1.0.0")] -pub fn from_u32(i: u32) -> Option<char> { - char::try_from(i).ok() -} - -/// Converts a `u32` to a `char`, ignoring validity. -/// -/// Note that all [`char`]s are valid [`u32`]s, and can be cast to one with -/// [`as`]: -/// -/// ``` -/// let c = '💯'; -/// let i = c as u32; -/// -/// assert_eq!(128175, i); -/// ``` -/// -/// However, the reverse is not true: not all valid [`u32`]s are valid -/// [`char`]s. `from_u32_unchecked()` will ignore this, and blindly cast to -/// [`char`], possibly creating an invalid one. -/// -/// [`char`]: ../../std/primitive.char.html -/// [`u32`]: ../../std/primitive.u32.html -/// [`as`]: ../../book/first-edition/casting-between-types.html#as -/// -/// # Safety -/// -/// This function is unsafe, as it may construct invalid `char` values. -/// -/// For a safe version of this function, see the [`from_u32`] function. -/// -/// [`from_u32`]: fn.from_u32.html -/// -/// # Examples -/// -/// Basic usage: -/// -/// ``` -/// use std::char; -/// -/// let c = unsafe { char::from_u32_unchecked(0x2764) }; -/// -/// assert_eq!('❤', c); -/// ``` -#[inline] -#[stable(feature = "char_from_unchecked", since = "1.5.0")] -pub unsafe fn from_u32_unchecked(i: u32) -> char { - transmute(i) -} - -#[stable(feature = "char_convert", since = "1.13.0")] -impl From<char> for u32 { - #[inline] - fn from(c: char) -> Self { - c as u32 - } -} - -/// Maps a byte in 0x00...0xFF to a `char` whose code point has the same value, in U+0000 to U+00FF. -/// -/// Unicode is designed such that this effectively decodes bytes -/// with the character encoding that IANA calls ISO-8859-1. -/// This encoding is compatible with ASCII. -/// -/// Note that this is different from ISO/IEC 8859-1 a.k.a. ISO 8859-1 (with one less hyphen), -/// which leaves some "blanks", byte values that are not assigned to any character. -/// ISO-8859-1 (the IANA one) assigns them to the C0 and C1 control codes. -/// -/// Note that this is *also* different from Windows-1252 a.k.a. code page 1252, -/// which is a superset ISO/IEC 8859-1 that assigns some (not all!) blanks -/// to punctuation and various Latin characters. -/// -/// To confuse things further, [on the Web](https://encoding.spec.whatwg.org/) -/// `ascii`, `iso-8859-1`, and `windows-1252` are all aliases -/// for a superset of Windows-1252 that fills the remaining blanks with corresponding -/// C0 and C1 control codes. -#[stable(feature = "char_convert", since = "1.13.0")] -impl From<u8> for char { - #[inline] - fn from(i: u8) -> Self { - i as char - } -} - - -/// An error which can be returned when parsing a char. -#[stable(feature = "char_from_str", since = "1.20.0")] -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct ParseCharError { - kind: CharErrorKind, -} - -impl ParseCharError { - #[unstable(feature = "char_error_internals", - reason = "this method should not be available publicly", - issue = "0")] - #[doc(hidden)] - pub fn __description(&self) -> &str { - match self.kind { - CharErrorKind::EmptyString => { - "cannot parse char from empty string" - }, - CharErrorKind::TooManyChars => "too many characters in string" - } - } -} - -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -enum CharErrorKind { - EmptyString, - TooManyChars, -} - -#[stable(feature = "char_from_str", since = "1.20.0")] -impl fmt::Display for ParseCharError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - self.__description().fmt(f) - } -} - - -#[stable(feature = "char_from_str", since = "1.20.0")] -impl FromStr for char { - type Err = ParseCharError; - - #[inline] - fn from_str(s: &str) -> Result<Self, Self::Err> { - let mut chars = s.chars(); - match (chars.next(), chars.next()) { - (None, _) => { - Err(ParseCharError { kind: CharErrorKind::EmptyString }) - }, - (Some(c), None) => Ok(c), - _ => { - Err(ParseCharError { kind: CharErrorKind::TooManyChars }) - } - } - } -} - - -#[stable(feature = "try_from", since = "1.26.0")] -impl TryFrom<u32> for char { - type Error = CharTryFromError; - - #[inline] - fn try_from(i: u32) -> Result<Self, Self::Error> { - if (i > MAX as u32) || (i >= 0xD800 && i <= 0xDFFF) { - Err(CharTryFromError(())) - } else { - Ok(unsafe { from_u32_unchecked(i) }) - } - } -} - -/// The error type returned when a conversion from u32 to char fails. -#[stable(feature = "try_from", since = "1.26.0")] -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub struct CharTryFromError(()); - -#[stable(feature = "try_from", since = "1.26.0")] -impl fmt::Display for CharTryFromError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - "converted integer out of range for `char`".fmt(f) - } -} - -/// Converts a digit in the given radix to a `char`. -/// -/// A 'radix' here is sometimes also called a 'base'. A radix of two -/// indicates a binary number, a radix of ten, decimal, and a radix of -/// sixteen, hexadecimal, to give some common values. Arbitrary -/// radices are supported. -/// -/// `from_digit()` will return `None` if the input is not a digit in -/// the given radix. -/// -/// # Panics -/// -/// Panics if given a radix larger than 36. -/// -/// # Examples -/// -/// Basic usage: -/// -/// ``` -/// use std::char; -/// -/// let c = char::from_digit(4, 10); -/// -/// assert_eq!(Some('4'), c); -/// -/// // Decimal 11 is a single digit in base 16 -/// let c = char::from_digit(11, 16); -/// -/// assert_eq!(Some('b'), c); -/// ``` -/// -/// Returning `None` when the input is not a digit: -/// -/// ``` -/// use std::char; -/// -/// let c = char::from_digit(20, 10); -/// -/// assert_eq!(None, c); -/// ``` -/// -/// Passing a large radix, causing a panic: -/// -/// ``` -/// use std::thread; -/// use std::char; -/// -/// let result = thread::spawn(|| { -/// // this panics -/// let c = char::from_digit(1, 37); -/// }).join(); -/// -/// assert!(result.is_err()); -/// ``` -#[inline] -#[stable(feature = "rust1", since = "1.0.0")] -pub fn from_digit(num: u32, radix: u32) -> Option<char> { - if radix > 36 { - panic!("from_digit: radix is too high (maximum 36)"); - } - if num < radix { - let num = num as u8; - if num < 10 { - Some((b'0' + num) as char) - } else { - Some((b'a' + num - 10) as char) - } - } else { - None - } -} - -// NB: the stabilization and documentation for this trait is in -// unicode/char.rs, not here -#[allow(missing_docs)] // docs in libunicode/u_char.rs -#[doc(hidden)] -#[unstable(feature = "core_char_ext", - reason = "the stable interface is `impl char` in later crate", - issue = "32110")] -pub trait CharExt { - #[stable(feature = "core", since = "1.6.0")] - fn is_digit(self, radix: u32) -> bool; - #[stable(feature = "core", since = "1.6.0")] - fn to_digit(self, radix: u32) -> Option<u32>; - #[stable(feature = "core", since = "1.6.0")] - fn escape_unicode(self) -> EscapeUnicode; - #[stable(feature = "core", since = "1.6.0")] - fn escape_default(self) -> EscapeDefault; - #[stable(feature = "char_escape_debug", since = "1.20.0")] - fn escape_debug(self) -> EscapeDebug; - #[stable(feature = "core", since = "1.6.0")] - fn len_utf8(self) -> usize; - #[stable(feature = "core", since = "1.6.0")] - fn len_utf16(self) -> usize; - #[stable(feature = "unicode_encode_char", since = "1.15.0")] - fn encode_utf8(self, dst: &mut [u8]) -> &mut str; - #[stable(feature = "unicode_encode_char", since = "1.15.0")] - fn encode_utf16(self, dst: &mut [u16]) -> &mut [u16]; -} - -#[stable(feature = "core", since = "1.6.0")] -impl CharExt for char { - #[inline] - fn is_digit(self, radix: u32) -> bool { - self.to_digit(radix).is_some() - } - - #[inline] - fn to_digit(self, radix: u32) -> Option<u32> { - if radix > 36 { - panic!("to_digit: radix is too high (maximum 36)"); - } - let val = match self { - '0' ... '9' => self as u32 - '0' as u32, - 'a' ... 'z' => self as u32 - 'a' as u32 + 10, - 'A' ... 'Z' => self as u32 - 'A' as u32 + 10, - _ => return None, - }; - if val < radix { Some(val) } - else { None } - } - - #[inline] - fn escape_unicode(self) -> EscapeUnicode { - let c = self as u32; - - // or-ing 1 ensures that for c==0 the code computes that one - // digit should be printed and (which is the same) avoids the - // (31 - 32) underflow - let msb = 31 - (c | 1).leading_zeros(); - - // the index of the most significant hex digit - let ms_hex_digit = msb / 4; - EscapeUnicode { - c: self, - state: EscapeUnicodeState::Backslash, - hex_digit_idx: ms_hex_digit as usize, - } - } - - #[inline] - fn escape_default(self) -> EscapeDefault { - let init_state = match self { - '\t' => EscapeDefaultState::Backslash('t'), - '\r' => EscapeDefaultState::Backslash('r'), - '\n' => EscapeDefaultState::Backslash('n'), - '\\' | '\'' | '"' => EscapeDefaultState::Backslash(self), - '\x20' ... '\x7e' => EscapeDefaultState::Char(self), - _ => EscapeDefaultState::Unicode(self.escape_unicode()) - }; - EscapeDefault { state: init_state } - } - - #[inline] - fn escape_debug(self) -> EscapeDebug { - let init_state = match self { - '\t' => EscapeDefaultState::Backslash('t'), - '\r' => EscapeDefaultState::Backslash('r'), - '\n' => EscapeDefaultState::Backslash('n'), - '\\' | '\'' | '"' => EscapeDefaultState::Backslash(self), - c if is_printable(c) => EscapeDefaultState::Char(c), - c => EscapeDefaultState::Unicode(c.escape_unicode()), - }; - EscapeDebug(EscapeDefault { state: init_state }) - } - - #[inline] - fn len_utf8(self) -> usize { - let code = self as u32; - if code < MAX_ONE_B { - 1 - } else if code < MAX_TWO_B { - 2 - } else if code < MAX_THREE_B { - 3 - } else { - 4 - } - } - - #[inline] - fn len_utf16(self) -> usize { - let ch = self as u32; - if (ch & 0xFFFF) == ch { 1 } else { 2 } - } - - #[inline] - fn encode_utf8(self, dst: &mut [u8]) -> &mut str { - let code = self as u32; - unsafe { - let len = - if code < MAX_ONE_B && !dst.is_empty() { - *dst.get_unchecked_mut(0) = code as u8; - 1 - } else if code < MAX_TWO_B && dst.len() >= 2 { - *dst.get_unchecked_mut(0) = (code >> 6 & 0x1F) as u8 | TAG_TWO_B; - *dst.get_unchecked_mut(1) = (code & 0x3F) as u8 | TAG_CONT; - 2 - } else if code < MAX_THREE_B && dst.len() >= 3 { - *dst.get_unchecked_mut(0) = (code >> 12 & 0x0F) as u8 | TAG_THREE_B; - *dst.get_unchecked_mut(1) = (code >> 6 & 0x3F) as u8 | TAG_CONT; - *dst.get_unchecked_mut(2) = (code & 0x3F) as u8 | TAG_CONT; - 3 - } else if dst.len() >= 4 { - *dst.get_unchecked_mut(0) = (code >> 18 & 0x07) as u8 | TAG_FOUR_B; - *dst.get_unchecked_mut(1) = (code >> 12 & 0x3F) as u8 | TAG_CONT; - *dst.get_unchecked_mut(2) = (code >> 6 & 0x3F) as u8 | TAG_CONT; - *dst.get_unchecked_mut(3) = (code & 0x3F) as u8 | TAG_CONT; - 4 - } else { - panic!("encode_utf8: need {} bytes to encode U+{:X}, but the buffer has {}", - from_u32_unchecked(code).len_utf8(), - code, - dst.len()) - }; - from_utf8_unchecked_mut(dst.get_unchecked_mut(..len)) - } - } - - #[inline] - fn encode_utf16(self, dst: &mut [u16]) -> &mut [u16] { - let mut code = self as u32; - unsafe { - if (code & 0xFFFF) == code && !dst.is_empty() { - // The BMP falls through (assuming non-surrogate, as it should) - *dst.get_unchecked_mut(0) = code as u16; - slice::from_raw_parts_mut(dst.as_mut_ptr(), 1) - } else if dst.len() >= 2 { - // Supplementary planes break into surrogates. - code -= 0x1_0000; - *dst.get_unchecked_mut(0) = 0xD800 | ((code >> 10) as u16); - *dst.get_unchecked_mut(1) = 0xDC00 | ((code as u16) & 0x3FF); - slice::from_raw_parts_mut(dst.as_mut_ptr(), 2) - } else { - panic!("encode_utf16: need {} units to encode U+{:X}, but the buffer has {}", - from_u32_unchecked(code).len_utf16(), - code, - dst.len()) - } - } - } -} - -/// Returns an iterator that yields the hexadecimal Unicode escape of a -/// character, as `char`s. -/// -/// This `struct` is created by the [`escape_unicode`] method on [`char`]. See -/// its documentation for more. -/// -/// [`escape_unicode`]: ../../std/primitive.char.html#method.escape_unicode -/// [`char`]: ../../std/primitive.char.html -#[derive(Clone, Debug)] -#[stable(feature = "rust1", since = "1.0.0")] -pub struct EscapeUnicode { - c: char, - state: EscapeUnicodeState, - - // The index of the next hex digit to be printed (0 if none), - // i.e. the number of remaining hex digits to be printed; - // increasing from the least significant digit: 0x543210 - hex_digit_idx: usize, -} - -// The enum values are ordered so that their representation is the -// same as the remaining length (besides the hexadecimal digits). This -// likely makes `len()` a single load from memory) and inline-worth. -#[derive(Clone, Debug)] -enum EscapeUnicodeState { - Done, - RightBrace, - Value, - LeftBrace, - Type, - Backslash, -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Iterator for EscapeUnicode { - type Item = char; - - fn next(&mut self) -> Option<char> { - match self.state { - EscapeUnicodeState::Backslash => { - self.state = EscapeUnicodeState::Type; - Some('\\') - } - EscapeUnicodeState::Type => { - self.state = EscapeUnicodeState::LeftBrace; - Some('u') - } - EscapeUnicodeState::LeftBrace => { - self.state = EscapeUnicodeState::Value; - Some('{') - } - EscapeUnicodeState::Value => { - let hex_digit = ((self.c as u32) >> (self.hex_digit_idx * 4)) & 0xf; - let c = from_digit(hex_digit, 16).unwrap(); - if self.hex_digit_idx == 0 { - self.state = EscapeUnicodeState::RightBrace; - } else { - self.hex_digit_idx -= 1; - } - Some(c) - } - EscapeUnicodeState::RightBrace => { - self.state = EscapeUnicodeState::Done; - Some('}') - } - EscapeUnicodeState::Done => None, - } - } - - #[inline] - fn size_hint(&self) -> (usize, Option<usize>) { - let n = self.len(); - (n, Some(n)) - } - - #[inline] - fn count(self) -> usize { - self.len() - } - - fn last(self) -> Option<char> { - match self.state { - EscapeUnicodeState::Done => None, - - EscapeUnicodeState::RightBrace | - EscapeUnicodeState::Value | - EscapeUnicodeState::LeftBrace | - EscapeUnicodeState::Type | - EscapeUnicodeState::Backslash => Some('}'), - } - } -} - -#[stable(feature = "exact_size_escape", since = "1.11.0")] -impl ExactSizeIterator for EscapeUnicode { - #[inline] - fn len(&self) -> usize { - // The match is a single memory access with no branching - self.hex_digit_idx + match self.state { - EscapeUnicodeState::Done => 0, - EscapeUnicodeState::RightBrace => 1, - EscapeUnicodeState::Value => 2, - EscapeUnicodeState::LeftBrace => 3, - EscapeUnicodeState::Type => 4, - EscapeUnicodeState::Backslash => 5, - } - } -} - -#[stable(feature = "fused", since = "1.26.0")] -impl FusedIterator for EscapeUnicode {} - -#[stable(feature = "char_struct_display", since = "1.16.0")] -impl fmt::Display for EscapeUnicode { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - for c in self.clone() { - f.write_char(c)?; - } - Ok(()) - } -} - -/// An iterator that yields the literal escape code of a `char`. -/// -/// This `struct` is created by the [`escape_default`] method on [`char`]. See -/// its documentation for more. -/// -/// [`escape_default`]: ../../std/primitive.char.html#method.escape_default -/// [`char`]: ../../std/primitive.char.html -#[derive(Clone, Debug)] -#[stable(feature = "rust1", since = "1.0.0")] -pub struct EscapeDefault { - state: EscapeDefaultState -} - -#[derive(Clone, Debug)] -enum EscapeDefaultState { - Done, - Char(char), - Backslash(char), - Unicode(EscapeUnicode), -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl Iterator for EscapeDefault { - type Item = char; - - fn next(&mut self) -> Option<char> { - match self.state { - EscapeDefaultState::Backslash(c) => { - self.state = EscapeDefaultState::Char(c); - Some('\\') - } - EscapeDefaultState::Char(c) => { - self.state = EscapeDefaultState::Done; - Some(c) - } - EscapeDefaultState::Done => None, - EscapeDefaultState::Unicode(ref mut iter) => iter.next(), - } - } - - #[inline] - fn size_hint(&self) -> (usize, Option<usize>) { - let n = self.len(); - (n, Some(n)) - } - - #[inline] - fn count(self) -> usize { - self.len() - } - - fn nth(&mut self, n: usize) -> Option<char> { - match self.state { - EscapeDefaultState::Backslash(c) if n == 0 => { - self.state = EscapeDefaultState::Char(c); - Some('\\') - }, - EscapeDefaultState::Backslash(c) if n == 1 => { - self.state = EscapeDefaultState::Done; - Some(c) - }, - EscapeDefaultState::Backslash(_) => { - self.state = EscapeDefaultState::Done; - None - }, - EscapeDefaultState::Char(c) => { - self.state = EscapeDefaultState::Done; - - if n == 0 { - Some(c) - } else { - None - } - }, - EscapeDefaultState::Done => return None, - EscapeDefaultState::Unicode(ref mut i) => return i.nth(n), - } - } - - fn last(self) -> Option<char> { - match self.state { - EscapeDefaultState::Unicode(iter) => iter.last(), - EscapeDefaultState::Done => None, - EscapeDefaultState::Backslash(c) | EscapeDefaultState::Char(c) => Some(c), - } - } -} - -#[stable(feature = "exact_size_escape", since = "1.11.0")] -impl ExactSizeIterator for EscapeDefault { - fn len(&self) -> usize { - match self.state { - EscapeDefaultState::Done => 0, - EscapeDefaultState::Char(_) => 1, - EscapeDefaultState::Backslash(_) => 2, - EscapeDefaultState::Unicode(ref iter) => iter.len(), - } - } -} - -#[stable(feature = "fused", since = "1.26.0")] -impl FusedIterator for EscapeDefault {} - -#[stable(feature = "char_struct_display", since = "1.16.0")] -impl fmt::Display for EscapeDefault { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - for c in self.clone() { - f.write_char(c)?; - } - Ok(()) - } -} - -/// An iterator that yields the literal escape code of a `char`. -/// -/// This `struct` is created by the [`escape_debug`] method on [`char`]. See its -/// documentation for more. -/// -/// [`escape_debug`]: ../../std/primitive.char.html#method.escape_debug -/// [`char`]: ../../std/primitive.char.html -#[stable(feature = "char_escape_debug", since = "1.20.0")] -#[derive(Clone, Debug)] -pub struct EscapeDebug(EscapeDefault); - -#[stable(feature = "char_escape_debug", since = "1.20.0")] -impl Iterator for EscapeDebug { - type Item = char; - fn next(&mut self) -> Option<char> { self.0.next() } - fn size_hint(&self) -> (usize, Option<usize>) { self.0.size_hint() } -} - -#[stable(feature = "char_escape_debug", since = "1.20.0")] -impl ExactSizeIterator for EscapeDebug { } - -#[stable(feature = "fused", since = "1.26.0")] -impl FusedIterator for EscapeDebug {} - -#[stable(feature = "char_escape_debug", since = "1.20.0")] -impl fmt::Display for EscapeDebug { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - fmt::Display::fmt(&self.0, f) - } -} - - - -/// An iterator over an iterator of bytes of the characters the bytes represent -/// as UTF-8 -#[unstable(feature = "decode_utf8", issue = "33906")] -#[derive(Clone, Debug)] -pub struct DecodeUtf8<I: Iterator<Item = u8>>(::iter::Peekable<I>); - -/// Decodes an `Iterator` of bytes as UTF-8. -#[unstable(feature = "decode_utf8", issue = "33906")] -#[inline] -pub fn decode_utf8<I: IntoIterator<Item = u8>>(i: I) -> DecodeUtf8<I::IntoIter> { - DecodeUtf8(i.into_iter().peekable()) -} - -/// `<DecodeUtf8 as Iterator>::next` returns this for an invalid input sequence. -#[unstable(feature = "decode_utf8", issue = "33906")] -#[derive(PartialEq, Eq, Debug)] -pub struct InvalidSequence(()); - -#[unstable(feature = "decode_utf8", issue = "33906")] -impl<I: Iterator<Item = u8>> Iterator for DecodeUtf8<I> { - type Item = Result<char, InvalidSequence>; - #[inline] - - fn next(&mut self) -> Option<Result<char, InvalidSequence>> { - self.0.next().map(|first_byte| { - // Emit InvalidSequence according to - // Unicode §5.22 Best Practice for U+FFFD Substitution - // http://www.unicode.org/versions/Unicode9.0.0/ch05.pdf#G40630 - - // Roughly: consume at least one byte, - // then validate one byte at a time and stop before the first unexpected byte - // (which might be the valid start of the next byte sequence). - - let mut code_point; - macro_rules! first_byte { - ($mask: expr) => { - code_point = u32::from(first_byte & $mask) - } - } - macro_rules! continuation_byte { - () => { continuation_byte!(0x80...0xBF) }; - ($range: pat) => { - match self.0.peek() { - Some(&byte @ $range) => { - code_point = (code_point << 6) | u32::from(byte & 0b0011_1111); - self.0.next(); - } - _ => return Err(InvalidSequence(())) - } - } - } - - match first_byte { - 0x00...0x7F => { - first_byte!(0b1111_1111); - } - 0xC2...0xDF => { - first_byte!(0b0001_1111); - continuation_byte!(); - } - 0xE0 => { - first_byte!(0b0000_1111); - continuation_byte!(0xA0...0xBF); // 0x80...0x9F here are overlong - continuation_byte!(); - } - 0xE1...0xEC | 0xEE...0xEF => { - first_byte!(0b0000_1111); - continuation_byte!(); - continuation_byte!(); - } - 0xED => { - first_byte!(0b0000_1111); - continuation_byte!(0x80...0x9F); // 0xA0..0xBF here are surrogates - continuation_byte!(); - } - 0xF0 => { - first_byte!(0b0000_0111); - continuation_byte!(0x90...0xBF); // 0x80..0x8F here are overlong - continuation_byte!(); - continuation_byte!(); - } - 0xF1...0xF3 => { - first_byte!(0b0000_0111); - continuation_byte!(); - continuation_byte!(); - continuation_byte!(); - } - 0xF4 => { - first_byte!(0b0000_0111); - continuation_byte!(0x80...0x8F); // 0x90..0xBF here are beyond char::MAX - continuation_byte!(); - continuation_byte!(); - } - _ => return Err(InvalidSequence(())) // Illegal first byte, overlong, or beyond MAX - } - unsafe { - Ok(from_u32_unchecked(code_point)) - } - }) - } - - #[inline] - fn size_hint(&self) -> (usize, Option<usize>) { - let (lower, upper) = self.0.size_hint(); - - // A code point is at most 4 bytes long. - let min_code_points = lower / 4; - - (min_code_points, upper) - } -} - -#[unstable(feature = "decode_utf8", issue = "33906")] -impl<I: FusedIterator<Item = u8>> FusedIterator for DecodeUtf8<I> {} diff --git a/src/libcore/char/convert.rs b/src/libcore/char/convert.rs new file mode 100644 index 00000000000..150562a4a9b --- /dev/null +++ b/src/libcore/char/convert.rs @@ -0,0 +1,304 @@ +// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license +// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +//! Character conversions. + +use convert::TryFrom; +use fmt; +use mem::transmute; +use str::FromStr; +use super::MAX; + +/// Converts a `u32` to a `char`. +/// +/// Note that all [`char`]s are valid [`u32`]s, and can be cast to one with +/// [`as`]: +/// +/// ``` +/// let c = '💯'; +/// let i = c as u32; +/// +/// assert_eq!(128175, i); +/// ``` +/// +/// However, the reverse is not true: not all valid [`u32`]s are valid +/// [`char`]s. `from_u32()` will return `None` if the input is not a valid value +/// for a [`char`]. +/// +/// [`char`]: ../../std/primitive.char.html +/// [`u32`]: ../../std/primitive.u32.html +/// [`as`]: ../../book/first-edition/casting-between-types.html#as +/// +/// For an unsafe version of this function which ignores these checks, see +/// [`from_u32_unchecked`]. +/// +/// [`from_u32_unchecked`]: fn.from_u32_unchecked.html +/// +/// # Examples +/// +/// Basic usage: +/// +/// ``` +/// use std::char; +/// +/// let c = char::from_u32(0x2764); +/// +/// assert_eq!(Some('❤'), c); +/// ``` +/// +/// Returning `None` when the input is not a valid [`char`]: +/// +/// ``` +/// use std::char; +/// +/// let c = char::from_u32(0x110000); +/// +/// assert_eq!(None, c); +/// ``` +#[inline] +#[stable(feature = "rust1", since = "1.0.0")] +pub fn from_u32(i: u32) -> Option<char> { + char::try_from(i).ok() +} + +/// Converts a `u32` to a `char`, ignoring validity. +/// +/// Note that all [`char`]s are valid [`u32`]s, and can be cast to one with +/// [`as`]: +/// +/// ``` +/// let c = '💯'; +/// let i = c as u32; +/// +/// assert_eq!(128175, i); +/// ``` +/// +/// However, the reverse is not true: not all valid [`u32`]s are valid +/// [`char`]s. `from_u32_unchecked()` will ignore this, and blindly cast to +/// [`char`], possibly creating an invalid one. +/// +/// [`char`]: ../../std/primitive.char.html +/// [`u32`]: ../../std/primitive.u32.html +/// [`as`]: ../../book/first-edition/casting-between-types.html#as +/// +/// # Safety +/// +/// This function is unsafe, as it may construct invalid `char` values. +/// +/// For a safe version of this function, see the [`from_u32`] function. +/// +/// [`from_u32`]: fn.from_u32.html +/// +/// # Examples +/// +/// Basic usage: +/// +/// ``` +/// use std::char; +/// +/// let c = unsafe { char::from_u32_unchecked(0x2764) }; +/// +/// assert_eq!('❤', c); +/// ``` +#[inline] +#[stable(feature = "char_from_unchecked", since = "1.5.0")] +pub unsafe fn from_u32_unchecked(i: u32) -> char { + transmute(i) +} + +#[stable(feature = "char_convert", since = "1.13.0")] +impl From<char> for u32 { + #[inline] + fn from(c: char) -> Self { + c as u32 + } +} + +/// Maps a byte in 0x00...0xFF to a `char` whose code point has the same value, in U+0000 to U+00FF. +/// +/// Unicode is designed such that this effectively decodes bytes +/// with the character encoding that IANA calls ISO-8859-1. +/// This encoding is compatible with ASCII. +/// +/// Note that this is different from ISO/IEC 8859-1 a.k.a. ISO 8859-1 (with one less hyphen), +/// which leaves some "blanks", byte values that are not assigned to any character. +/// ISO-8859-1 (the IANA one) assigns them to the C0 and C1 control codes. +/// +/// Note that this is *also* different from Windows-1252 a.k.a. code page 1252, +/// which is a superset ISO/IEC 8859-1 that assigns some (not all!) blanks +/// to punctuation and various Latin characters. +/// +/// To confuse things further, [on the Web](https://encoding.spec.whatwg.org/) +/// `ascii`, `iso-8859-1`, and `windows-1252` are all aliases +/// for a superset of Windows-1252 that fills the remaining blanks with corresponding +/// C0 and C1 control codes. +#[stable(feature = "char_convert", since = "1.13.0")] +impl From<u8> for char { + #[inline] + fn from(i: u8) -> Self { + i as char + } +} + + +/// An error which can be returned when parsing a char. +#[stable(feature = "char_from_str", since = "1.20.0")] +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ParseCharError { + kind: CharErrorKind, +} + +impl ParseCharError { + #[unstable(feature = "char_error_internals", + reason = "this method should not be available publicly", + issue = "0")] + #[doc(hidden)] + pub fn __description(&self) -> &str { + match self.kind { + CharErrorKind::EmptyString => { + "cannot parse char from empty string" + }, + CharErrorKind::TooManyChars => "too many characters in string" + } + } +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +enum CharErrorKind { + EmptyString, + TooManyChars, +} + +#[stable(feature = "char_from_str", since = "1.20.0")] +impl fmt::Display for ParseCharError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.__description().fmt(f) + } +} + + +#[stable(feature = "char_from_str", since = "1.20.0")] +impl FromStr for char { + type Err = ParseCharError; + + #[inline] + fn from_str(s: &str) -> Result<Self, Self::Err> { + let mut chars = s.chars(); + match (chars.next(), chars.next()) { + (None, _) => { + Err(ParseCharError { kind: CharErrorKind::EmptyString }) + }, + (Some(c), None) => Ok(c), + _ => { + Err(ParseCharError { kind: CharErrorKind::TooManyChars }) + } + } + } +} + + +#[stable(feature = "try_from", since = "1.26.0")] +impl TryFrom<u32> for char { + type Error = CharTryFromError; + + #[inline] + fn try_from(i: u32) -> Result<Self, Self::Error> { + if (i > MAX as u32) || (i >= 0xD800 && i <= 0xDFFF) { + Err(CharTryFromError(())) + } else { + Ok(unsafe { from_u32_unchecked(i) }) + } + } +} + +/// The error type returned when a conversion from u32 to char fails. +#[stable(feature = "try_from", since = "1.26.0")] +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub struct CharTryFromError(()); + +#[stable(feature = "try_from", since = "1.26.0")] +impl fmt::Display for CharTryFromError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + "converted integer out of range for `char`".fmt(f) + } +} + +/// Converts a digit in the given radix to a `char`. +/// +/// A 'radix' here is sometimes also called a 'base'. A radix of two +/// indicates a binary number, a radix of ten, decimal, and a radix of +/// sixteen, hexadecimal, to give some common values. Arbitrary +/// radices are supported. +/// +/// `from_digit()` will return `None` if the input is not a digit in +/// the given radix. +/// +/// # Panics +/// +/// Panics if given a radix larger than 36. +/// +/// # Examples +/// +/// Basic usage: +/// +/// ``` +/// use std::char; +/// +/// let c = char::from_digit(4, 10); +/// +/// assert_eq!(Some('4'), c); +/// +/// // Decimal 11 is a single digit in base 16 +/// let c = char::from_digit(11, 16); +/// +/// assert_eq!(Some('b'), c); +/// ``` +/// +/// Returning `None` when the input is not a digit: +/// +/// ``` +/// use std::char; +/// +/// let c = char::from_digit(20, 10); +/// +/// assert_eq!(None, c); +/// ``` +/// +/// Passing a large radix, causing a panic: +/// +/// ``` +/// use std::thread; +/// use std::char; +/// +/// let result = thread::spawn(|| { +/// // this panics +/// let c = char::from_digit(1, 37); +/// }).join(); +/// +/// assert!(result.is_err()); +/// ``` +#[inline] +#[stable(feature = "rust1", since = "1.0.0")] +pub fn from_digit(num: u32, radix: u32) -> Option<char> { + if radix > 36 { + panic!("from_digit: radix is too high (maximum 36)"); + } + if num < radix { + let num = num as u8; + if num < 10 { + Some((b'0' + num) as char) + } else { + Some((b'a' + num - 10) as char) + } + } else { + None + } +} + diff --git a/src/libcore/char/decode.rs b/src/libcore/char/decode.rs new file mode 100644 index 00000000000..48b531104f8 --- /dev/null +++ b/src/libcore/char/decode.rs @@ -0,0 +1,259 @@ +// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license +// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +//! UTF-8 and UTF-16 decoding iterators + +use fmt; +use iter::FusedIterator; +use super::from_u32_unchecked; + +/// An iterator over an iterator of bytes of the characters the bytes represent +/// as UTF-8 +#[unstable(feature = "decode_utf8", issue = "33906")] +#[derive(Clone, Debug)] +pub struct DecodeUtf8<I: Iterator<Item = u8>>(::iter::Peekable<I>); + +/// Decodes an `Iterator` of bytes as UTF-8. +#[unstable(feature = "decode_utf8", issue = "33906")] +#[inline] +pub fn decode_utf8<I: IntoIterator<Item = u8>>(i: I) -> DecodeUtf8<I::IntoIter> { + DecodeUtf8(i.into_iter().peekable()) +} + +/// `<DecodeUtf8 as Iterator>::next` returns this for an invalid input sequence. +#[unstable(feature = "decode_utf8", issue = "33906")] +#[derive(PartialEq, Eq, Debug)] +pub struct InvalidSequence(()); + +#[unstable(feature = "decode_utf8", issue = "33906")] +impl<I: Iterator<Item = u8>> Iterator for DecodeUtf8<I> { + type Item = Result<char, InvalidSequence>; + #[inline] + + fn next(&mut self) -> Option<Result<char, InvalidSequence>> { + self.0.next().map(|first_byte| { + // Emit InvalidSequence according to + // Unicode §5.22 Best Practice for U+FFFD Substitution + // http://www.unicode.org/versions/Unicode9.0.0/ch05.pdf#G40630 + + // Roughly: consume at least one byte, + // then validate one byte at a time and stop before the first unexpected byte + // (which might be the valid start of the next byte sequence). + + let mut code_point; + macro_rules! first_byte { + ($mask: expr) => { + code_point = u32::from(first_byte & $mask) + } + } + macro_rules! continuation_byte { + () => { continuation_byte!(0x80...0xBF) }; + ($range: pat) => { + match self.0.peek() { + Some(&byte @ $range) => { + code_point = (code_point << 6) | u32::from(byte & 0b0011_1111); + self.0.next(); + } + _ => return Err(InvalidSequence(())) + } + } + } + + match first_byte { + 0x00...0x7F => { + first_byte!(0b1111_1111); + } + 0xC2...0xDF => { + first_byte!(0b0001_1111); + continuation_byte!(); + } + 0xE0 => { + first_byte!(0b0000_1111); + continuation_byte!(0xA0...0xBF); // 0x80...0x9F here are overlong + continuation_byte!(); + } + 0xE1...0xEC | 0xEE...0xEF => { + first_byte!(0b0000_1111); + continuation_byte!(); + continuation_byte!(); + } + 0xED => { + first_byte!(0b0000_1111); + continuation_byte!(0x80...0x9F); // 0xA0..0xBF here are surrogates + continuation_byte!(); + } + 0xF0 => { + first_byte!(0b0000_0111); + continuation_byte!(0x90...0xBF); // 0x80..0x8F here are overlong + continuation_byte!(); + continuation_byte!(); + } + 0xF1...0xF3 => { + first_byte!(0b0000_0111); + continuation_byte!(); + continuation_byte!(); + continuation_byte!(); + } + 0xF4 => { + first_byte!(0b0000_0111); + continuation_byte!(0x80...0x8F); // 0x90..0xBF here are beyond char::MAX + continuation_byte!(); + continuation_byte!(); + } + _ => return Err(InvalidSequence(())) // Illegal first byte, overlong, or beyond MAX + } + unsafe { + Ok(from_u32_unchecked(code_point)) + } + }) + } + + #[inline] + fn size_hint(&self) -> (usize, Option<usize>) { + let (lower, upper) = self.0.size_hint(); + + // A code point is at most 4 bytes long. + let min_code_points = lower / 4; + + (min_code_points, upper) + } +} + +#[unstable(feature = "decode_utf8", issue = "33906")] +impl<I: FusedIterator<Item = u8>> FusedIterator for DecodeUtf8<I> {} + +/// An iterator that decodes UTF-16 encoded code points from an iterator of `u16`s. +#[stable(feature = "decode_utf16", since = "1.9.0")] +#[derive(Clone, Debug)] +pub struct DecodeUtf16<I> + where I: Iterator<Item = u16> +{ + iter: I, + buf: Option<u16>, +} + +/// An error that can be returned when decoding UTF-16 code points. +#[stable(feature = "decode_utf16", since = "1.9.0")] +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct DecodeUtf16Error { + code: u16, +} + +/// Create an iterator over the UTF-16 encoded code points in `iter`, +/// returning unpaired surrogates as `Err`s. +/// +/// # Examples +/// +/// Basic usage: +/// +/// ``` +/// use std::char::decode_utf16; +/// +/// fn main() { +/// // 𝄞mus<invalid>ic<invalid> +/// let v = [0xD834, 0xDD1E, 0x006d, 0x0075, +/// 0x0073, 0xDD1E, 0x0069, 0x0063, +/// 0xD834]; +/// +/// assert_eq!(decode_utf16(v.iter().cloned()) +/// .map(|r| r.map_err(|e| e.unpaired_surrogate())) +/// .collect::<Vec<_>>(), +/// vec![Ok('𝄞'), +/// Ok('m'), Ok('u'), Ok('s'), +/// Err(0xDD1E), +/// Ok('i'), Ok('c'), +/// Err(0xD834)]); +/// } +/// ``` +/// +/// A lossy decoder can be obtained by replacing `Err` results with the replacement character: +/// +/// ``` +/// use std::char::{decode_utf16, REPLACEMENT_CHARACTER}; +/// +/// fn main() { +/// // 𝄞mus<invalid>ic<invalid> +/// let v = [0xD834, 0xDD1E, 0x006d, 0x0075, +/// 0x0073, 0xDD1E, 0x0069, 0x0063, +/// 0xD834]; +/// +/// assert_eq!(decode_utf16(v.iter().cloned()) +/// .map(|r| r.unwrap_or(REPLACEMENT_CHARACTER)) +/// .collect::<String>(), +/// "𝄞mus�ic�"); +/// } +/// ``` +#[stable(feature = "decode_utf16", since = "1.9.0")] +#[inline] +pub fn decode_utf16<I: IntoIterator<Item = u16>>(iter: I) -> DecodeUtf16<I::IntoIter> { + DecodeUtf16 { + iter: iter.into_iter(), + buf: None, + } +} + +#[stable(feature = "decode_utf16", since = "1.9.0")] +impl<I: Iterator<Item = u16>> Iterator for DecodeUtf16<I> { + type Item = Result<char, DecodeUtf16Error>; + + fn next(&mut self) -> Option<Result<char, DecodeUtf16Error>> { + let u = match self.buf.take() { + Some(buf) => buf, + None => self.iter.next()? + }; + + if u < 0xD800 || 0xDFFF < u { + // not a surrogate + Some(Ok(unsafe { from_u32_unchecked(u as u32) })) + } else if u >= 0xDC00 { + // a trailing surrogate + Some(Err(DecodeUtf16Error { code: u })) + } else { + let u2 = match self.iter.next() { + Some(u2) => u2, + // eof + None => return Some(Err(DecodeUtf16Error { code: u })), + }; + if u2 < 0xDC00 || u2 > 0xDFFF { + // not a trailing surrogate so we're not a valid + // surrogate pair, so rewind to redecode u2 next time. + self.buf = Some(u2); + return Some(Err(DecodeUtf16Error { code: u })); + } + + // all ok, so lets decode it. + let c = (((u - 0xD800) as u32) << 10 | (u2 - 0xDC00) as u32) + 0x1_0000; + Some(Ok(unsafe { from_u32_unchecked(c) })) + } + } + + #[inline] + fn size_hint(&self) -> (usize, Option<usize>) { + let (low, high) = self.iter.size_hint(); + // we could be entirely valid surrogates (2 elements per + // char), or entirely non-surrogates (1 element per char) + (low / 2, high) + } +} + +impl DecodeUtf16Error { + /// Returns the unpaired surrogate which caused this error. + #[stable(feature = "decode_utf16", since = "1.9.0")] + pub fn unpaired_surrogate(&self) -> u16 { + self.code + } +} + +#[stable(feature = "decode_utf16", since = "1.9.0")] +impl fmt::Display for DecodeUtf16Error { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "unpaired surrogate found: {:x}", self.code) + } +} diff --git a/src/libstd_unicode/char.rs b/src/libcore/char/methods.rs index 33e47ade8cb..374adafef64 100644 --- a/src/libstd_unicode/char.rs +++ b/src/libcore/char/methods.rs @@ -8,169 +8,13 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. -//! A character type. -//! -//! The `char` type represents a single character. More specifically, since -//! 'character' isn't a well-defined concept in Unicode, `char` is a '[Unicode -//! scalar value]', which is similar to, but not the same as, a '[Unicode code -//! point]'. -//! -//! [Unicode scalar value]: http://www.unicode.org/glossary/#unicode_scalar_value -//! [Unicode code point]: http://www.unicode.org/glossary/#code_point -//! -//! This module exists for technical reasons, the primary documentation for -//! `char` is directly on [the `char` primitive type](../../std/primitive.char.html) -//! itself. -//! -//! This module is the home of the iterator implementations for the iterators -//! implemented on `char`, as well as some useful constants and conversion -//! functions that convert various types to `char`. +//! impl char {} -#![stable(feature = "rust1", since = "1.0.0")] - -use core::char::CharExt as C; -use core::iter::FusedIterator; -use core::fmt::{self, Write}; -use tables::{conversions, derived_property, general_category, property}; - -// stable re-exports -#[stable(feature = "rust1", since = "1.0.0")] -pub use core::char::{MAX, from_digit, from_u32, from_u32_unchecked}; -#[stable(feature = "rust1", since = "1.0.0")] -pub use core::char::{EscapeDebug, EscapeDefault, EscapeUnicode}; -#[stable(feature = "char_from_str", since = "1.20.0")] -pub use core::char::ParseCharError; - -// unstable re-exports -#[stable(feature = "try_from", since = "1.26.0")] -pub use core::char::CharTryFromError; -#[unstable(feature = "decode_utf8", issue = "33906")] -pub use core::char::{DecodeUtf8, decode_utf8}; -#[unstable(feature = "unicode", issue = "27783")] -pub use tables::{UNICODE_VERSION}; -#[unstable(feature = "unicode", issue = "27783")] -pub use version::UnicodeVersion; - -/// Returns an iterator that yields the lowercase equivalent of a `char`. -/// -/// This `struct` is created by the [`to_lowercase`] method on [`char`]. See -/// its documentation for more. -/// -/// [`to_lowercase`]: ../../std/primitive.char.html#method.to_lowercase -/// [`char`]: ../../std/primitive.char.html -#[stable(feature = "rust1", since = "1.0.0")] -#[derive(Debug, Clone)] -pub struct ToLowercase(CaseMappingIter); - -#[stable(feature = "rust1", since = "1.0.0")] -impl Iterator for ToLowercase { - type Item = char; - fn next(&mut self) -> Option<char> { - self.0.next() - } -} - -#[stable(feature = "fused", since = "1.26.0")] -impl FusedIterator for ToLowercase {} - -/// Returns an iterator that yields the uppercase equivalent of a `char`. -/// -/// This `struct` is created by the [`to_uppercase`] method on [`char`]. See -/// its documentation for more. -/// -/// [`to_uppercase`]: ../../std/primitive.char.html#method.to_uppercase -/// [`char`]: ../../std/primitive.char.html -#[stable(feature = "rust1", since = "1.0.0")] -#[derive(Debug, Clone)] -pub struct ToUppercase(CaseMappingIter); - -#[stable(feature = "rust1", since = "1.0.0")] -impl Iterator for ToUppercase { - type Item = char; - fn next(&mut self) -> Option<char> { - self.0.next() - } -} - -#[stable(feature = "fused", since = "1.26.0")] -impl FusedIterator for ToUppercase {} - -#[derive(Debug, Clone)] -enum CaseMappingIter { - Three(char, char, char), - Two(char, char), - One(char), - Zero, -} - -impl CaseMappingIter { - fn new(chars: [char; 3]) -> CaseMappingIter { - if chars[2] == '\0' { - if chars[1] == '\0' { - CaseMappingIter::One(chars[0]) // Including if chars[0] == '\0' - } else { - CaseMappingIter::Two(chars[0], chars[1]) - } - } else { - CaseMappingIter::Three(chars[0], chars[1], chars[2]) - } - } -} - -impl Iterator for CaseMappingIter { - type Item = char; - fn next(&mut self) -> Option<char> { - match *self { - CaseMappingIter::Three(a, b, c) => { - *self = CaseMappingIter::Two(b, c); - Some(a) - } - CaseMappingIter::Two(b, c) => { - *self = CaseMappingIter::One(c); - Some(b) - } - CaseMappingIter::One(c) => { - *self = CaseMappingIter::Zero; - Some(c) - } - CaseMappingIter::Zero => None, - } - } -} - -impl fmt::Display for CaseMappingIter { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match *self { - CaseMappingIter::Three(a, b, c) => { - f.write_char(a)?; - f.write_char(b)?; - f.write_char(c) - } - CaseMappingIter::Two(b, c) => { - f.write_char(b)?; - f.write_char(c) - } - CaseMappingIter::One(c) => { - f.write_char(c) - } - CaseMappingIter::Zero => Ok(()), - } - } -} - -#[stable(feature = "char_struct_display", since = "1.16.0")] -impl fmt::Display for ToLowercase { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - fmt::Display::fmt(&self.0, f) - } -} - -#[stable(feature = "char_struct_display", since = "1.16.0")] -impl fmt::Display for ToUppercase { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - fmt::Display::fmt(&self.0, f) - } -} +use slice; +use str::from_utf8_unchecked_mut; +use super::*; +use unicode::printable::is_printable; +use unicode::tables::{conversions, derived_property, general_category, property}; #[lang = "char"] impl char { @@ -223,7 +67,7 @@ impl char { #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn is_digit(self, radix: u32) -> bool { - C::is_digit(self, radix) + self.to_digit(radix).is_some() } /// Converts a `char` to a digit in the given radix. @@ -277,7 +121,17 @@ impl char { #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn to_digit(self, radix: u32) -> Option<u32> { - C::to_digit(self, radix) + if radix > 36 { + panic!("to_digit: radix is too high (maximum 36)"); + } + let val = match self { + '0' ... '9' => self as u32 - '0' as u32, + 'a' ... 'z' => self as u32 - 'a' as u32 + 10, + 'A' ... 'Z' => self as u32 - 'A' as u32 + 10, + _ => return None, + }; + if val < radix { Some(val) } + else { None } } /// Returns an iterator that yields the hexadecimal Unicode escape of a @@ -317,7 +171,20 @@ impl char { #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn escape_unicode(self) -> EscapeUnicode { - C::escape_unicode(self) + let c = self as u32; + + // or-ing 1 ensures that for c==0 the code computes that one + // digit should be printed and (which is the same) avoids the + // (31 - 32) underflow + let msb = 31 - (c | 1).leading_zeros(); + + // the index of the most significant hex digit + let ms_hex_digit = msb / 4; + EscapeUnicode { + c: self, + state: EscapeUnicodeState::Backslash, + hex_digit_idx: ms_hex_digit as usize, + } } /// Returns an iterator that yields the literal escape code of a character @@ -357,7 +224,15 @@ impl char { #[stable(feature = "char_escape_debug", since = "1.20.0")] #[inline] pub fn escape_debug(self) -> EscapeDebug { - C::escape_debug(self) + let init_state = match self { + '\t' => EscapeDefaultState::Backslash('t'), + '\r' => EscapeDefaultState::Backslash('r'), + '\n' => EscapeDefaultState::Backslash('n'), + '\\' | '\'' | '"' => EscapeDefaultState::Backslash(self), + c if is_printable(c) => EscapeDefaultState::Char(c), + c => EscapeDefaultState::Unicode(c.escape_unicode()), + }; + EscapeDebug(EscapeDefault { state: init_state }) } /// Returns an iterator that yields the literal escape code of a character @@ -412,7 +287,15 @@ impl char { #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn escape_default(self) -> EscapeDefault { - C::escape_default(self) + let init_state = match self { + '\t' => EscapeDefaultState::Backslash('t'), + '\r' => EscapeDefaultState::Backslash('r'), + '\n' => EscapeDefaultState::Backslash('n'), + '\\' | '\'' | '"' => EscapeDefaultState::Backslash(self), + '\x20' ... '\x7e' => EscapeDefaultState::Char(self), + _ => EscapeDefaultState::Unicode(self.escape_unicode()) + }; + EscapeDefault { state: init_state } } /// Returns the number of bytes this `char` would need if encoded in UTF-8. @@ -463,7 +346,16 @@ impl char { #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn len_utf8(self) -> usize { - C::len_utf8(self) + let code = self as u32; + if code < MAX_ONE_B { + 1 + } else if code < MAX_TWO_B { + 2 + } else if code < MAX_THREE_B { + 3 + } else { + 4 + } } /// Returns the number of 16-bit code units this `char` would need if @@ -488,7 +380,8 @@ impl char { #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn len_utf16(self) -> usize { - C::len_utf16(self) + let ch = self as u32; + if (ch & 0xFFFF) == ch { 1 } else { 2 } } /// Encodes this character as UTF-8 into the provided byte buffer, @@ -530,7 +423,35 @@ impl char { #[stable(feature = "unicode_encode_char", since = "1.15.0")] #[inline] pub fn encode_utf8(self, dst: &mut [u8]) -> &mut str { - C::encode_utf8(self, dst) + let code = self as u32; + unsafe { + let len = + if code < MAX_ONE_B && !dst.is_empty() { + *dst.get_unchecked_mut(0) = code as u8; + 1 + } else if code < MAX_TWO_B && dst.len() >= 2 { + *dst.get_unchecked_mut(0) = (code >> 6 & 0x1F) as u8 | TAG_TWO_B; + *dst.get_unchecked_mut(1) = (code & 0x3F) as u8 | TAG_CONT; + 2 + } else if code < MAX_THREE_B && dst.len() >= 3 { + *dst.get_unchecked_mut(0) = (code >> 12 & 0x0F) as u8 | TAG_THREE_B; + *dst.get_unchecked_mut(1) = (code >> 6 & 0x3F) as u8 | TAG_CONT; + *dst.get_unchecked_mut(2) = (code & 0x3F) as u8 | TAG_CONT; + 3 + } else if dst.len() >= 4 { + *dst.get_unchecked_mut(0) = (code >> 18 & 0x07) as u8 | TAG_FOUR_B; + *dst.get_unchecked_mut(1) = (code >> 12 & 0x3F) as u8 | TAG_CONT; + *dst.get_unchecked_mut(2) = (code >> 6 & 0x3F) as u8 | TAG_CONT; + *dst.get_unchecked_mut(3) = (code & 0x3F) as u8 | TAG_CONT; + 4 + } else { + panic!("encode_utf8: need {} bytes to encode U+{:X}, but the buffer has {}", + from_u32_unchecked(code).len_utf8(), + code, + dst.len()) + }; + from_utf8_unchecked_mut(dst.get_unchecked_mut(..len)) + } } /// Encodes this character as UTF-16 into the provided `u16` buffer, @@ -570,7 +491,25 @@ impl char { #[stable(feature = "unicode_encode_char", since = "1.15.0")] #[inline] pub fn encode_utf16(self, dst: &mut [u16]) -> &mut [u16] { - C::encode_utf16(self, dst) + let mut code = self as u32; + unsafe { + if (code & 0xFFFF) == code && !dst.is_empty() { + // The BMP falls through (assuming non-surrogate, as it should) + *dst.get_unchecked_mut(0) = code as u16; + slice::from_raw_parts_mut(dst.as_mut_ptr(), 1) + } else if dst.len() >= 2 { + // Supplementary planes break into surrogates. + code -= 0x1_0000; + *dst.get_unchecked_mut(0) = 0xD800 | ((code >> 10) as u16); + *dst.get_unchecked_mut(1) = 0xDC00 | ((code as u16) & 0x3FF); + slice::from_raw_parts_mut(dst.as_mut_ptr(), 2) + } else { + panic!("encode_utf16: need {} units to encode U+{:X}, but the buffer has {}", + from_u32_unchecked(code).len_utf16(), + code, + dst.len()) + } + } } /// Returns true if this `char` is an alphabetic code point, and false if not. @@ -1452,140 +1391,3 @@ impl char { self.is_ascii() && (*self as u8).is_ascii_control() } } - -/// An iterator that decodes UTF-16 encoded code points from an iterator of `u16`s. -#[stable(feature = "decode_utf16", since = "1.9.0")] -#[derive(Clone, Debug)] -pub struct DecodeUtf16<I> - where I: Iterator<Item = u16> -{ - iter: I, - buf: Option<u16>, -} - -/// An error that can be returned when decoding UTF-16 code points. -#[stable(feature = "decode_utf16", since = "1.9.0")] -#[derive(Debug, Clone, Eq, PartialEq)] -pub struct DecodeUtf16Error { - code: u16, -} - -/// Create an iterator over the UTF-16 encoded code points in `iter`, -/// returning unpaired surrogates as `Err`s. -/// -/// # Examples -/// -/// Basic usage: -/// -/// ``` -/// use std::char::decode_utf16; -/// -/// fn main() { -/// // 𝄞mus<invalid>ic<invalid> -/// let v = [0xD834, 0xDD1E, 0x006d, 0x0075, -/// 0x0073, 0xDD1E, 0x0069, 0x0063, -/// 0xD834]; -/// -/// assert_eq!(decode_utf16(v.iter().cloned()) -/// .map(|r| r.map_err(|e| e.unpaired_surrogate())) -/// .collect::<Vec<_>>(), -/// vec![Ok('𝄞'), -/// Ok('m'), Ok('u'), Ok('s'), -/// Err(0xDD1E), -/// Ok('i'), Ok('c'), -/// Err(0xD834)]); -/// } -/// ``` -/// -/// A lossy decoder can be obtained by replacing `Err` results with the replacement character: -/// -/// ``` -/// use std::char::{decode_utf16, REPLACEMENT_CHARACTER}; -/// -/// fn main() { -/// // 𝄞mus<invalid>ic<invalid> -/// let v = [0xD834, 0xDD1E, 0x006d, 0x0075, -/// 0x0073, 0xDD1E, 0x0069, 0x0063, -/// 0xD834]; -/// -/// assert_eq!(decode_utf16(v.iter().cloned()) -/// .map(|r| r.unwrap_or(REPLACEMENT_CHARACTER)) -/// .collect::<String>(), -/// "𝄞mus�ic�"); -/// } -/// ``` -#[stable(feature = "decode_utf16", since = "1.9.0")] -#[inline] -pub fn decode_utf16<I: IntoIterator<Item = u16>>(iter: I) -> DecodeUtf16<I::IntoIter> { - DecodeUtf16 { - iter: iter.into_iter(), - buf: None, - } -} - -#[stable(feature = "decode_utf16", since = "1.9.0")] -impl<I: Iterator<Item = u16>> Iterator for DecodeUtf16<I> { - type Item = Result<char, DecodeUtf16Error>; - - fn next(&mut self) -> Option<Result<char, DecodeUtf16Error>> { - let u = match self.buf.take() { - Some(buf) => buf, - None => self.iter.next()? - }; - - if u < 0xD800 || 0xDFFF < u { - // not a surrogate - Some(Ok(unsafe { from_u32_unchecked(u as u32) })) - } else if u >= 0xDC00 { - // a trailing surrogate - Some(Err(DecodeUtf16Error { code: u })) - } else { - let u2 = match self.iter.next() { - Some(u2) => u2, - // eof - None => return Some(Err(DecodeUtf16Error { code: u })), - }; - if u2 < 0xDC00 || u2 > 0xDFFF { - // not a trailing surrogate so we're not a valid - // surrogate pair, so rewind to redecode u2 next time. - self.buf = Some(u2); - return Some(Err(DecodeUtf16Error { code: u })); - } - - // all ok, so lets decode it. - let c = (((u - 0xD800) as u32) << 10 | (u2 - 0xDC00) as u32) + 0x1_0000; - Some(Ok(unsafe { from_u32_unchecked(c) })) - } - } - - #[inline] - fn size_hint(&self) -> (usize, Option<usize>) { - let (low, high) = self.iter.size_hint(); - // we could be entirely valid surrogates (2 elements per - // char), or entirely non-surrogates (1 element per char) - (low / 2, high) - } -} - -impl DecodeUtf16Error { - /// Returns the unpaired surrogate which caused this error. - #[stable(feature = "decode_utf16", since = "1.9.0")] - pub fn unpaired_surrogate(&self) -> u16 { - self.code - } -} - -#[stable(feature = "decode_utf16", since = "1.9.0")] -impl fmt::Display for DecodeUtf16Error { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "unpaired surrogate found: {:x}", self.code) - } -} - -/// `U+FFFD REPLACEMENT CHARACTER` (�) is used in Unicode to represent a -/// decoding error. -/// -/// It can occur, for example, when giving ill-formed UTF-8 bytes to -/// [`String::from_utf8_lossy`](../../std/string/struct.String.html#method.from_utf8_lossy). -#[stable(feature = "decode_utf16", since = "1.9.0")] -pub const REPLACEMENT_CHARACTER: char = '\u{FFFD}'; diff --git a/src/libcore/char/mod.rs b/src/libcore/char/mod.rs new file mode 100644 index 00000000000..9edc0c88756 --- /dev/null +++ b/src/libcore/char/mod.rs @@ -0,0 +1,506 @@ +// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license +// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +//! A character type. +//! +//! The `char` type represents a single character. More specifically, since +//! 'character' isn't a well-defined concept in Unicode, `char` is a '[Unicode +//! scalar value]', which is similar to, but not the same as, a '[Unicode code +//! point]'. +//! +//! [Unicode scalar value]: http://www.unicode.org/glossary/#unicode_scalar_value +//! [Unicode code point]: http://www.unicode.org/glossary/#code_point +//! +//! This module exists for technical reasons, the primary documentation for +//! `char` is directly on [the `char` primitive type](../../std/primitive.char.html) +//! itself. +//! +//! This module is the home of the iterator implementations for the iterators +//! implemented on `char`, as well as some useful constants and conversion +//! functions that convert various types to `char`. + +#![allow(non_snake_case)] +#![stable(feature = "core_char", since = "1.2.0")] + +mod convert; +mod decode; +mod methods; + +// stable re-exports +#[stable(feature = "rust1", since = "1.0.0")] +pub use self::convert::{from_u32, from_digit}; +#[stable(feature = "char_from_unchecked", since = "1.5.0")] +pub use self::convert::from_u32_unchecked; +#[stable(feature = "char_from_str", since = "1.20.0")] +pub use self::convert::ParseCharError; +#[stable(feature = "try_from", since = "1.26.0")] +pub use self::convert::CharTryFromError; +#[stable(feature = "decode_utf16", since = "1.9.0")] +pub use self::decode::{decode_utf16, DecodeUtf16, DecodeUtf16Error}; + +// unstable re-exports +#[unstable(feature = "unicode_version", issue = "49726")] +pub use unicode::tables::UNICODE_VERSION; +#[unstable(feature = "unicode_version", issue = "49726")] +pub use unicode::version::UnicodeVersion; +#[unstable(feature = "decode_utf8", issue = "33906")] +pub use self::decode::{decode_utf8, DecodeUtf8, InvalidSequence}; + +use fmt::{self, Write}; +use iter::FusedIterator; + +// UTF-8 ranges and tags for encoding characters +const TAG_CONT: u8 = 0b1000_0000; +const TAG_TWO_B: u8 = 0b1100_0000; +const TAG_THREE_B: u8 = 0b1110_0000; +const TAG_FOUR_B: u8 = 0b1111_0000; +const MAX_ONE_B: u32 = 0x80; +const MAX_TWO_B: u32 = 0x800; +const MAX_THREE_B: u32 = 0x10000; + +/* + Lu Uppercase_Letter an uppercase letter + Ll Lowercase_Letter a lowercase letter + Lt Titlecase_Letter a digraphic character, with first part uppercase + Lm Modifier_Letter a modifier letter + Lo Other_Letter other letters, including syllables and ideographs + Mn Nonspacing_Mark a nonspacing combining mark (zero advance width) + Mc Spacing_Mark a spacing combining mark (positive advance width) + Me Enclosing_Mark an enclosing combining mark + Nd Decimal_Number a decimal digit + Nl Letter_Number a letterlike numeric character + No Other_Number a numeric character of other type + Pc Connector_Punctuation a connecting punctuation mark, like a tie + Pd Dash_Punctuation a dash or hyphen punctuation mark + Ps Open_Punctuation an opening punctuation mark (of a pair) + Pe Close_Punctuation a closing punctuation mark (of a pair) + Pi Initial_Punctuation an initial quotation mark + Pf Final_Punctuation a final quotation mark + Po Other_Punctuation a punctuation mark of other type + Sm Math_Symbol a symbol of primarily mathematical use + Sc Currency_Symbol a currency sign + Sk Modifier_Symbol a non-letterlike modifier symbol + So Other_Symbol a symbol of other type + Zs Space_Separator a space character (of various non-zero widths) + Zl Line_Separator U+2028 LINE SEPARATOR only + Zp Paragraph_Separator U+2029 PARAGRAPH SEPARATOR only + Cc Control a C0 or C1 control code + Cf Format a format control character + Cs Surrogate a surrogate code point + Co Private_Use a private-use character + Cn Unassigned a reserved unassigned code point or a noncharacter +*/ + +/// The highest valid code point a `char` can have. +/// +/// A [`char`] is a [Unicode Scalar Value], which means that it is a [Code +/// Point], but only ones within a certain range. `MAX` is the highest valid +/// code point that's a valid [Unicode Scalar Value]. +/// +/// [`char`]: ../../std/primitive.char.html +/// [Unicode Scalar Value]: http://www.unicode.org/glossary/#unicode_scalar_value +/// [Code Point]: http://www.unicode.org/glossary/#code_point +#[stable(feature = "rust1", since = "1.0.0")] +pub const MAX: char = '\u{10ffff}'; + +/// `U+FFFD REPLACEMENT CHARACTER` (�) is used in Unicode to represent a +/// decoding error. +/// +/// It can occur, for example, when giving ill-formed UTF-8 bytes to +/// [`String::from_utf8_lossy`](../../std/string/struct.String.html#method.from_utf8_lossy). +#[stable(feature = "decode_utf16", since = "1.9.0")] +pub const REPLACEMENT_CHARACTER: char = '\u{FFFD}'; + +/// Returns an iterator that yields the hexadecimal Unicode escape of a +/// character, as `char`s. +/// +/// This `struct` is created by the [`escape_unicode`] method on [`char`]. See +/// its documentation for more. +/// +/// [`escape_unicode`]: ../../std/primitive.char.html#method.escape_unicode +/// [`char`]: ../../std/primitive.char.html +#[derive(Clone, Debug)] +#[stable(feature = "rust1", since = "1.0.0")] +pub struct EscapeUnicode { + c: char, + state: EscapeUnicodeState, + + // The index of the next hex digit to be printed (0 if none), + // i.e. the number of remaining hex digits to be printed; + // increasing from the least significant digit: 0x543210 + hex_digit_idx: usize, +} + +// The enum values are ordered so that their representation is the +// same as the remaining length (besides the hexadecimal digits). This +// likely makes `len()` a single load from memory) and inline-worth. +#[derive(Clone, Debug)] +enum EscapeUnicodeState { + Done, + RightBrace, + Value, + LeftBrace, + Type, + Backslash, +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Iterator for EscapeUnicode { + type Item = char; + + fn next(&mut self) -> Option<char> { + match self.state { + EscapeUnicodeState::Backslash => { + self.state = EscapeUnicodeState::Type; + Some('\\') + } + EscapeUnicodeState::Type => { + self.state = EscapeUnicodeState::LeftBrace; + Some('u') + } + EscapeUnicodeState::LeftBrace => { + self.state = EscapeUnicodeState::Value; + Some('{') + } + EscapeUnicodeState::Value => { + let hex_digit = ((self.c as u32) >> (self.hex_digit_idx * 4)) & 0xf; + let c = from_digit(hex_digit, 16).unwrap(); + if self.hex_digit_idx == 0 { + self.state = EscapeUnicodeState::RightBrace; + } else { + self.hex_digit_idx -= 1; + } + Some(c) + } + EscapeUnicodeState::RightBrace => { + self.state = EscapeUnicodeState::Done; + Some('}') + } + EscapeUnicodeState::Done => None, + } + } + + #[inline] + fn size_hint(&self) -> (usize, Option<usize>) { + let n = self.len(); + (n, Some(n)) + } + + #[inline] + fn count(self) -> usize { + self.len() + } + + fn last(self) -> Option<char> { + match self.state { + EscapeUnicodeState::Done => None, + + EscapeUnicodeState::RightBrace | + EscapeUnicodeState::Value | + EscapeUnicodeState::LeftBrace | + EscapeUnicodeState::Type | + EscapeUnicodeState::Backslash => Some('}'), + } + } +} + +#[stable(feature = "exact_size_escape", since = "1.11.0")] +impl ExactSizeIterator for EscapeUnicode { + #[inline] + fn len(&self) -> usize { + // The match is a single memory access with no branching + self.hex_digit_idx + match self.state { + EscapeUnicodeState::Done => 0, + EscapeUnicodeState::RightBrace => 1, + EscapeUnicodeState::Value => 2, + EscapeUnicodeState::LeftBrace => 3, + EscapeUnicodeState::Type => 4, + EscapeUnicodeState::Backslash => 5, + } + } +} + +#[stable(feature = "fused", since = "1.26.0")] +impl FusedIterator for EscapeUnicode {} + +#[stable(feature = "char_struct_display", since = "1.16.0")] +impl fmt::Display for EscapeUnicode { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + for c in self.clone() { + f.write_char(c)?; + } + Ok(()) + } +} + +/// An iterator that yields the literal escape code of a `char`. +/// +/// This `struct` is created by the [`escape_default`] method on [`char`]. See +/// its documentation for more. +/// +/// [`escape_default`]: ../../std/primitive.char.html#method.escape_default +/// [`char`]: ../../std/primitive.char.html +#[derive(Clone, Debug)] +#[stable(feature = "rust1", since = "1.0.0")] +pub struct EscapeDefault { + state: EscapeDefaultState +} + +#[derive(Clone, Debug)] +enum EscapeDefaultState { + Done, + Char(char), + Backslash(char), + Unicode(EscapeUnicode), +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Iterator for EscapeDefault { + type Item = char; + + fn next(&mut self) -> Option<char> { + match self.state { + EscapeDefaultState::Backslash(c) => { + self.state = EscapeDefaultState::Char(c); + Some('\\') + } + EscapeDefaultState::Char(c) => { + self.state = EscapeDefaultState::Done; + Some(c) + } + EscapeDefaultState::Done => None, + EscapeDefaultState::Unicode(ref mut iter) => iter.next(), + } + } + + #[inline] + fn size_hint(&self) -> (usize, Option<usize>) { + let n = self.len(); + (n, Some(n)) + } + + #[inline] + fn count(self) -> usize { + self.len() + } + + fn nth(&mut self, n: usize) -> Option<char> { + match self.state { + EscapeDefaultState::Backslash(c) if n == 0 => { + self.state = EscapeDefaultState::Char(c); + Some('\\') + }, + EscapeDefaultState::Backslash(c) if n == 1 => { + self.state = EscapeDefaultState::Done; + Some(c) + }, + EscapeDefaultState::Backslash(_) => { + self.state = EscapeDefaultState::Done; + None + }, + EscapeDefaultState::Char(c) => { + self.state = EscapeDefaultState::Done; + + if n == 0 { + Some(c) + } else { + None + } + }, + EscapeDefaultState::Done => return None, + EscapeDefaultState::Unicode(ref mut i) => return i.nth(n), + } + } + + fn last(self) -> Option<char> { + match self.state { + EscapeDefaultState::Unicode(iter) => iter.last(), + EscapeDefaultState::Done => None, + EscapeDefaultState::Backslash(c) | EscapeDefaultState::Char(c) => Some(c), + } + } +} + +#[stable(feature = "exact_size_escape", since = "1.11.0")] +impl ExactSizeIterator for EscapeDefault { + fn len(&self) -> usize { + match self.state { + EscapeDefaultState::Done => 0, + EscapeDefaultState::Char(_) => 1, + EscapeDefaultState::Backslash(_) => 2, + EscapeDefaultState::Unicode(ref iter) => iter.len(), + } + } +} + +#[stable(feature = "fused", since = "1.26.0")] +impl FusedIterator for EscapeDefault {} + +#[stable(feature = "char_struct_display", since = "1.16.0")] +impl fmt::Display for EscapeDefault { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + for c in self.clone() { + f.write_char(c)?; + } + Ok(()) + } +} + +/// An iterator that yields the literal escape code of a `char`. +/// +/// This `struct` is created by the [`escape_debug`] method on [`char`]. See its +/// documentation for more. +/// +/// [`escape_debug`]: ../../std/primitive.char.html#method.escape_debug +/// [`char`]: ../../std/primitive.char.html +#[stable(feature = "char_escape_debug", since = "1.20.0")] +#[derive(Clone, Debug)] +pub struct EscapeDebug(EscapeDefault); + +#[stable(feature = "char_escape_debug", since = "1.20.0")] +impl Iterator for EscapeDebug { + type Item = char; + fn next(&mut self) -> Option<char> { self.0.next() } + fn size_hint(&self) -> (usize, Option<usize>) { self.0.size_hint() } +} + +#[stable(feature = "char_escape_debug", since = "1.20.0")] +impl ExactSizeIterator for EscapeDebug { } + +#[stable(feature = "fused", since = "1.26.0")] +impl FusedIterator for EscapeDebug {} + +#[stable(feature = "char_escape_debug", since = "1.20.0")] +impl fmt::Display for EscapeDebug { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Display::fmt(&self.0, f) + } +} + +/// Returns an iterator that yields the lowercase equivalent of a `char`. +/// +/// This `struct` is created by the [`to_lowercase`] method on [`char`]. See +/// its documentation for more. +/// +/// [`to_lowercase`]: ../../std/primitive.char.html#method.to_lowercase +/// [`char`]: ../../std/primitive.char.html +#[stable(feature = "rust1", since = "1.0.0")] +#[derive(Debug, Clone)] +pub struct ToLowercase(CaseMappingIter); + +#[stable(feature = "rust1", since = "1.0.0")] +impl Iterator for ToLowercase { + type Item = char; + fn next(&mut self) -> Option<char> { + self.0.next() + } +} + +#[stable(feature = "fused", since = "1.26.0")] +impl FusedIterator for ToLowercase {} + +/// Returns an iterator that yields the uppercase equivalent of a `char`. +/// +/// This `struct` is created by the [`to_uppercase`] method on [`char`]. See +/// its documentation for more. +/// +/// [`to_uppercase`]: ../../std/primitive.char.html#method.to_uppercase +/// [`char`]: ../../std/primitive.char.html +#[stable(feature = "rust1", since = "1.0.0")] +#[derive(Debug, Clone)] +pub struct ToUppercase(CaseMappingIter); + +#[stable(feature = "rust1", since = "1.0.0")] +impl Iterator for ToUppercase { + type Item = char; + fn next(&mut self) -> Option<char> { + self.0.next() + } +} + +#[stable(feature = "fused", since = "1.26.0")] +impl FusedIterator for ToUppercase {} + +#[derive(Debug, Clone)] +enum CaseMappingIter { + Three(char, char, char), + Two(char, char), + One(char), + Zero, +} + +impl CaseMappingIter { + fn new(chars: [char; 3]) -> CaseMappingIter { + if chars[2] == '\0' { + if chars[1] == '\0' { + CaseMappingIter::One(chars[0]) // Including if chars[0] == '\0' + } else { + CaseMappingIter::Two(chars[0], chars[1]) + } + } else { + CaseMappingIter::Three(chars[0], chars[1], chars[2]) + } + } +} + +impl Iterator for CaseMappingIter { + type Item = char; + fn next(&mut self) -> Option<char> { + match *self { + CaseMappingIter::Three(a, b, c) => { + *self = CaseMappingIter::Two(b, c); + Some(a) + } + CaseMappingIter::Two(b, c) => { + *self = CaseMappingIter::One(c); + Some(b) + } + CaseMappingIter::One(c) => { + *self = CaseMappingIter::Zero; + Some(c) + } + CaseMappingIter::Zero => None, + } + } +} + +impl fmt::Display for CaseMappingIter { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match *self { + CaseMappingIter::Three(a, b, c) => { + f.write_char(a)?; + f.write_char(b)?; + f.write_char(c) + } + CaseMappingIter::Two(b, c) => { + f.write_char(b)?; + f.write_char(c) + } + CaseMappingIter::One(c) => { + f.write_char(c) + } + CaseMappingIter::Zero => Ok(()), + } + } +} + +#[stable(feature = "char_struct_display", since = "1.16.0")] +impl fmt::Display for ToLowercase { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Display::fmt(&self.0, f) + } +} + +#[stable(feature = "char_struct_display", since = "1.16.0")] +impl fmt::Display for ToUppercase { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Display::fmt(&self.0, f) + } +} diff --git a/src/libcore/lib.rs b/src/libcore/lib.rs index e194b173aa7..9ff8465bc0f 100644 --- a/src/libcore/lib.rs +++ b/src/libcore/lib.rs @@ -180,12 +180,13 @@ pub mod hash; pub mod fmt; pub mod time; +pub mod unicode; + /* Heap memory allocator trait */ #[allow(missing_docs)] pub mod heap; // note: does not need to be public -mod char_private; mod iter_private; mod tuple; mod unit; diff --git a/src/libcore/prelude/v1.rs b/src/libcore/prelude/v1.rs index d43496c387c..cc3ad71117a 100644 --- a/src/libcore/prelude/v1.rs +++ b/src/libcore/prelude/v1.rs @@ -62,6 +62,3 @@ pub use slice::SliceExt; #[stable(feature = "core_prelude", since = "1.4.0")] #[doc(no_inline)] pub use str::StrExt; -#[stable(feature = "core_prelude", since = "1.4.0")] -#[doc(no_inline)] -pub use char::CharExt; diff --git a/src/libstd_unicode/lossy.rs b/src/libcore/str/lossy.rs index cc8e93308a5..30b7267da7c 100644 --- a/src/libstd_unicode/lossy.rs +++ b/src/libcore/str/lossy.rs @@ -8,12 +8,11 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. -use core::str as core_str; -use core::fmt; -use core::fmt::Write; use char; -use core::mem; - +use str as core_str; +use fmt; +use fmt::Write; +use mem; /// Lossy UTF-8 string. #[unstable(feature = "str_internals", issue = "0")] diff --git a/src/libcore/str/mod.rs b/src/libcore/str/mod.rs index 1185b7acaae..f1fe23092de 100644 --- a/src/libcore/str/mod.rs +++ b/src/libcore/str/mod.rs @@ -19,13 +19,17 @@ use self::pattern::{Searcher, ReverseSearcher, DoubleEndedSearcher}; use char; use fmt; -use iter::{Map, Cloned, FusedIterator, TrustedLen}; +use iter::{Map, Cloned, FusedIterator, TrustedLen, Filter}; use iter_private::TrustedRandomAccess; use slice::{self, SliceIndex}; use mem; pub mod pattern; +#[unstable(feature = "str_internals", issue = "0")] +#[allow(missing_docs)] +pub mod lossy; + /// A trait to abstract the idea of creating a new instance of a type from a /// string. /// @@ -2212,6 +2216,18 @@ pub trait StrExt { fn is_empty(&self) -> bool; #[stable(feature = "core", since = "1.6.0")] fn parse<T: FromStr>(&self) -> Result<T, T::Err>; + #[stable(feature = "split_whitespace", since = "1.1.0")] + fn split_whitespace<'a>(&'a self) -> SplitWhitespace<'a>; + #[stable(feature = "unicode_methods_on_intrinsics", since = "1.27.0")] + fn is_whitespace(&self) -> bool; + #[stable(feature = "unicode_methods_on_intrinsics", since = "1.27.0")] + fn is_alphanumeric(&self) -> bool; + #[stable(feature = "rust1", since = "1.0.0")] + fn trim(&self) -> &str; + #[stable(feature = "rust1", since = "1.0.0")] + fn trim_left(&self) -> &str; + #[stable(feature = "rust1", since = "1.0.0")] + fn trim_right(&self) -> &str; } // truncate `&str` to length at most equal to `max` @@ -2532,6 +2548,36 @@ impl StrExt for str { #[inline] fn parse<T: FromStr>(&self) -> Result<T, T::Err> { FromStr::from_str(self) } + + #[inline] + fn split_whitespace(&self) -> SplitWhitespace { + SplitWhitespace { inner: self.split(IsWhitespace).filter(IsNotEmpty) } + } + + #[inline] + fn is_whitespace(&self) -> bool { + self.chars().all(|c| c.is_whitespace()) + } + + #[inline] + fn is_alphanumeric(&self) -> bool { + self.chars().all(|c| c.is_alphanumeric()) + } + + #[inline] + fn trim(&self) -> &str { + self.trim_matches(|c: char| c.is_whitespace()) + } + + #[inline] + fn trim_left(&self) -> &str { + self.trim_left_matches(|c: char| c.is_whitespace()) + } + + #[inline] + fn trim_right(&self) -> &str { + self.trim_right_matches(|c: char| c.is_whitespace()) + } } #[stable(feature = "rust1", since = "1.0.0")] @@ -2547,3 +2593,75 @@ impl<'a> Default for &'a str { /// Creates an empty str fn default() -> &'a str { "" } } + +/// An iterator over the non-whitespace substrings of a string, +/// separated by any amount of whitespace. +/// +/// This struct is created by the [`split_whitespace`] method on [`str`]. +/// See its documentation for more. +/// +/// [`split_whitespace`]: ../../std/primitive.str.html#method.split_whitespace +/// [`str`]: ../../std/primitive.str.html +#[stable(feature = "split_whitespace", since = "1.1.0")] +#[derive(Clone, Debug)] +pub struct SplitWhitespace<'a> { + inner: Filter<Split<'a, IsWhitespace>, IsNotEmpty>, +} + +#[derive(Clone)] +struct IsWhitespace; + +impl FnOnce<(char, )> for IsWhitespace { + type Output = bool; + + #[inline] + extern "rust-call" fn call_once(mut self, arg: (char, )) -> bool { + self.call_mut(arg) + } +} + +impl FnMut<(char, )> for IsWhitespace { + #[inline] + extern "rust-call" fn call_mut(&mut self, arg: (char, )) -> bool { + arg.0.is_whitespace() + } +} + +#[derive(Clone)] +struct IsNotEmpty; + +impl<'a, 'b> FnOnce<(&'a &'b str, )> for IsNotEmpty { + type Output = bool; + + #[inline] + extern "rust-call" fn call_once(mut self, arg: (&&str, )) -> bool { + self.call_mut(arg) + } +} + +impl<'a, 'b> FnMut<(&'a &'b str, )> for IsNotEmpty { + #[inline] + extern "rust-call" fn call_mut(&mut self, arg: (&&str, )) -> bool { + !arg.0.is_empty() + } +} + + +#[stable(feature = "split_whitespace", since = "1.1.0")] +impl<'a> Iterator for SplitWhitespace<'a> { + type Item = &'a str; + + fn next(&mut self) -> Option<&'a str> { + self.inner.next() + } +} + +#[stable(feature = "split_whitespace", since = "1.1.0")] +impl<'a> DoubleEndedIterator for SplitWhitespace<'a> { + fn next_back(&mut self) -> Option<&'a str> { + self.inner.next_back() + } +} + +#[stable(feature = "fused", since = "1.26.0")] +impl<'a> FusedIterator for SplitWhitespace<'a> {} diff --git a/src/libcore/tests/lib.rs b/src/libcore/tests/lib.rs index c3162899bbd..149269263dc 100644 --- a/src/libcore/tests/lib.rs +++ b/src/libcore/tests/lib.rs @@ -33,6 +33,7 @@ #![feature(sort_internals)] #![feature(specialization)] #![feature(step_trait)] +#![feature(str_internals)] #![feature(test)] #![feature(trusted_len)] #![feature(try_trait)] @@ -68,4 +69,5 @@ mod ptr; mod result; mod slice; mod str; +mod str_lossy; mod tuple; diff --git a/src/libstd_unicode/tests/lossy.rs b/src/libcore/tests/str_lossy.rs index e05d0668556..69e28256da9 100644 --- a/src/libstd_unicode/tests/lossy.rs +++ b/src/libcore/tests/str_lossy.rs @@ -8,7 +8,7 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. -use std_unicode::lossy::*; +use core::str::lossy::*; #[test] fn chunks() { diff --git a/src/libstd_unicode/bool_trie.rs b/src/libcore/unicode/bool_trie.rs index 3e45b08f399..3e45b08f399 100644 --- a/src/libstd_unicode/bool_trie.rs +++ b/src/libcore/unicode/bool_trie.rs diff --git a/src/libcore/unicode/mod.rs b/src/libcore/unicode/mod.rs new file mode 100644 index 00000000000..b6b033adc04 --- /dev/null +++ b/src/libcore/unicode/mod.rs @@ -0,0 +1,27 @@ +// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license +// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +#![unstable(feature = "unicode_internals", issue = "0")] +#![allow(missing_docs)] + +mod bool_trie; +pub(crate) mod printable; +pub(crate) mod tables; +pub(crate) mod version; + +// For use in liballoc, not re-exported in libstd. +pub mod derived_property { + pub use unicode::tables::derived_property::{Case_Ignorable, Cased}; +} + +// For use in libsyntax +pub mod property { + pub use unicode::tables::property::Pattern_White_Space; +} diff --git a/src/etc/char_private.py b/src/libcore/unicode/printable.py index cfe5b01e934..9410dafbbc3 100644 --- a/src/etc/char_private.py +++ b/src/libcore/unicode/printable.py @@ -187,7 +187,7 @@ def main(): // option. This file may not be copied, modified, or distributed // except according to those terms. -// NOTE: The following code was generated by "src/etc/char_private.py", +// NOTE: The following code was generated by "src/libcore/unicode/printable.py", // do not edit directly! fn check(x: u16, singletonuppers: &[(u8, u8)], singletonlowers: &[u8], diff --git a/src/libcore/char_private.rs b/src/libcore/unicode/printable.rs index e6803745ab5..4426c32eebc 100644 --- a/src/libcore/char_private.rs +++ b/src/libcore/unicode/printable.rs @@ -8,7 +8,7 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. -// NOTE: The following code was generated by "src/etc/char_private.py", +// NOTE: The following code was generated by "src/libcore/unicode/printable.py", // do not edit directly! fn check(x: u16, singletonuppers: &[(u8, u8)], singletonlowers: &[u8], diff --git a/src/libstd_unicode/tables.rs b/src/libcore/unicode/tables.rs index b53953b62a7..3fbbc011bc4 100644 --- a/src/libstd_unicode/tables.rs +++ b/src/libcore/unicode/tables.rs @@ -12,11 +12,12 @@ #![allow(missing_docs, non_upper_case_globals, non_snake_case)] -use version::UnicodeVersion; -use bool_trie::{BoolTrie, SmallBoolTrie}; +use unicode::version::UnicodeVersion; +use unicode::bool_trie::{BoolTrie, SmallBoolTrie}; /// The version of [Unicode](http://www.unicode.org/) that the Unicode parts of -/// `CharExt` and `UnicodeStrPrelude` traits are based on. +/// `char` and `str` methods are based on. +#[unstable(feature = "unicode_version", issue = "49726")] pub const UNICODE_VERSION: UnicodeVersion = UnicodeVersion { major: 10, minor: 0, @@ -1138,9 +1139,6 @@ pub mod property { } pub mod conversions { - use core::option::Option; - use core::option::Option::{Some, None}; - pub fn to_lower(c: char) -> [char; 3] { match bsearch_case_table(c, to_lowercase_table) { None => [c, '\0', '\0'], diff --git a/src/libstd_unicode/unicode.py b/src/libcore/unicode/unicode.py index a8629493086..75ec01944bf 100755 --- a/src/libstd_unicode/unicode.py +++ b/src/libcore/unicode/unicode.py @@ -39,8 +39,8 @@ preamble = '''// Copyright 2012-2016 The Rust Project Developers. See the COPYRI #![allow(missing_docs, non_upper_case_globals, non_snake_case)] -use version::UnicodeVersion; -use bool_trie::{BoolTrie, SmallBoolTrie}; +use unicode::version::UnicodeVersion; +use unicode::bool_trie::{BoolTrie, SmallBoolTrie}; ''' # Mapping taken from Table 12 from: @@ -408,9 +408,6 @@ def emit_property_module(f, mod, tbl, emit): def emit_conversions_module(f, to_upper, to_lower, to_title): f.write("pub mod conversions {") f.write(""" - use core::option::Option; - use core::option::Option::{Some, None}; - pub fn to_lower(c: char) -> [char; 3] { match bsearch_case_table(c, to_lowercase_table) { None => [c, '\\0', '\\0'], @@ -473,7 +470,8 @@ if __name__ == "__main__": unicode_version = re.search(pattern, readme.read()).groups() rf.write(""" /// The version of [Unicode](http://www.unicode.org/) that the Unicode parts of -/// `CharExt` and `UnicodeStrPrelude` traits are based on. +/// `char` and `str` methods are based on. +#[unstable(feature = "unicode_version", issue = "49726")] pub const UNICODE_VERSION: UnicodeVersion = UnicodeVersion { major: %s, minor: %s, diff --git a/src/libstd_unicode/version.rs b/src/libcore/unicode/version.rs index d82a749d917..59ebf5f5012 100644 --- a/src/libstd_unicode/version.rs +++ b/src/libcore/unicode/version.rs @@ -12,6 +12,7 @@ /// /// See also: <http://www.unicode.org/versions/> #[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +#[unstable(feature = "unicode_version", issue = "49726")] pub struct UnicodeVersion { /// Major version. pub major: u32, diff --git a/src/librustdoc/lib.rs b/src/librustdoc/lib.rs index b87777ac4b5..8463475afab 100644 --- a/src/librustdoc/lib.rs +++ b/src/librustdoc/lib.rs @@ -21,7 +21,6 @@ #![feature(set_stdio)] #![feature(slice_sort_by_cached_key)] #![feature(test)] -#![feature(unicode)] #![feature(vec_remove_item)] #![feature(entry_and_modify)] @@ -42,7 +41,6 @@ extern crate serialize; #[macro_use] extern crate syntax; extern crate syntax_pos; extern crate test as testing; -extern crate std_unicode; #[macro_use] extern crate log; extern crate rustc_errors as errors; extern crate pulldown_cmark; diff --git a/src/libstd/lib.rs b/src/libstd/lib.rs index 672723341eb..c82d600e4a1 100644 --- a/src/libstd/lib.rs +++ b/src/libstd/lib.rs @@ -307,7 +307,6 @@ #![feature(toowned_clone_into)] #![feature(try_reserve)] #![feature(unboxed_closures)] -#![feature(unicode)] #![feature(untagged_unions)] #![feature(unwind_attributes)] #![feature(vec_push_all)] @@ -354,7 +353,6 @@ extern crate core as __core; #[macro_reexport(vec, format)] extern crate alloc; extern crate alloc_system; -extern crate std_unicode; #[doc(masked)] extern crate libc; @@ -455,7 +453,7 @@ pub use alloc::string; #[stable(feature = "rust1", since = "1.0.0")] pub use alloc::vec; #[stable(feature = "rust1", since = "1.0.0")] -pub use std_unicode::char; +pub use core::char; #[stable(feature = "i128", since = "1.26.0")] pub use core::u128; diff --git a/src/libstd/sys/redox/os_str.rs b/src/libstd/sys/redox/os_str.rs index da27787babb..eb3a1ead58c 100644 --- a/src/libstd/sys/redox/os_str.rs +++ b/src/libstd/sys/redox/os_str.rs @@ -19,7 +19,7 @@ use rc::Rc; use sync::Arc; use sys_common::{AsInner, IntoInner}; use sys_common::bytestring::debug_fmt_bytestring; -use std_unicode::lossy::Utf8Lossy; +use core::str::lossy::Utf8Lossy; #[derive(Clone, Hash)] pub struct Buf { diff --git a/src/libstd/sys/unix/os_str.rs b/src/libstd/sys/unix/os_str.rs index e43bc6da5f1..01c0fb830aa 100644 --- a/src/libstd/sys/unix/os_str.rs +++ b/src/libstd/sys/unix/os_str.rs @@ -19,7 +19,7 @@ use rc::Rc; use sync::Arc; use sys_common::{AsInner, IntoInner}; use sys_common::bytestring::debug_fmt_bytestring; -use std_unicode::lossy::Utf8Lossy; +use core::str::lossy::Utf8Lossy; #[derive(Clone, Hash)] pub struct Buf { diff --git a/src/libstd/sys/wasm/os_str.rs b/src/libstd/sys/wasm/os_str.rs index 84f560af69b..e0da5bdf36c 100644 --- a/src/libstd/sys/wasm/os_str.rs +++ b/src/libstd/sys/wasm/os_str.rs @@ -19,7 +19,7 @@ use rc::Rc; use sync::Arc; use sys_common::{AsInner, IntoInner}; use sys_common::bytestring::debug_fmt_bytestring; -use std_unicode::lossy::Utf8Lossy; +use core::str::lossy::Utf8Lossy; #[derive(Clone, Hash)] pub struct Buf { diff --git a/src/libstd/sys_common/bytestring.rs b/src/libstd/sys_common/bytestring.rs index eb9cad09915..971b83938c1 100644 --- a/src/libstd/sys_common/bytestring.rs +++ b/src/libstd/sys_common/bytestring.rs @@ -11,7 +11,7 @@ #![allow(dead_code)] use fmt::{Formatter, Result, Write}; -use std_unicode::lossy::{Utf8Lossy, Utf8LossyChunk}; +use core::str::lossy::{Utf8Lossy, Utf8LossyChunk}; pub fn debug_fmt_bytestring(slice: &[u8], f: &mut Formatter) -> Result { // Writes out a valid unicode string with the correct escape sequences diff --git a/src/libstd_unicode/Cargo.toml b/src/libstd_unicode/Cargo.toml index 283070a0e2c..b1c55c2e4b6 100644 --- a/src/libstd_unicode/Cargo.toml +++ b/src/libstd_unicode/Cargo.toml @@ -9,10 +9,6 @@ path = "lib.rs" test = false bench = false -[[test]] -name = "std_unicode_tests" -path = "tests/lib.rs" - [dependencies] core = { path = "../libcore" } compiler_builtins = { path = "../rustc/compiler_builtins_shim" } diff --git a/src/libstd_unicode/lib.rs b/src/libstd_unicode/lib.rs index cf8c101a2f9..c0d47f1fcb4 100644 --- a/src/libstd_unicode/lib.rs +++ b/src/libstd_unicode/lib.rs @@ -27,38 +27,10 @@ html_playground_url = "https://play.rust-lang.org/", issue_tracker_base_url = "https://github.com/rust-lang/rust/issues/", test(no_crate_inject, attr(allow(unused_variables), deny(warnings))))] -#![deny(missing_debug_implementations)] #![no_std] -#![feature(ascii_ctype)] -#![feature(core_char_ext)] -#![feature(str_internals)] -#![feature(decode_utf8)] -#![feature(fn_traits)] -#![feature(lang_items)] -#![feature(non_exhaustive)] +#![feature(unicode_internals)] #![feature(staged_api)] -#![feature(unboxed_closures)] +#![rustc_deprecated(since = "1.27.0", reason = "moved into libcore")] -mod bool_trie; -mod tables; -mod u_str; -mod version; -pub mod char; -pub mod lossy; - -#[allow(deprecated)] -pub mod str { - pub use u_str::{SplitWhitespace, UnicodeStr}; - pub use u_str::Utf16Encoder; -} - -// For use in liballoc, not re-exported in libstd. -pub mod derived_property { - pub use tables::derived_property::{Case_Ignorable, Cased}; -} - -// For use in libsyntax -pub mod property { - pub use tables::property::Pattern_White_Space; -} +pub use core::unicode::*; diff --git a/src/libstd_unicode/tests/lib.rs b/src/libstd_unicode/tests/lib.rs deleted file mode 100644 index 9535ec18763..00000000000 --- a/src/libstd_unicode/tests/lib.rs +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2012-2017 The Rust Project Developers. See the COPYRIGHT -// file at the top-level directory of this distribution and at -// http://rust-lang.org/COPYRIGHT. -// -// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or -// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license -// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -#![feature(str_internals, unicode)] - -extern crate std_unicode; - -mod lossy; diff --git a/src/libstd_unicode/u_str.rs b/src/libstd_unicode/u_str.rs deleted file mode 100644 index a72e1210d93..00000000000 --- a/src/libstd_unicode/u_str.rs +++ /dev/null @@ -1,189 +0,0 @@ -// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT -// file at the top-level directory of this distribution and at -// http://rust-lang.org/COPYRIGHT. -// -// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or -// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license -// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -//! Unicode-intensive string manipulations. -//! -//! This module provides functionality to `str` that requires the Unicode -//! methods provided by the unicode parts of the CharExt trait. - -use core::char; -use core::iter::{Filter, FusedIterator}; -use core::str::Split; - -/// An iterator over the non-whitespace substrings of a string, -/// separated by any amount of whitespace. -/// -/// This struct is created by the [`split_whitespace`] method on [`str`]. -/// See its documentation for more. -/// -/// [`split_whitespace`]: ../../std/primitive.str.html#method.split_whitespace -/// [`str`]: ../../std/primitive.str.html -#[stable(feature = "split_whitespace", since = "1.1.0")] -#[derive(Clone, Debug)] -pub struct SplitWhitespace<'a> { - inner: Filter<Split<'a, IsWhitespace>, IsNotEmpty>, -} - -/// Methods for Unicode string slices -#[allow(missing_docs)] // docs in liballoc -pub trait UnicodeStr { - fn split_whitespace<'a>(&'a self) -> SplitWhitespace<'a>; - fn is_whitespace(&self) -> bool; - fn is_alphanumeric(&self) -> bool; - fn trim(&self) -> &str; - fn trim_left(&self) -> &str; - fn trim_right(&self) -> &str; -} - -impl UnicodeStr for str { - #[inline] - fn split_whitespace(&self) -> SplitWhitespace { - SplitWhitespace { inner: self.split(IsWhitespace).filter(IsNotEmpty) } - } - - #[inline] - fn is_whitespace(&self) -> bool { - self.chars().all(|c| c.is_whitespace()) - } - - #[inline] - fn is_alphanumeric(&self) -> bool { - self.chars().all(|c| c.is_alphanumeric()) - } - - #[inline] - fn trim(&self) -> &str { - self.trim_matches(|c: char| c.is_whitespace()) - } - - #[inline] - fn trim_left(&self) -> &str { - self.trim_left_matches(|c: char| c.is_whitespace()) - } - - #[inline] - fn trim_right(&self) -> &str { - self.trim_right_matches(|c: char| c.is_whitespace()) - } -} - -/// Iterator adaptor for encoding `char`s to UTF-16. -#[derive(Clone)] -#[allow(missing_debug_implementations)] -pub struct Utf16Encoder<I> { - chars: I, - extra: u16, -} - -impl<I> Utf16Encoder<I> { - /// Create a UTF-16 encoder from any `char` iterator. - pub fn new(chars: I) -> Utf16Encoder<I> - where I: Iterator<Item = char> - { - Utf16Encoder { - chars, - extra: 0, - } - } -} - -impl<I> Iterator for Utf16Encoder<I> - where I: Iterator<Item = char> -{ - type Item = u16; - - #[inline] - fn next(&mut self) -> Option<u16> { - if self.extra != 0 { - let tmp = self.extra; - self.extra = 0; - return Some(tmp); - } - - let mut buf = [0; 2]; - self.chars.next().map(|ch| { - let n = CharExt::encode_utf16(ch, &mut buf).len(); - if n == 2 { - self.extra = buf[1]; - } - buf[0] - }) - } - - #[inline] - fn size_hint(&self) -> (usize, Option<usize>) { - let (low, high) = self.chars.size_hint(); - // every char gets either one u16 or two u16, - // so this iterator is between 1 or 2 times as - // long as the underlying iterator. - (low, high.and_then(|n| n.checked_mul(2))) - } -} - -impl<I> FusedIterator for Utf16Encoder<I> - where I: FusedIterator<Item = char> {} - -#[derive(Clone)] -struct IsWhitespace; - -impl FnOnce<(char, )> for IsWhitespace { - type Output = bool; - - #[inline] - extern "rust-call" fn call_once(mut self, arg: (char, )) -> bool { - self.call_mut(arg) - } -} - -impl FnMut<(char, )> for IsWhitespace { - #[inline] - extern "rust-call" fn call_mut(&mut self, arg: (char, )) -> bool { - arg.0.is_whitespace() - } -} - -#[derive(Clone)] -struct IsNotEmpty; - -impl<'a, 'b> FnOnce<(&'a &'b str, )> for IsNotEmpty { - type Output = bool; - - #[inline] - extern "rust-call" fn call_once(mut self, arg: (&&str, )) -> bool { - self.call_mut(arg) - } -} - -impl<'a, 'b> FnMut<(&'a &'b str, )> for IsNotEmpty { - #[inline] - extern "rust-call" fn call_mut(&mut self, arg: (&&str, )) -> bool { - !arg.0.is_empty() - } -} - - -#[stable(feature = "split_whitespace", since = "1.1.0")] -impl<'a> Iterator for SplitWhitespace<'a> { - type Item = &'a str; - - fn next(&mut self) -> Option<&'a str> { - self.inner.next() - } -} - -#[stable(feature = "split_whitespace", since = "1.1.0")] -impl<'a> DoubleEndedIterator for SplitWhitespace<'a> { - fn next_back(&mut self) -> Option<&'a str> { - self.inner.next_back() - } -} - -#[stable(feature = "fused", since = "1.26.0")] -impl<'a> FusedIterator for SplitWhitespace<'a> {} diff --git a/src/libsyntax/lib.rs b/src/libsyntax/lib.rs index dbf2123fb54..ad98e2a6b71 100644 --- a/src/libsyntax/lib.rs +++ b/src/libsyntax/lib.rs @@ -19,7 +19,7 @@ html_root_url = "https://doc.rust-lang.org/nightly/", test(attr(deny(warnings))))] -#![feature(unicode)] +#![feature(unicode_internals)] #![feature(rustc_diagnostic_macros)] #![feature(slice_sort_by_cached_key)] #![feature(non_exhaustive)] @@ -33,9 +33,9 @@ extern crate rustc_cratesio_shim; #[macro_use] extern crate bitflags; +extern crate core; extern crate serialize; #[macro_use] extern crate log; -extern crate std_unicode; pub extern crate rustc_errors as errors; extern crate syntax_pos; extern crate rustc_data_structures; diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs index 39b2f77f230..cb3323c7eca 100644 --- a/src/libsyntax/parse/lexer/mod.rs +++ b/src/libsyntax/parse/lexer/mod.rs @@ -15,7 +15,7 @@ use errors::{FatalError, DiagnosticBuilder}; use parse::{token, ParseSess}; use str::char_at; use symbol::{Symbol, keywords}; -use std_unicode::property::Pattern_White_Space; +use core::unicode::property::Pattern_White_Space; use std::borrow::Cow; use std::char; diff --git a/src/test/compile-fail/single-primitive-inherent-impl.rs b/src/test/compile-fail/single-primitive-inherent-impl.rs index 5ceb870528a..365387c3e5e 100644 --- a/src/test/compile-fail/single-primitive-inherent-impl.rs +++ b/src/test/compile-fail/single-primitive-inherent-impl.rs @@ -15,9 +15,9 @@ #![no_std] // OK -#[lang = "char"] -impl char {} +#[lang = "str"] +impl str {} -impl char { -//~^ error: only a single inherent implementation marked with `#[lang = "char"]` is allowed for the `char` primitive +impl str { +//~^ error: only a single inherent implementation marked with `#[lang = "str"]` is allowed for the `str` primitive } diff --git a/src/test/run-pass/char_unicode.rs b/src/test/run-pass/char_unicode.rs index b4884acdd07..bfc7faac06e 100644 --- a/src/test/run-pass/char_unicode.rs +++ b/src/test/run-pass/char_unicode.rs @@ -8,9 +8,7 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. - -#![feature(unicode)] - +#![feature(unicode_version)] /// Tests access to the internal Unicode Version type and value. pub fn main() { |