diff options
| -rw-r--r-- | src/libcollectionstest/lib.rs | 1 | ||||
| -rw-r--r-- | src/libcollectionstest/str.rs | 30 | ||||
| -rw-r--r-- | src/libcore/str/mod.rs | 80 |
3 files changed, 89 insertions, 22 deletions
diff --git a/src/libcollectionstest/lib.rs b/src/libcollectionstest/lib.rs index d97d9b8ab83..98d0b1c8e15 100644 --- a/src/libcollectionstest/lib.rs +++ b/src/libcollectionstest/lib.rs @@ -28,6 +28,7 @@ #![feature(test)] #![feature(unboxed_closures)] #![feature(unicode)] +#![feature(utf8_error_error_len)] extern crate collections; extern crate test; diff --git a/src/libcollectionstest/str.rs b/src/libcollectionstest/str.rs index 8071c7e8c20..c9b7104fec4 100644 --- a/src/libcollectionstest/str.rs +++ b/src/libcollectionstest/str.rs @@ -541,6 +541,36 @@ fn from_utf8_mostly_ascii() { } #[test] +fn from_utf8_error() { + macro_rules! test { + ($input: expr, $expected_valid_up_to: expr, $expected_error_len: expr) => { + let error = from_utf8($input).unwrap_err(); + assert_eq!(error.valid_up_to(), $expected_valid_up_to); + assert_eq!(error.error_len(), $expected_error_len); + } + } + test!(b"A\xC3\xA9 \xFF ", 4, Some(1)); + test!(b"A\xC3\xA9 \x80 ", 4, Some(1)); + test!(b"A\xC3\xA9 \xC1 ", 4, Some(1)); + test!(b"A\xC3\xA9 \xC1", 4, Some(1)); + test!(b"A\xC3\xA9 \xC2", 4, None); + test!(b"A\xC3\xA9 \xC2 ", 4, Some(1)); + test!(b"A\xC3\xA9 \xC2\xC0", 4, Some(1)); + test!(b"A\xC3\xA9 \xE0", 4, None); + test!(b"A\xC3\xA9 \xE0\x9F", 4, Some(1)); + test!(b"A\xC3\xA9 \xE0\xA0", 4, None); + test!(b"A\xC3\xA9 \xE0\xA0\xC0", 4, Some(2)); + test!(b"A\xC3\xA9 \xE0\xA0 ", 4, Some(2)); + test!(b"A\xC3\xA9 \xED\xA0\x80 ", 4, Some(1)); + test!(b"A\xC3\xA9 \xF1", 4, None); + test!(b"A\xC3\xA9 \xF1\x80", 4, None); + test!(b"A\xC3\xA9 \xF1\x80\x80", 4, None); + test!(b"A\xC3\xA9 \xF1 ", 4, Some(1)); + test!(b"A\xC3\xA9 \xF1\x80 ", 4, Some(2)); + test!(b"A\xC3\xA9 \xF1\x80\x80 ", 4, Some(3)); +} + +#[test] fn test_as_bytes() { // no null let v = [ diff --git a/src/libcore/str/mod.rs b/src/libcore/str/mod.rs index 52e33016310..2919adc1cbc 100644 --- a/src/libcore/str/mod.rs +++ b/src/libcore/str/mod.rs @@ -125,13 +125,14 @@ Section: Creating a string #[stable(feature = "rust1", since = "1.0.0")] pub struct Utf8Error { valid_up_to: usize, + error_len: Option<u8>, } impl Utf8Error { /// Returns the index in the given string up to which valid UTF-8 was /// verified. /// - /// It is the maximum index such that `from_utf8(input[..index])` + /// It is the maximum index such that `from_utf8(&input[..index])` /// would return `Ok(_)`. /// /// # Examples @@ -152,6 +153,23 @@ impl Utf8Error { /// ``` #[stable(feature = "utf8_error", since = "1.5.0")] pub fn valid_up_to(&self) -> usize { self.valid_up_to } + + /// Provide more information about the failure: + /// + /// * `None`: the end of the input was reached unexpectedly. + /// `self.valid_up_to()` is 1 to 3 bytes from the end of the input. + /// If a byte stream (such as a file or a network socket) is being decoded incrementally, + /// this could be a valid `char` whose UTF-8 byte sequence is spanning multiple chunks. + /// + /// * `Some(len)`: an unexpected byte was encountered. + /// The length provided is that of the invalid byte sequence + /// that starts at the index given by `valid_up_to()`. + /// Decoding should resume after that sequence + /// (after inserting a U+FFFD REPLACEMENT CHARACTER) in case of lossy decoding. + #[unstable(feature = "utf8_error_error_len", reason ="new", issue = "40494")] + pub fn error_len(&self) -> Option<usize> { + self.error_len.map(|len| len as usize) + } } /// Converts a slice of bytes to a string slice. @@ -300,7 +318,12 @@ pub unsafe fn from_utf8_unchecked(v: &[u8]) -> &str { #[stable(feature = "rust1", since = "1.0.0")] impl fmt::Display for Utf8Error { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "invalid utf-8: invalid byte near index {}", self.valid_up_to) + if let Some(error_len) = self.error_len { + write!(f, "invalid utf-8 sequence of {} bytes from index {}", + error_len, self.valid_up_to) + } else { + write!(f, "incomplete utf-8 byte sequence from index {}", self.valid_up_to) + } } } @@ -1241,17 +1264,20 @@ fn run_utf8_validation(v: &[u8]) -> Result<(), Utf8Error> { while index < len { let old_offset = index; - macro_rules! err { () => {{ - return Err(Utf8Error { - valid_up_to: old_offset - }) - }}} + macro_rules! err { + ($error_len: expr) => { + return Err(Utf8Error { + valid_up_to: old_offset, + error_len: $error_len, + }) + } + } macro_rules! next { () => {{ index += 1; // we needed data, but there was none: error! if index >= len { - err!() + err!(None) } v[index] }}} @@ -1259,7 +1285,6 @@ fn run_utf8_validation(v: &[u8]) -> Result<(), Utf8Error> { let first = v[index]; if first >= 128 { let w = UTF8_CHAR_WIDTH[first as usize]; - let second = next!(); // 2-byte encoding is for codepoints \u{0080} to \u{07ff} // first C2 80 last DF BF // 3-byte encoding is for codepoints \u{0800} to \u{ffff} @@ -1279,25 +1304,36 @@ fn run_utf8_validation(v: &[u8]) -> Result<(), Utf8Error> { // UTF8-4 = %xF0 %x90-BF 2( UTF8-tail ) / %xF1-F3 3( UTF8-tail ) / // %xF4 %x80-8F 2( UTF8-tail ) match w { - 2 => if second & !CONT_MASK != TAG_CONT_U8 {err!()}, + 2 => if next!() & !CONT_MASK != TAG_CONT_U8 { + err!(Some(1)) + }, 3 => { - match (first, second, next!() & !CONT_MASK) { - (0xE0 , 0xA0 ... 0xBF, TAG_CONT_U8) | - (0xE1 ... 0xEC, 0x80 ... 0xBF, TAG_CONT_U8) | - (0xED , 0x80 ... 0x9F, TAG_CONT_U8) | - (0xEE ... 0xEF, 0x80 ... 0xBF, TAG_CONT_U8) => {} - _ => err!() + match (first, next!()) { + (0xE0 , 0xA0 ... 0xBF) | + (0xE1 ... 0xEC, 0x80 ... 0xBF) | + (0xED , 0x80 ... 0x9F) | + (0xEE ... 0xEF, 0x80 ... 0xBF) => {} + _ => err!(Some(1)) + } + if next!() & !CONT_MASK != TAG_CONT_U8 { + err!(Some(2)) } } 4 => { - match (first, second, next!() & !CONT_MASK, next!() & !CONT_MASK) { - (0xF0 , 0x90 ... 0xBF, TAG_CONT_U8, TAG_CONT_U8) | - (0xF1 ... 0xF3, 0x80 ... 0xBF, TAG_CONT_U8, TAG_CONT_U8) | - (0xF4 , 0x80 ... 0x8F, TAG_CONT_U8, TAG_CONT_U8) => {} - _ => err!() + match (first, next!()) { + (0xF0 , 0x90 ... 0xBF) | + (0xF1 ... 0xF3, 0x80 ... 0xBF) | + (0xF4 , 0x80 ... 0x8F) => {} + _ => err!(Some(1)) + } + if next!() & !CONT_MASK != TAG_CONT_U8 { + err!(Some(2)) + } + if next!() & !CONT_MASK != TAG_CONT_U8 { + err!(Some(3)) } } - _ => err!() + _ => err!(Some(1)) } index += 1; } else { |