Auto merge of #109216 - martingms:unicode-case-lut-shrink, r=Mark-Simulacrum

Shrink unicode case-mapping LUTs by 24k

I was looking into the binary bloat of a small program using `str::to_lowercase` and `str::to_uppercase`, and noticed that the lookup tables used for case mapping had a lot of zero-bytes in them. The reason for this is that since some characters map to up to three other characters when lower or uppercased, the LUTs store a `[char; 3]` for each character. However, the vast majority of cases only map to a single new character, in other words most of the entries are e.g. `(lowerc, [upperc, '\0', '\0'])`.
This PR introduces a new encoding scheme for these tables.

The changes reduces the size of my test binary by about 24K.

I've also done some `#[bench]`marks on unicode-heavy test data, and found that the performance of both `str::to_lowercase` and `str::to_uppercase` improves by up to 20%. These measurements are obviously very dependent on the character distribution of the data.

Someone else will have to decide whether this more complex scheme is worth it or not, I was just goofing around a bit and here's what came out of it :man_shrugging: No hard feelings if this isn't wanted!

1 files changed, 69 insertions, 35 deletions

diff --git a/src/tools/unicode-table-generator/src/case_mapping.rs b/src/tools/unicode-table-generator/src/case_mapping.rs
index 992aac1f857..7a978db62b4 100644
--- a/src/tools/unicode-table-generator/src/case_mapping.rs
+++ b/src/tools/unicode-table-generator/src/case_mapping.rs
@@ -1,36 +1,62 @@
 use crate::{fmt_list, UnicodeData};
-use std::fmt;
+use std::{
+    char,
+    collections::BTreeMap,
+    fmt::{self, Write},
+};
+
+const INDEX_MASK: u32 = 1 << 22;
 
 pub(crate) fn generate_case_mapping(data: &UnicodeData) -> String {
     let mut file = String::new();
 
+    write!(file, "const INDEX_MASK: u32 = 0x{:x};", INDEX_MASK).unwrap();
+    file.push_str("\n\n");
     file.push_str(HEADER.trim_start());
-
-    let decl_type = "&[(char, [char; 3])]";
-
-    file.push_str(&format!(
-        "static LOWERCASE_TABLE: {} = &[{}];",
-        decl_type,
-        fmt_list(data.to_lower.iter().map(to_mapping))
-    ));
+    file.push('\n');
+    file.push_str(&generate_tables("LOWER", &data.to_lower));
     file.push_str("\n\n");
-    file.push_str(&format!(
-        "static UPPERCASE_TABLE: {} = &[{}];",
-        decl_type,
-        fmt_list(data.to_upper.iter().map(to_mapping))
-    ));
+    file.push_str(&generate_tables("UPPER", &data.to_upper));
     file
 }
 
-fn to_mapping((key, (a, b, c)): (&u32, &(u32, u32, u32))) -> (CharEscape, [CharEscape; 3]) {
-    (
-        CharEscape(std::char::from_u32(*key).unwrap()),
-        [
-            CharEscape(std::char::from_u32(*a).unwrap()),
-            CharEscape(std::char::from_u32(*b).unwrap()),
-            CharEscape(std::char::from_u32(*c).unwrap()),
-        ],
-    )
+fn generate_tables(case: &str, data: &BTreeMap<u32, (u32, u32, u32)>) -> String {
+    let mut mappings = Vec::with_capacity(data.len());
+    let mut multis = Vec::new();
+
+    for (&key, &(a, b, c)) in data.iter() {
+        let key = char::from_u32(key).unwrap();
+
+        if key.is_ascii() {
+            continue;
+        }
+
+        let value = if b == 0 && c == 0 {
+            a
+        } else {
+            multis.push([
+                CharEscape(char::from_u32(a).unwrap()),
+                CharEscape(char::from_u32(b).unwrap()),
+                CharEscape(char::from_u32(c).unwrap()),
+            ]);
+
+            INDEX_MASK | (u32::try_from(multis.len()).unwrap() - 1)
+        };
+
+        mappings.push((CharEscape(key), value));
+    }
+
+    let mut tables = String::new();
+
+    write!(tables, "static {}CASE_TABLE: &[(char, u32)] = &[{}];", case, fmt_list(mappings))
+        .unwrap();
+
+    tables.push_str("\n\n");
+
+    write!(tables, "static {}CASE_TABLE_MULTI: &[[char; 3]] = &[{}];", case, fmt_list(multis))
+        .unwrap();
+
+    tables
 }
 
 struct CharEscape(char);
@@ -46,10 +72,16 @@ pub fn to_lower(c: char) -> [char; 3] {
     if c.is_ascii() {
         [(c as u8).to_ascii_lowercase() as char, '\0', '\0']
     } else {
-        match bsearch_case_table(c, LOWERCASE_TABLE) {
-            None => [c, '\0', '\0'],
-            Some(index) => LOWERCASE_TABLE[index].1,
-        }
+        LOWERCASE_TABLE
+            .binary_search_by(|&(key, _)| key.cmp(&c))
+            .map(|i| {
+                let u = LOWERCASE_TABLE[i].1;
+                char::from_u32(u).map(|c| [c, '\0', '\0']).unwrap_or_else(|| {
+                    // SAFETY: Index comes from statically generated table
+                    unsafe { *LOWERCASE_TABLE_MULTI.get_unchecked((u & (INDEX_MASK - 1)) as usize) }
+                })
+            })
+            .unwrap_or([c, '\0', '\0'])
     }
 }
 
@@ -57,14 +89,16 @@ pub fn to_upper(c: char) -> [char; 3] {
     if c.is_ascii() {
         [(c as u8).to_ascii_uppercase() as char, '\0', '\0']
     } else {
-        match bsearch_case_table(c, UPPERCASE_TABLE) {
-            None => [c, '\0', '\0'],
-            Some(index) => UPPERCASE_TABLE[index].1,
-        }
+        UPPERCASE_TABLE
+            .binary_search_by(|&(key, _)| key.cmp(&c))
+            .map(|i| {
+                let u = UPPERCASE_TABLE[i].1;
+                char::from_u32(u).map(|c| [c, '\0', '\0']).unwrap_or_else(|| {
+                    // SAFETY: Index comes from statically generated table
+                    unsafe { *UPPERCASE_TABLE_MULTI.get_unchecked((u & (INDEX_MASK - 1)) as usize) }
+                })
+            })
+            .unwrap_or([c, '\0', '\0'])
     }
 }
-
-fn bsearch_case_table(c: char, table: &[(char, [char; 3])]) -> Option<usize> {
-    table.binary_search_by(|&(key, _)| key.cmp(&c)).ok()
-}
 ";