From a7ec6f8fe0fb10fa91ac40f68beccd2675cba50c Mon Sep 17 00:00:00 2001 From: Mark Rousskov Date: Sat, 21 Mar 2020 17:20:57 -0400 Subject: Arrange for zero to be canonical We find that it is common for large ranges of chars to be false -- and that means that it is plausibly common for us to ask about a word that is entirely empty. Therefore, we should make sure that we do not need to rotate bits or otherwise perform some operation to map to the zero word; canonicalize it first if possible. --- src/tools/unicode-table-generator/src/raw_emitter.rs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) (limited to 'src/tools') diff --git a/src/tools/unicode-table-generator/src/raw_emitter.rs b/src/tools/unicode-table-generator/src/raw_emitter.rs index 4898df3c800..e5b15224795 100644 --- a/src/tools/unicode-table-generator/src/raw_emitter.rs +++ b/src/tools/unicode-table-generator/src/raw_emitter.rs @@ -301,7 +301,21 @@ impl Canonicalized { Canonicalized(usize), } - while let Some((&to, _)) = mappings.iter().max_by_key(|m| m.1.len()) { + // Map 0 first, so that it is the first canonical word. + // This is realistically not inefficient because 0 is not mapped to by + // anything else (a shift pattern could do it, but would be wasteful). + // + // However, 0s are quite common in the overall dataset, and it is quite + // wasteful to have to go through a mapping function to determine that + // we have a zero. + // + // FIXME: Experiment with choosing most common words in overall data set + // for canonical when possible. + while let Some((&to, _)) = mappings + .iter() + .find(|(&to, _)| to == 0) + .or_else(|| mappings.iter().max_by_key(|m| m.1.len())) + { // Get the mapping with the most entries. Currently, no mapping can // only exist transitively (i.e., there is no A, B, C such that A // does not map to C and but A maps to B maps to C), so this is -- cgit 1.4.1-3-g733a5