From 8752b403695a8830913571f0fd5ebfcf1483db37 Mon Sep 17 00:00:00 2001 From: Alex Huszagh Date: Sat, 17 Jul 2021 00:30:34 -0500 Subject: Changed dec2flt to use the Eisel-Lemire algorithm. Implementation is based off fast-float-rust, with a few notable changes. - Some unsafe methods have been removed. - Safe methods with inherently unsafe functionality have been removed. - All unsafe functionality is documented and provably safe. - Extensive documentation has been added for simpler maintenance. - Inline annotations on internal routines has been removed. - Fixed Python errors in src/etc/test-float-parse/runtests.py. - Updated test-float-parse to be a library, to avoid missing rand dependency. - Added regression tests for #31109 and #31407 in core tests. - Added regression tests for #31109 and #31407 in ui tests. - Use the existing slice primitive to simplify shared dec2flt methods - Remove Miri ignores from dec2flt, due to faster parsing times. - resolves #85198 - resolves #85214 - resolves #85234 - fixes #31407 - fixes #31109 - fixes #53015 - resolves #68396 - closes https://github.com/aldanor/fast-float-rust/issues/15 --- src/etc/test-float-parse/Cargo.toml | 13 ++++++++ src/etc/test-float-parse/_common.rs | 16 ---------- src/etc/test-float-parse/few-ones.rs | 17 ---------- src/etc/test-float-parse/huge-pow10.rs | 11 ------- src/etc/test-float-parse/long-fractions.rs | 17 ---------- src/etc/test-float-parse/many-digits.rs | 27 ---------------- src/etc/test-float-parse/rand-f64.rs | 20 ------------ src/etc/test-float-parse/runtests.py | 37 ++++++++++++---------- src/etc/test-float-parse/short-decimals.rs | 19 ----------- src/etc/test-float-parse/src/bin/few-ones.rs | 15 +++++++++ src/etc/test-float-parse/src/bin/huge-pow10.rs | 9 ++++++ src/etc/test-float-parse/src/bin/long-fractions.rs | 15 +++++++++ src/etc/test-float-parse/src/bin/many-digits.rs | 25 +++++++++++++++ src/etc/test-float-parse/src/bin/rand-f64.rs | 18 +++++++++++ src/etc/test-float-parse/src/bin/short-decimals.rs | 17 ++++++++++ src/etc/test-float-parse/src/bin/subnorm.rs | 11 +++++++ src/etc/test-float-parse/src/bin/tiny-pow10.rs | 9 ++++++ src/etc/test-float-parse/src/bin/u32-small.rs | 7 ++++ src/etc/test-float-parse/src/bin/u64-pow2.rs | 15 +++++++++ src/etc/test-float-parse/src/lib.rs | 16 ++++++++++ src/etc/test-float-parse/subnorm.rs | 13 -------- src/etc/test-float-parse/tiny-pow10.rs | 11 ------- src/etc/test-float-parse/u32-small.rs | 9 ------ src/etc/test-float-parse/u64-pow2.rs | 17 ---------- 24 files changed, 190 insertions(+), 194 deletions(-) create mode 100644 src/etc/test-float-parse/Cargo.toml delete mode 100644 src/etc/test-float-parse/_common.rs delete mode 100644 src/etc/test-float-parse/few-ones.rs delete mode 100644 src/etc/test-float-parse/huge-pow10.rs delete mode 100644 src/etc/test-float-parse/long-fractions.rs delete mode 100644 src/etc/test-float-parse/many-digits.rs delete mode 100644 src/etc/test-float-parse/rand-f64.rs delete mode 100644 src/etc/test-float-parse/short-decimals.rs create mode 100644 src/etc/test-float-parse/src/bin/few-ones.rs create mode 100644 src/etc/test-float-parse/src/bin/huge-pow10.rs create mode 100644 src/etc/test-float-parse/src/bin/long-fractions.rs create mode 100644 src/etc/test-float-parse/src/bin/many-digits.rs create mode 100644 src/etc/test-float-parse/src/bin/rand-f64.rs create mode 100644 src/etc/test-float-parse/src/bin/short-decimals.rs create mode 100644 src/etc/test-float-parse/src/bin/subnorm.rs create mode 100644 src/etc/test-float-parse/src/bin/tiny-pow10.rs create mode 100644 src/etc/test-float-parse/src/bin/u32-small.rs create mode 100644 src/etc/test-float-parse/src/bin/u64-pow2.rs create mode 100644 src/etc/test-float-parse/src/lib.rs delete mode 100644 src/etc/test-float-parse/subnorm.rs delete mode 100644 src/etc/test-float-parse/tiny-pow10.rs delete mode 100644 src/etc/test-float-parse/u32-small.rs delete mode 100644 src/etc/test-float-parse/u64-pow2.rs (limited to 'src/etc/test-float-parse') diff --git a/src/etc/test-float-parse/Cargo.toml b/src/etc/test-float-parse/Cargo.toml new file mode 100644 index 00000000000..8226e815c2c --- /dev/null +++ b/src/etc/test-float-parse/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "test-float-parse" +version = "0.1.0" +edition = "2018" +publish = false + +[workspace] + +[dependencies] +rand = "0.4" + +[lib] +name = "test_float_parse" diff --git a/src/etc/test-float-parse/_common.rs b/src/etc/test-float-parse/_common.rs deleted file mode 100644 index 9cbad5486b4..00000000000 --- a/src/etc/test-float-parse/_common.rs +++ /dev/null @@ -1,16 +0,0 @@ -use std::io; -use std::io::prelude::*; -use std::mem::transmute; - -// Nothing up my sleeve: Just (PI - 3) in base 16. -#[allow(dead_code)] -pub const SEED: [u32; 3] = [0x243f_6a88, 0x85a3_08d3, 0x1319_8a2e]; - -pub fn validate(text: &str) { - let mut out = io::stdout(); - let x: f64 = text.parse().unwrap(); - let f64_bytes: u64 = unsafe { transmute(x) }; - let x: f32 = text.parse().unwrap(); - let f32_bytes: u32 = unsafe { transmute(x) }; - writeln!(&mut out, "{:016x} {:08x} {}", f64_bytes, f32_bytes, text).unwrap(); -} diff --git a/src/etc/test-float-parse/few-ones.rs b/src/etc/test-float-parse/few-ones.rs deleted file mode 100644 index 2d82918aabb..00000000000 --- a/src/etc/test-float-parse/few-ones.rs +++ /dev/null @@ -1,17 +0,0 @@ -mod _common; - -use _common::validate; - -fn main() { - let mut pow = vec![]; - for i in 0..63 { - pow.push(1u64 << i); - } - for a in &pow { - for b in &pow { - for c in &pow { - validate(&(a | b | c).to_string()); - } - } - } -} diff --git a/src/etc/test-float-parse/huge-pow10.rs b/src/etc/test-float-parse/huge-pow10.rs deleted file mode 100644 index 9a16d9c6028..00000000000 --- a/src/etc/test-float-parse/huge-pow10.rs +++ /dev/null @@ -1,11 +0,0 @@ -mod _common; - -use _common::validate; - -fn main() { - for e in 300..310 { - for i in 0..100000 { - validate(&format!("{}e{}", i, e)); - } - } -} diff --git a/src/etc/test-float-parse/long-fractions.rs b/src/etc/test-float-parse/long-fractions.rs deleted file mode 100644 index 60cf85c4a60..00000000000 --- a/src/etc/test-float-parse/long-fractions.rs +++ /dev/null @@ -1,17 +0,0 @@ -mod _common; - -use _common::validate; -use std::char; - -fn main() { - for n in 0..10 { - let digit = char::from_digit(n, 10).unwrap(); - let mut s = "0.".to_string(); - for _ in 0..400 { - s.push(digit); - if s.parse::().is_ok() { - validate(&s); - } - } - } -} diff --git a/src/etc/test-float-parse/many-digits.rs b/src/etc/test-float-parse/many-digits.rs deleted file mode 100644 index 599986e20dd..00000000000 --- a/src/etc/test-float-parse/many-digits.rs +++ /dev/null @@ -1,27 +0,0 @@ -extern crate rand; - -mod _common; - -use _common::{validate, SEED}; -use rand::distributions::{Range, Sample}; -use rand::{IsaacRng, Rng, SeedableRng}; -use std::char; - -fn main() { - let mut rnd = IsaacRng::from_seed(&SEED); - let mut range = Range::new(0, 10); - for _ in 0..5_000_000u64 { - let num_digits = rnd.gen_range(100, 400); - let digits = gen_digits(num_digits, &mut range, &mut rnd); - validate(&digits); - } -} - -fn gen_digits(n: u32, range: &mut Range, rnd: &mut R) -> String { - let mut s = String::new(); - for _ in 0..n { - let digit = char::from_digit(range.sample(rnd), 10).unwrap(); - s.push(digit); - } - s -} diff --git a/src/etc/test-float-parse/rand-f64.rs b/src/etc/test-float-parse/rand-f64.rs deleted file mode 100644 index 39ad63945cd..00000000000 --- a/src/etc/test-float-parse/rand-f64.rs +++ /dev/null @@ -1,20 +0,0 @@ -extern crate rand; - -mod _common; - -use _common::{validate, SEED}; -use rand::{IsaacRng, Rng, SeedableRng}; -use std::mem::transmute; - -fn main() { - let mut rnd = IsaacRng::from_seed(&SEED); - let mut i = 0; - while i < 10_000_000 { - let bits = rnd.next_u64(); - let x: f64 = unsafe { transmute(bits) }; - if x.is_finite() { - validate(&format!("{:e}", x)); - i += 1; - } - } -} diff --git a/src/etc/test-float-parse/runtests.py b/src/etc/test-float-parse/runtests.py index 218552a4597..cf7279534dc 100644 --- a/src/etc/test-float-parse/runtests.py +++ b/src/etc/test-float-parse/runtests.py @@ -131,22 +131,20 @@ def write_errors(): exit_status = 101 -def rustc(test): - rs = test + '.rs' - exe = test + '.exe' # hopefully this makes it work on *nix - print("compiling", test) +def cargo(): + print("compiling tests") sys.stdout.flush() - check_call(['rustc', rs, '-o', exe]) + check_call(['cargo', 'build', '--release']) def run(test): global test_name test_name = test - t0 = time.clock() + t0 = time.perf_counter() msg("setting up supervisor") - exe = test + '.exe' - proc = Popen(exe, bufsize=1<<20 , stdin=PIPE, stdout=PIPE, stderr=PIPE) + command = ['cargo', 'run', '--bin', test, '--release'] + proc = Popen(command, bufsize=1<<20 , stdin=PIPE, stdout=PIPE, stderr=PIPE) done = multiprocessing.Value(ctypes.c_bool) queue = multiprocessing.Queue(maxsize=5)#(maxsize=1024) workers = [] @@ -166,7 +164,7 @@ def run(test): worker.join() msg("python is done") assert queue.empty(), "did not validate everything" - dt = time.clock() - t0 + dt = time.perf_counter() - t0 msg("took", round(dt, 3), "seconds") @@ -176,7 +174,7 @@ def interact(proc, queue): line = proc.stdout.readline() if not line: continue - assert line.endswith('\n'), "incomplete line: " + repr(line) + assert line.endswith(b'\n'), "incomplete line: " + repr(line) queue.put(line) n += 1 if n % UPDATE_EVERY_N == 0: @@ -185,7 +183,7 @@ def interact(proc, queue): rest, stderr = proc.communicate() if stderr: msg("rust stderr output:", stderr) - for line in rest.split('\n'): + for line in rest.split(b'\n'): if not line: continue queue.put(line) @@ -193,18 +191,19 @@ def interact(proc, queue): def main(): global MAILBOX - all_tests = [os.path.splitext(f)[0] for f in glob('*.rs') if not f.startswith('_')] + files = glob('src/bin/*.rs') + basenames = [os.path.basename(i) for i in files] + all_tests = [os.path.splitext(f)[0] for f in basenames if not f.startswith('_')] args = sys.argv[1:] if args: tests = [test for test in all_tests if test in args] - else + else: tests = all_tests if not tests: print("Error: No tests to run") sys.exit(1) # Compile first for quicker feedback - for test in tests: - rustc(test) + cargo() # Set up mailbox once for all tests MAILBOX = multiprocessing.Queue() mailman = threading.Thread(target=write_errors) @@ -251,7 +250,7 @@ def do_work(queue): else: continue bin64, bin32, text = line.rstrip().split() - validate(bin64, bin32, text) + validate(bin64, bin32, text.decode('utf-8')) def decode_binary64(x): @@ -331,7 +330,11 @@ SINGLE_ZERO_CUTOFF = MIN_SUBNORMAL_SINGLE / 2 SINGLE_INF_CUTOFF = MAX_SINGLE + 2 ** (MAX_ULP_SINGLE - 1) def validate(bin64, bin32, text): - double = decode_binary64(bin64) + try: + double = decode_binary64(bin64) + except AssertionError: + print(bin64, bin32, text) + raise single = decode_binary32(bin32) real = Fraction(text) diff --git a/src/etc/test-float-parse/short-decimals.rs b/src/etc/test-float-parse/short-decimals.rs deleted file mode 100644 index 8b9b6f78ae3..00000000000 --- a/src/etc/test-float-parse/short-decimals.rs +++ /dev/null @@ -1,19 +0,0 @@ -mod _common; - -use _common::validate; - -fn main() { - // Skip e = 0 because small-u32 already does those. - for e in 1..301 { - for i in 0..10000 { - // If it ends in zeros, the parser will strip those (and adjust the exponent), - // which almost always (except for exponents near +/- 300) result in an input - // equivalent to something we already generate in a different way. - if i % 10 == 0 { - continue; - } - validate(&format!("{}e{}", i, e)); - validate(&format!("{}e-{}", i, e)); - } - } -} diff --git a/src/etc/test-float-parse/src/bin/few-ones.rs b/src/etc/test-float-parse/src/bin/few-ones.rs new file mode 100644 index 00000000000..6bb406a5947 --- /dev/null +++ b/src/etc/test-float-parse/src/bin/few-ones.rs @@ -0,0 +1,15 @@ +use test_float_parse::validate; + +fn main() { + let mut pow = vec![]; + for i in 0..63 { + pow.push(1u64 << i); + } + for a in &pow { + for b in &pow { + for c in &pow { + validate(&(a | b | c).to_string()); + } + } + } +} diff --git a/src/etc/test-float-parse/src/bin/huge-pow10.rs b/src/etc/test-float-parse/src/bin/huge-pow10.rs new file mode 100644 index 00000000000..722a24ffcd8 --- /dev/null +++ b/src/etc/test-float-parse/src/bin/huge-pow10.rs @@ -0,0 +1,9 @@ +use test_float_parse::validate; + +fn main() { + for e in 300..310 { + for i in 0..100000 { + validate(&format!("{}e{}", i, e)); + } + } +} diff --git a/src/etc/test-float-parse/src/bin/long-fractions.rs b/src/etc/test-float-parse/src/bin/long-fractions.rs new file mode 100644 index 00000000000..c715bc1ac2b --- /dev/null +++ b/src/etc/test-float-parse/src/bin/long-fractions.rs @@ -0,0 +1,15 @@ +use std::char; +use test_float_parse::validate; + +fn main() { + for n in 0..10 { + let digit = char::from_digit(n, 10).unwrap(); + let mut s = "0.".to_string(); + for _ in 0..400 { + s.push(digit); + if s.parse::().is_ok() { + validate(&s); + } + } + } +} diff --git a/src/etc/test-float-parse/src/bin/many-digits.rs b/src/etc/test-float-parse/src/bin/many-digits.rs new file mode 100644 index 00000000000..ba166fd5607 --- /dev/null +++ b/src/etc/test-float-parse/src/bin/many-digits.rs @@ -0,0 +1,25 @@ +extern crate rand; + +use rand::distributions::{Range, Sample}; +use rand::{IsaacRng, Rng, SeedableRng}; +use std::char; +use test_float_parse::{validate, SEED}; + +fn main() { + let mut rnd = IsaacRng::from_seed(&SEED); + let mut range = Range::new(0, 10); + for _ in 0..5_000_000u64 { + let num_digits = rnd.gen_range(100, 400); + let digits = gen_digits(num_digits, &mut range, &mut rnd); + validate(&digits); + } +} + +fn gen_digits(n: u32, range: &mut Range, rnd: &mut R) -> String { + let mut s = String::new(); + for _ in 0..n { + let digit = char::from_digit(range.sample(rnd), 10).unwrap(); + s.push(digit); + } + s +} diff --git a/src/etc/test-float-parse/src/bin/rand-f64.rs b/src/etc/test-float-parse/src/bin/rand-f64.rs new file mode 100644 index 00000000000..6991e8be15e --- /dev/null +++ b/src/etc/test-float-parse/src/bin/rand-f64.rs @@ -0,0 +1,18 @@ +extern crate rand; + +use rand::{IsaacRng, Rng, SeedableRng}; +use std::mem::transmute; +use test_float_parse::{validate, SEED}; + +fn main() { + let mut rnd = IsaacRng::from_seed(&SEED); + let mut i = 0; + while i < 10_000_000 { + let bits = rnd.next_u64(); + let x: f64 = unsafe { transmute(bits) }; + if x.is_finite() { + validate(&format!("{:e}", x)); + i += 1; + } + } +} diff --git a/src/etc/test-float-parse/src/bin/short-decimals.rs b/src/etc/test-float-parse/src/bin/short-decimals.rs new file mode 100644 index 00000000000..49084eb35e8 --- /dev/null +++ b/src/etc/test-float-parse/src/bin/short-decimals.rs @@ -0,0 +1,17 @@ +use test_float_parse::validate; + +fn main() { + // Skip e = 0 because small-u32 already does those. + for e in 1..301 { + for i in 0..10000 { + // If it ends in zeros, the parser will strip those (and adjust the exponent), + // which almost always (except for exponents near +/- 300) result in an input + // equivalent to something we already generate in a different way. + if i % 10 == 0 { + continue; + } + validate(&format!("{}e{}", i, e)); + validate(&format!("{}e-{}", i, e)); + } + } +} diff --git a/src/etc/test-float-parse/src/bin/subnorm.rs b/src/etc/test-float-parse/src/bin/subnorm.rs new file mode 100644 index 00000000000..ac88747eacd --- /dev/null +++ b/src/etc/test-float-parse/src/bin/subnorm.rs @@ -0,0 +1,11 @@ +use std::mem::transmute; +use test_float_parse::validate; + +fn main() { + for bits in 0u32..(1 << 21) { + let single: f32 = unsafe { transmute(bits) }; + validate(&format!("{:e}", single)); + let double: f64 = unsafe { transmute(bits as u64) }; + validate(&format!("{:e}", double)); + } +} diff --git a/src/etc/test-float-parse/src/bin/tiny-pow10.rs b/src/etc/test-float-parse/src/bin/tiny-pow10.rs new file mode 100644 index 00000000000..fb6ba166380 --- /dev/null +++ b/src/etc/test-float-parse/src/bin/tiny-pow10.rs @@ -0,0 +1,9 @@ +use test_float_parse::validate; + +fn main() { + for e in 301..327 { + for i in 0..100000 { + validate(&format!("{}e-{}", i, e)); + } + } +} diff --git a/src/etc/test-float-parse/src/bin/u32-small.rs b/src/etc/test-float-parse/src/bin/u32-small.rs new file mode 100644 index 00000000000..5ec9d1eea5f --- /dev/null +++ b/src/etc/test-float-parse/src/bin/u32-small.rs @@ -0,0 +1,7 @@ +use test_float_parse::validate; + +fn main() { + for i in 0..(1 << 19) { + validate(&i.to_string()); + } +} diff --git a/src/etc/test-float-parse/src/bin/u64-pow2.rs b/src/etc/test-float-parse/src/bin/u64-pow2.rs new file mode 100644 index 00000000000..984e49200cd --- /dev/null +++ b/src/etc/test-float-parse/src/bin/u64-pow2.rs @@ -0,0 +1,15 @@ +use test_float_parse::validate; + +fn main() { + for exp in 19..64 { + let power: u64 = 1 << exp; + validate(&power.to_string()); + for offset in 1..123 { + validate(&(power + offset).to_string()); + validate(&(power - offset).to_string()); + } + } + for offset in 0..123 { + validate(&(u64::MAX - offset).to_string()); + } +} diff --git a/src/etc/test-float-parse/src/lib.rs b/src/etc/test-float-parse/src/lib.rs new file mode 100644 index 00000000000..9cbad5486b4 --- /dev/null +++ b/src/etc/test-float-parse/src/lib.rs @@ -0,0 +1,16 @@ +use std::io; +use std::io::prelude::*; +use std::mem::transmute; + +// Nothing up my sleeve: Just (PI - 3) in base 16. +#[allow(dead_code)] +pub const SEED: [u32; 3] = [0x243f_6a88, 0x85a3_08d3, 0x1319_8a2e]; + +pub fn validate(text: &str) { + let mut out = io::stdout(); + let x: f64 = text.parse().unwrap(); + let f64_bytes: u64 = unsafe { transmute(x) }; + let x: f32 = text.parse().unwrap(); + let f32_bytes: u32 = unsafe { transmute(x) }; + writeln!(&mut out, "{:016x} {:08x} {}", f64_bytes, f32_bytes, text).unwrap(); +} diff --git a/src/etc/test-float-parse/subnorm.rs b/src/etc/test-float-parse/subnorm.rs deleted file mode 100644 index ba68d31e4ed..00000000000 --- a/src/etc/test-float-parse/subnorm.rs +++ /dev/null @@ -1,13 +0,0 @@ -mod _common; - -use _common::validate; -use std::mem::transmute; - -fn main() { - for bits in 0u32..(1 << 21) { - let single: f32 = unsafe { transmute(bits) }; - validate(&format!("{:e}", single)); - let double: f64 = unsafe { transmute(bits as u64) }; - validate(&format!("{:e}", double)); - } -} diff --git a/src/etc/test-float-parse/tiny-pow10.rs b/src/etc/test-float-parse/tiny-pow10.rs deleted file mode 100644 index 43fad5b49ee..00000000000 --- a/src/etc/test-float-parse/tiny-pow10.rs +++ /dev/null @@ -1,11 +0,0 @@ -mod _common; - -use _common::validate; - -fn main() { - for e in 301..327 { - for i in 0..100000 { - validate(&format!("{}e-{}", i, e)); - } - } -} diff --git a/src/etc/test-float-parse/u32-small.rs b/src/etc/test-float-parse/u32-small.rs deleted file mode 100644 index 3ae62425adf..00000000000 --- a/src/etc/test-float-parse/u32-small.rs +++ /dev/null @@ -1,9 +0,0 @@ -mod _common; - -use _common::validate; - -fn main() { - for i in 0..(1 << 19) { - validate(&i.to_string()); - } -} diff --git a/src/etc/test-float-parse/u64-pow2.rs b/src/etc/test-float-parse/u64-pow2.rs deleted file mode 100644 index 7e67e2b1246..00000000000 --- a/src/etc/test-float-parse/u64-pow2.rs +++ /dev/null @@ -1,17 +0,0 @@ -mod _common; - -use _common::validate; - -fn main() { - for exp in 19..64 { - let power: u64 = 1 << exp; - validate(&power.to_string()); - for offset in 1..123 { - validate(&(power + offset).to_string()); - validate(&(power - offset).to_string()); - } - } - for offset in 0..123 { - validate(&(u64::MAX - offset).to_string()); - } -} -- cgit 1.4.1-3-g733a5