Auto merge of #43025 - est31:nan_cross_platform, r=BurntSushi

Make sNaN removal code tolerate different sNaN encodings

IEEE 754-1985 specifies the encoding of NaN floating point numbers,
but while it mentions that NaNs can be subdivided into signaling
and quiet ones, it doesn't fix the encoding of signaling NaNs in binary
formats. This led to different implementations (CPUs) having different
encodings. IEEE 754-2008 finally specified the encoding of signaling NaNs
but some architectures are compatible with it, while others aren't.
Certain MIPS and PA-RISC CPUs have different encodings for signaling
NaNs.

In order to have the float <-> binary cast feature of the std library be
portable to them, we don't mask any quiet NaNs like we did before (only
being compliant to IEEE 754-2008 and nothing else), but instead we
simply pass a known good NaN instead.

Note that in the code removed there was a bug; the 64 bit mask for quiet
NaNs should have been `0x0008000000000000` instead of the specified
`0x0001000000000000`.

2 files changed, 24 insertions, 13 deletions

diff --git a/src/libstd/f32.rs b/src/libstd/f32.rs
index 7d79fb07811..a6eb17c8fa4 100644
--- a/src/libstd/f32.rs
+++ b/src/libstd/f32.rs
@@ -1131,13 +1131,16 @@ impl f32 {
     #[inline]
     pub fn from_bits(mut v: u32) -> Self {
         const EXP_MASK: u32   = 0x7F800000;
-        const QNAN_MASK: u32  = 0x00400000;
         const FRACT_MASK: u32 = 0x007FFFFF;
         if v & EXP_MASK == EXP_MASK && v & FRACT_MASK != 0 {
-            // If we have a NaN value, we
-            // convert signaling NaN values to quiet NaN
-            // by setting the the highest bit of the fraction
-            v |= QNAN_MASK;
+            // While IEEE 754-2008 specifies encodings for quiet NaNs
+            // and signaling ones, certain MIPS and PA-RISC
+            // CPUs treat signaling NaNs differently.
+            // Therefore to be safe, we pass a known quiet NaN
+            // if v is any kind of NaN.
+            // The check above only assumes IEEE 754-1985 to be
+            // valid.
+            v = unsafe { ::mem::transmute(NAN) };
         }
         unsafe { ::mem::transmute(v) }
     }
@@ -1732,8 +1735,15 @@ mod tests {
     }
     #[test]
     fn test_snan_masking() {
+        // NOTE: this test assumes that our current platform
+        // implements IEEE 754-2008 that specifies the difference
+        // in encoding of quiet and signaling NaNs.
+        // If you are porting Rust to a platform that does not
+        // implement IEEE 754-2008 (but e.g. IEEE 754-1985, which
+        // only says that "Signaling NaNs shall be reserved operands"
+        // but doesn't specify the actual setup), feel free to
+        // cfg out this test.
         let snan: u32 = 0x7F801337;
-        const PAYLOAD_MASK: u32 = 0x003FFFFF;
         const QNAN_MASK: u32  = 0x00400000;
         let nan_masked_fl = f32::from_bits(snan);
         let nan_masked = nan_masked_fl.to_bits();
@@ -1742,7 +1752,5 @@ mod tests {
         // Ensure that we have a quiet NaN
         assert_ne!(nan_masked & QNAN_MASK, 0);
         assert!(nan_masked_fl.is_nan());
-        // Ensure the payload wasn't touched during conversion
-        assert_eq!(nan_masked & PAYLOAD_MASK, snan & PAYLOAD_MASK);
     }
 }
diff --git a/src/libstd/f64.rs b/src/libstd/f64.rs
index d5b0cd3a1fc..4d8d8b4ebe6 100644
--- a/src/libstd/f64.rs
+++ b/src/libstd/f64.rs
@@ -1046,13 +1046,16 @@ impl f64 {
     #[inline]
     pub fn from_bits(mut v: u64) -> Self {
         const EXP_MASK: u64   = 0x7FF0000000000000;
-        const QNAN_MASK: u64  = 0x0001000000000000;
         const FRACT_MASK: u64 = 0x000FFFFFFFFFFFFF;
         if v & EXP_MASK == EXP_MASK && v & FRACT_MASK != 0 {
-            // If we have a NaN value, we
-            // convert signaling NaN values to quiet NaN
-            // by setting the the highest bit of the fraction
-            v |= QNAN_MASK;
+            // While IEEE 754-2008 specifies encodings for quiet NaNs
+            // and signaling ones, certain MIPS and PA-RISC
+            // CPUs treat signaling NaNs differently.
+            // Therefore to be safe, we pass a known quiet NaN
+            // if v is any kind of NaN.
+            // The check above only assumes IEEE 754-1985 to be
+            // valid.
+            v = unsafe { ::mem::transmute(NAN) };
         }
         unsafe { ::mem::transmute(v) }
     }