Fix issue #50811 (`NaN > NaN` was true).

Fix #50811

Make sure the float comparison output is consistent with the expected behavior when NaN is involved.

----

Note: This PR is a **BREAKING CHANGE**. If you have used `>` or `>=` to compare floats, and make the result as the length of a fixed array type, like:

```rust
use std::f64::NAN;
let x: [u8; (NAN > NAN) as usize] = [1];
```

then the code will no longer compile. Previously, all float comparison involving NaN will just return "Greater", i.e. `NAN > NAN` would wrongly return `true` during const evaluation. If you need to retain the old behavior (why), you may replace `a > b` with `a != a || b != b || a > b`.

2 files changed, 71 insertions, 7 deletions

diff --git a/src/librustc_mir/interpret/operator.rs b/src/librustc_mir/interpret/operator.rs
index dfc0c4a824a..727832ec4f4 100644
--- a/src/librustc_mir/interpret/operator.rs
+++ b/src/librustc_mir/interpret/operator.rs
@@ -2,7 +2,6 @@ use rustc::mir;
 use rustc::ty::{self, Ty};
 use rustc_const_math::ConstFloat;
 use syntax::ast::FloatTy;
-use std::cmp::Ordering;
 use rustc::ty::layout::LayoutOf;
 
 use super::{EvalContext, Place, Machine, ValTy};
@@ -135,12 +134,12 @@ impl<'a, 'mir, 'tcx, M: Machine<'mir, 'tcx>> EvalContext<'a, 'mir, 'tcx, M> {
                 ty,
             };
             match op {
-                Eq => PrimVal::from_bool(l.try_cmp(r).unwrap() == Ordering::Equal),
-                Ne => PrimVal::from_bool(l.try_cmp(r).unwrap() != Ordering::Equal),
-                Lt => PrimVal::from_bool(l.try_cmp(r).unwrap() == Ordering::Less),
-                Le => PrimVal::from_bool(l.try_cmp(r).unwrap() != Ordering::Greater),
-                Gt => PrimVal::from_bool(l.try_cmp(r).unwrap() == Ordering::Greater),
-                Ge => PrimVal::from_bool(l.try_cmp(r).unwrap() != Ordering::Less),
+                Eq => PrimVal::from_bool(l == r),
+                Ne => PrimVal::from_bool(l != r),
+                Lt => PrimVal::from_bool(l < r),
+                Le => PrimVal::from_bool(l <= r),
+                Gt => PrimVal::from_bool(l > r),
+                Ge => PrimVal::from_bool(l >= r),
                 Add => PrimVal::Bytes((l + r).unwrap().bits),
                 Sub => PrimVal::Bytes((l - r).unwrap().bits),
                 Mul => PrimVal::Bytes((l * r).unwrap().bits),
diff --git a/src/test/run-pass/issue-50811.rs b/src/test/run-pass/issue-50811.rs
new file mode 100644
index 00000000000..05b168d98f1
--- /dev/null
+++ b/src/test/run-pass/issue-50811.rs
@@ -0,0 +1,65 @@
+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+#![feature(test)]
+
+extern crate test;
+
+use std::f64::{NAN, NEG_INFINITY, INFINITY, MAX};
+use std::mem::size_of;
+use test::black_box;
+
+// Ensure the const-eval result and runtime result of float comparison are equivalent.
+
+macro_rules! compare {
+    ($op:tt) => {
+        compare!(
+            [NEG_INFINITY, -MAX, -1.0, -0.0, 0.0, 1.0, MAX, INFINITY, NAN],
+            $op
+        );
+    };
+    ([$($lhs:expr),+], $op:tt) => {
+        $(compare!(
+            $lhs,
+            $op,
+            [NEG_INFINITY, -MAX, -1.0, -0.0, 0.0, 1.0, MAX, INFINITY, NAN]
+        );)+
+    };
+    ($lhs:expr, $op:tt, [$($rhs:expr),+]) => {
+        $({
+            // Wrap the check in its own function to reduce time needed to borrowck.
+            fn check() {
+                static CONST_EVAL: bool = $lhs $op $rhs;
+                let runtime_eval = black_box($lhs) $op black_box($rhs);
+                assert_eq!(CONST_EVAL, runtime_eval, stringify!($lhs $op $rhs));
+                assert_eq!(
+                    size_of::<[u8; ($lhs $op $rhs) as usize]>(),
+                    runtime_eval as usize,
+                    stringify!($lhs $op $rhs (forced const eval))
+                );
+            }
+            check();
+        })+
+    };
+}
+
+fn main() {
+    assert_eq!(0.0/0.0 < 0.0/0.0, false);
+    assert_eq!(0.0/0.0 > 0.0/0.0, false);
+    assert_eq!(NAN < NAN, false);
+    assert_eq!(NAN > NAN, false);
+
+    compare!(==);
+    compare!(!=);
+    compare!(<);
+    compare!(<=);
+    compare!(>);
+    compare!(>=);
+}