rollup merge of #21418: Aatch/assume-refcount

The reference count can never be 0, unless we're about to drop the data
completely. Using the `assume` intrinsic allows us to inform LLVM about
that invariant, meaning it can avoid unnecessary drops.

---

Before and after IR: https://gist.github.com/Aatch/3786d20df2edaad6a0e8

Generated from the example in #13018

Fixes #13018

1 files changed, 37 insertions, 4 deletions

diff --git a/src/liballoc/rc.rs b/src/liballoc/rc.rs
index 7191a7af346..e651e375e3d 100644
--- a/src/liballoc/rc.rs
+++ b/src/liballoc/rc.rs
@@ -160,6 +160,7 @@ use core::option::Option::{Some, None};
 use core::ptr::{self, PtrExt};
 use core::result::Result;
 use core::result::Result::{Ok, Err};
+use core::intrinsics::assume;
 
 use heap::deallocate;
 
@@ -905,10 +906,24 @@ trait RcBoxPtr<T> {
     fn strong(&self) -> uint { self.inner().strong.get() }
 
     #[inline]
-    fn inc_strong(&self) { self.inner().strong.set(self.strong() + 1); }
+    fn inc_strong(&self) {
+        let strong = self.strong();
+        // The reference count is always at least one unless we're about to drop the type
+        // This allows the bulk of the destructor to be omitted in cases where we know that
+        // the reference count must be > 0.
+        unsafe { assume(strong > 0); }
+        self.inner().strong.set(strong + 1);
+    }
 
     #[inline]
-    fn dec_strong(&self) { self.inner().strong.set(self.strong() - 1); }
+    fn dec_strong(&self) {
+        let strong = self.strong();
+        // The reference count is always at least one unless we're about to drop the type
+        // This allows the bulk of the destructor to be omitted in cases where we know that
+        // the reference count must be > 0
+        unsafe { assume(strong > 0); }
+        self.inner().strong.set(strong - 1);
+    }
 
     #[inline]
     fn weak(&self) -> uint { self.inner().weak.get() }
@@ -922,12 +937,30 @@ trait RcBoxPtr<T> {
 
 impl<T> RcBoxPtr<T> for Rc<T> {
     #[inline(always)]
-    fn inner(&self) -> &RcBox<T> { unsafe { &(**self._ptr) } }
+    fn inner(&self) -> &RcBox<T> {
+        unsafe {
+            // Safe to assume this here, as if it weren't true, we'd be breaking
+            // the contract anyway.
+            // This allows the null check to be elided in the destructor if we
+            // manipulated the reference count in the same function.
+            assume(!self._ptr.is_null());
+            &(**self._ptr)
+        }
+    }
 }
 
 impl<T> RcBoxPtr<T> for Weak<T> {
     #[inline(always)]
-    fn inner(&self) -> &RcBox<T> { unsafe { &(**self._ptr) } }
+    fn inner(&self) -> &RcBox<T> {
+        unsafe {
+            // Safe to assume this here, as if it weren't true, we'd be breaking
+            // the contract anyway
+            // This allows the null check to be elided in the destructor if we
+            // manipulated the reference count in the same function.
+            assume(!self._ptr.is_null());
+            &(**self._ptr)
+        }
+    }
 }
 
 #[cfg(test)]