std: More TLS micro-optimization

2 files changed, 31 insertions, 12 deletions

diff --git a/src/libstd/rt/local.rs b/src/libstd/rt/local.rs
index 1faad913b50..80beb5a2835 100644
--- a/src/libstd/rt/local.rs
+++ b/src/libstd/rt/local.rs
@@ -26,7 +26,9 @@ pub trait Local {
 }
 
 impl Local for Task {
+    #[inline]
     fn put(value: ~Task) { unsafe { local_ptr::put(value) } }
+    #[inline]
     fn take() -> ~Task { unsafe { local_ptr::take() } }
     fn exists() -> bool { local_ptr::exists() }
     fn borrow<T>(f: &fn(&mut Task) -> T) -> T {
@@ -43,7 +45,9 @@ impl Local for Task {
             None => { rtabort!("function failed in local_borrow") }
         }
     }
+    #[inline]
     unsafe fn unsafe_borrow() -> *mut Task { local_ptr::unsafe_borrow() }
+    #[inline]
     unsafe fn try_unsafe_borrow() -> Option<*mut Task> {
         local_ptr::try_unsafe_borrow()
     }
@@ -57,12 +61,12 @@ impl Local for Scheduler {
             task.sched = Some(value.take());
         };
     }
+    #[inline]
     fn take() -> ~Scheduler {
-        do Local::borrow::<Task,~Scheduler> |task| {
-            let sched = task.sched.take_unwrap();
-            let task = task;
-            task.sched = None;
-            sched
+        unsafe {
+            // XXX: Unsafe for speed
+            let task = Local::unsafe_borrow::<Task>();
+            (*task).sched.take_unwrap()
         }
     }
     fn exists() -> bool {
@@ -97,10 +101,17 @@ impl Local for Scheduler {
         }
     }
     unsafe fn try_unsafe_borrow() -> Option<*mut Scheduler> {
-        if Local::exists::<Scheduler>() {
-            Some(Local::unsafe_borrow())
-        } else {
-            None
+        match Local::try_unsafe_borrow::<Task>() {
+            Some(task) => {
+                match (*task).sched {
+                    Some(~ref mut sched) => {
+                        let s: *mut Scheduler = &mut *sched;
+                        Some(s)
+                    }
+                    None => None
+                }
+            }
+            None => None
         }
     }
 }
diff --git a/src/libstd/rt/local_ptr.rs b/src/libstd/rt/local_ptr.rs
index 77303cb8c06..491a864ebfe 100644
--- a/src/libstd/rt/local_ptr.rs
+++ b/src/libstd/rt/local_ptr.rs
@@ -40,6 +40,7 @@ pub fn init_tls_key() {
 /// # Safety note
 ///
 /// Does not validate the pointer type.
+#[inline]
 pub unsafe fn put<T>(sched: ~T) {
     let key = tls_key();
     let void_ptr: *mut c_void = cast::transmute(sched);
@@ -51,6 +52,7 @@ pub unsafe fn put<T>(sched: ~T) {
 /// # Safety note
 ///
 /// Does not validate the pointer type.
+#[inline]
 pub unsafe fn take<T>() -> ~T {
     let key = tls_key();
     let void_ptr: *mut c_void = tls::get(key);
@@ -99,10 +101,15 @@ pub unsafe fn borrow<T>(f: &fn(&mut T)) {
 /// Because this leaves the value in thread-local storage it is possible
 /// For the Scheduler pointer to be aliased
 pub unsafe fn unsafe_borrow<T>() -> *mut T {
-    match try_unsafe_borrow() {
-        Some(p) => p,
-        None => rtabort!("thread-local pointer is null. bogus!")
+    let key = tls_key();
+    let mut void_ptr: *mut c_void = tls::get(key);
+    if void_ptr.is_null() {
+        rtabort!("thread-local pointer is null. bogus!");
     }
+    let ptr: *mut *mut c_void = &mut void_ptr;
+    let ptr: *mut ~T = ptr as *mut ~T;
+    let ptr: *mut T = &mut **ptr;
+    return ptr;
 }
 
 pub unsafe fn try_unsafe_borrow<T>() -> Option<*mut T> {
@@ -119,6 +126,7 @@ pub unsafe fn try_unsafe_borrow<T>() -> Option<*mut T> {
     }
 }
 
+#[inline]
 fn tls_key() -> tls::Key {
     match maybe_tls_key() {
         Some(key) => key,