auto merge of #8740 : brson/rust/rt-opt, r=thestinger

See #8599

13 files changed, 232 insertions, 204 deletions

diff --git a/src/libstd/macros.rs b/src/libstd/macros.rs
index 600d0bb133e..5378a2c798d 100644
--- a/src/libstd/macros.rs
+++ b/src/libstd/macros.rs
@@ -27,8 +27,10 @@ macro_rules! rtdebug (
 
 macro_rules! rtassert (
     ( $arg:expr ) => ( {
-        if !$arg {
-            rtabort!("assertion failed: %s", stringify!($arg));
+        if ::rt::util::ENFORCE_SANITY {
+            if !$arg {
+                rtabort!("assertion failed: %s", stringify!($arg));
+            }
         }
     } )
 )
diff --git a/src/libstd/rt/comm.rs b/src/libstd/rt/comm.rs
index 8ef9c1332f9..bd83e286156 100644
--- a/src/libstd/rt/comm.rs
+++ b/src/libstd/rt/comm.rs
@@ -125,7 +125,7 @@ impl<T> ChanOne<T> {
         unsafe {
 
             // Install the payload
-            assert!((*packet).payload.is_none());
+            rtassert!((*packet).payload.is_none());
             (*packet).payload = Some(val);
 
             // Atomically swap out the old state to figure out what
@@ -144,16 +144,8 @@ impl<T> ChanOne<T> {
             match oldstate {
                 STATE_BOTH => {
                     // Port is not waiting yet. Nothing to do
-                    do Local::borrow::<Scheduler, ()> |sched| {
-                        rtdebug!("non-rendezvous send");
-                        sched.metrics.non_rendezvous_sends += 1;
-                    }
                 }
                 STATE_ONE => {
-                    do Local::borrow::<Scheduler, ()> |sched| {
-                        rtdebug!("rendezvous send");
-                        sched.metrics.rendezvous_sends += 1;
-                    }
                     // Port has closed. Need to clean up.
                     let _packet: ~Packet<T> = cast::transmute(this.void_packet);
                     recvr_active = false;
@@ -251,7 +243,6 @@ impl<T> SelectInner for PortOne<T> {
                 STATE_BOTH => {
                     // Data has not been sent. Now we're blocked.
                     rtdebug!("non-rendezvous recv");
-                    sched.metrics.non_rendezvous_recvs += 1;
                     false
                 }
                 STATE_ONE => {
@@ -267,7 +258,6 @@ impl<T> SelectInner for PortOne<T> {
                     (*self.packet()).state.store(STATE_ONE, Relaxed);
 
                     rtdebug!("rendezvous recv");
-                    sched.metrics.rendezvous_recvs += 1;
 
                     // Channel is closed. Switch back and check the data.
                     // NB: We have to drop back into the scheduler event loop here
@@ -307,7 +297,7 @@ impl<T> SelectInner for PortOne<T> {
                         STATE_ONE  => true, // Lost the race. Data available.
                         same_ptr   => {
                             // We successfully unblocked our task pointer.
-                            assert!(task_as_state == same_ptr);
+                            rtassert!(task_as_state == same_ptr);
                             let handle = BlockedTask::cast_from_uint(task_as_state);
                             // Because we are already awake, the handle we
                             // gave to this port shall already be empty.
@@ -341,7 +331,8 @@ impl<T> SelectPortInner<T> for PortOne<T> {
         unsafe {
             // See corresponding store() above in block_on for rationale.
             // FIXME(#8130) This can happen only in test builds.
-            assert!((*packet).state.load(Relaxed) == STATE_ONE);
+            // This load is not required for correctness and may be compiled out.
+            rtassert!((*packet).state.load(Relaxed) == STATE_ONE);
 
             let payload = (*packet).payload.take();
 
@@ -387,7 +378,7 @@ impl<T> Drop for ChanOne<T> {
                 },
                 task_as_state => {
                     // The port is blocked waiting for a message we will never send. Wake it.
-                    assert!((*this.packet()).payload.is_none());
+                    rtassert!((*this.packet()).payload.is_none());
                     let recvr = BlockedTask::cast_from_uint(task_as_state);
                     do recvr.wake().map_move |woken_task| {
                         Scheduler::run_task(woken_task);
diff --git a/src/libstd/rt/local.rs b/src/libstd/rt/local.rs
index 1faad913b50..18b7394700f 100644
--- a/src/libstd/rt/local.rs
+++ b/src/libstd/rt/local.rs
@@ -21,12 +21,15 @@ pub trait Local {
     fn take() -> ~Self;
     fn exists() -> bool;
     fn borrow<T>(f: &fn(&mut Self) -> T) -> T;
+    unsafe fn unsafe_take() -> ~Self;
     unsafe fn unsafe_borrow() -> *mut Self;
     unsafe fn try_unsafe_borrow() -> Option<*mut Self>;
 }
 
 impl Local for Task {
+    #[inline]
     fn put(value: ~Task) { unsafe { local_ptr::put(value) } }
+    #[inline]
     fn take() -> ~Task { unsafe { local_ptr::take() } }
     fn exists() -> bool { local_ptr::exists() }
     fn borrow<T>(f: &fn(&mut Task) -> T) -> T {
@@ -43,7 +46,11 @@ impl Local for Task {
             None => { rtabort!("function failed in local_borrow") }
         }
     }
+    #[inline]
+    unsafe fn unsafe_take() -> ~Task { local_ptr::unsafe_take() }
+    #[inline]
     unsafe fn unsafe_borrow() -> *mut Task { local_ptr::unsafe_borrow() }
+    #[inline]
     unsafe fn try_unsafe_borrow() -> Option<*mut Task> {
         local_ptr::try_unsafe_borrow()
     }
@@ -57,12 +64,12 @@ impl Local for Scheduler {
             task.sched = Some(value.take());
         };
     }
+    #[inline]
     fn take() -> ~Scheduler {
-        do Local::borrow::<Task,~Scheduler> |task| {
-            let sched = task.sched.take_unwrap();
-            let task = task;
-            task.sched = None;
-            sched
+        unsafe {
+            // XXX: Unsafe for speed
+            let task = Local::unsafe_borrow::<Task>();
+            (*task).sched.take_unwrap()
         }
     }
     fn exists() -> bool {
@@ -85,6 +92,7 @@ impl Local for Scheduler {
             }
         }
     }
+    unsafe fn unsafe_take() -> ~Scheduler { rtabort!("unimpl") }
     unsafe fn unsafe_borrow() -> *mut Scheduler {
         match (*Local::unsafe_borrow::<Task>()).sched {
             Some(~ref mut sched) => {
@@ -97,10 +105,17 @@ impl Local for Scheduler {
         }
     }
     unsafe fn try_unsafe_borrow() -> Option<*mut Scheduler> {
-        if Local::exists::<Scheduler>() {
-            Some(Local::unsafe_borrow())
-        } else {
-            None
+        match Local::try_unsafe_borrow::<Task>() {
+            Some(task) => {
+                match (*task).sched {
+                    Some(~ref mut sched) => {
+                        let s: *mut Scheduler = &mut *sched;
+                        Some(s)
+                    }
+                    None => None
+                }
+            }
+            None => None
         }
     }
 }
@@ -111,6 +126,7 @@ impl Local for IoFactoryObject {
     fn take() -> ~IoFactoryObject { rtabort!("unimpl") }
     fn exists() -> bool { rtabort!("unimpl") }
     fn borrow<T>(_f: &fn(&mut IoFactoryObject) -> T) -> T { rtabort!("unimpl") }
+    unsafe fn unsafe_take() -> ~IoFactoryObject { rtabort!("unimpl") }
     unsafe fn unsafe_borrow() -> *mut IoFactoryObject {
         let sched = Local::unsafe_borrow::<Scheduler>();
         let io: *mut IoFactoryObject = (*sched).event_loop.io().unwrap();
diff --git a/src/libstd/rt/local_ptr.rs b/src/libstd/rt/local_ptr.rs
index 77303cb8c06..e843fd1adef 100644
--- a/src/libstd/rt/local_ptr.rs
+++ b/src/libstd/rt/local_ptr.rs
@@ -23,14 +23,16 @@ use option::{Option, Some, None};
 use unstable::finally::Finally;
 use tls = rt::thread_local_storage;
 
+static mut RT_TLS_KEY: tls::Key = -1;
+
 /// Initialize the TLS key. Other ops will fail if this isn't executed first.
 #[fixed_stack_segment]
 #[inline(never)]
 pub fn init_tls_key() {
     unsafe {
-        rust_initialize_rt_tls_key();
+        rust_initialize_rt_tls_key(&mut RT_TLS_KEY);
         extern {
-            fn rust_initialize_rt_tls_key();
+            fn rust_initialize_rt_tls_key(key: *mut tls::Key);
         }
     }
 }
@@ -40,6 +42,7 @@ pub fn init_tls_key() {
 /// # Safety note
 ///
 /// Does not validate the pointer type.
+#[inline]
 pub unsafe fn put<T>(sched: ~T) {
     let key = tls_key();
     let void_ptr: *mut c_void = cast::transmute(sched);
@@ -51,6 +54,7 @@ pub unsafe fn put<T>(sched: ~T) {
 /// # Safety note
 ///
 /// Does not validate the pointer type.
+#[inline]
 pub unsafe fn take<T>() -> ~T {
     let key = tls_key();
     let void_ptr: *mut c_void = tls::get(key);
@@ -62,6 +66,23 @@ pub unsafe fn take<T>() -> ~T {
     return ptr;
 }
 
+/// Take ownership of a pointer from thread-local storage.
+///
+/// # Safety note
+///
+/// Does not validate the pointer type.
+/// Leaves the old pointer in TLS for speed.
+#[inline]
+pub unsafe fn unsafe_take<T>() -> ~T {
+    let key = tls_key();
+    let void_ptr: *mut c_void = tls::get(key);
+    if void_ptr.is_null() {
+        rtabort!("thread-local pointer is null. bogus!");
+    }
+    let ptr: ~T = cast::transmute(void_ptr);
+    return ptr;
+}
+
 /// Check whether there is a thread-local pointer installed.
 pub fn exists() -> bool {
     unsafe {
@@ -99,10 +120,15 @@ pub unsafe fn borrow<T>(f: &fn(&mut T)) {
 /// Because this leaves the value in thread-local storage it is possible
 /// For the Scheduler pointer to be aliased
 pub unsafe fn unsafe_borrow<T>() -> *mut T {
-    match try_unsafe_borrow() {
-        Some(p) => p,
-        None => rtabort!("thread-local pointer is null. bogus!")
+    let key = tls_key();
+    let mut void_ptr: *mut c_void = tls::get(key);
+    if void_ptr.is_null() {
+        rtabort!("thread-local pointer is null. bogus!");
     }
+    let ptr: *mut *mut c_void = &mut void_ptr;
+    let ptr: *mut ~T = ptr as *mut ~T;
+    let ptr: *mut T = &mut **ptr;
+    return ptr;
 }
 
 pub unsafe fn try_unsafe_borrow<T>() -> Option<*mut T> {
@@ -119,6 +145,7 @@ pub unsafe fn try_unsafe_borrow<T>() -> Option<*mut T> {
     }
 }
 
+#[inline]
 fn tls_key() -> tls::Key {
     match maybe_tls_key() {
         Some(key) => key,
@@ -126,15 +153,10 @@ fn tls_key() -> tls::Key {
     }
 }
 
-#[fixed_stack_segment]
-#[inline(never)]
-fn maybe_tls_key() -> Option<tls::Key> {
+#[inline]
+#[cfg(not(test))]
+pub fn maybe_tls_key() -> Option<tls::Key> {
     unsafe {
-        let key: *mut c_void = rust_get_rt_tls_key();
-        let key: &mut tls::Key = cast::transmute(key);
-        let key = *key;
-        // Check that the key has been initialized.
-
         // NB: This is a little racy because, while the key is
         // initalized under a mutex and it's assumed to be initalized
         // in the Scheduler ctor by any thread that needs to use it,
@@ -145,14 +167,19 @@ fn maybe_tls_key() -> Option<tls::Key> {
         // another thread. I think this is fine since the only action
         // they could take if it was initialized would be to check the
         // thread-local value and see that it's not set.
-        if key != -1 {
-            return Some(key);
+        if RT_TLS_KEY != -1 {
+            return Some(RT_TLS_KEY);
         } else {
             return None;
         }
     }
+}
 
-    extern {
-        fn rust_get_rt_tls_key() -> *mut c_void;
-    }
+// XXX: The boundary between the running runtime and the testing runtime
+// seems to be fuzzy at the moment, and trying to use two different keys
+// results in disaster. This should not be necessary.
+#[inline]
+#[cfg(test)]
+pub fn maybe_tls_key() -> Option<tls::Key> {
+    unsafe { ::cast::transmute(::realstd::rt::local_ptr::maybe_tls_key()) }
 }
diff --git a/src/libstd/rt/message_queue.rs b/src/libstd/rt/message_queue.rs
index 8518ddaeae1..2bbcaff6d28 100644
--- a/src/libstd/rt/message_queue.rs
+++ b/src/libstd/rt/message_queue.rs
@@ -16,33 +16,66 @@ use kinds::Send;
 use vec::OwnedVector;
 use cell::Cell;
 use option::*;
-use unstable::sync::Exclusive;
+use unstable::sync::{UnsafeAtomicRcBox, LittleLock};
 use clone::Clone;
 
 pub struct MessageQueue<T> {
-    // XXX: Another mystery bug fixed by boxing this lock
-    priv queue: ~Exclusive<~[T]>
+    priv state: UnsafeAtomicRcBox<State<T>>
+}
+
+struct State<T> {
+    count: uint,
+    queue: ~[T],
+    lock: LittleLock
 }
 
 impl<T: Send> MessageQueue<T> {
     pub fn new() -> MessageQueue<T> {
         MessageQueue {
-            queue: ~Exclusive::new(~[])
+            state: UnsafeAtomicRcBox::new(State {
+                count: 0,
+                queue: ~[],
+                lock: LittleLock::new()
+            })
         }
     }
 
     pub fn push(&mut self, value: T) {
         unsafe {
             let value = Cell::new(value);
-            self.queue.with(|q| q.push(value.take()) );
+            let state = self.state.get();
+            do (*state).lock.lock {
+                (*state).count += 1;
+                (*state).queue.push(value.take());
+            }
         }
     }
 
     pub fn pop(&mut self) -> Option<T> {
         unsafe {
-            do self.queue.with |q| {
-                if !q.is_empty() {
-                    Some(q.shift())
+            let state = self.state.get();
+            do (*state).lock.lock {
+                if !(*state).queue.is_empty() {
+                    (*state).count += 1;
+                    Some((*state).queue.shift())
+                } else {
+                    None
+                }
+            }
+        }
+    }
+
+    /// A pop that may sometimes miss enqueued elements, but is much faster
+    /// to give up without doing any synchronization
+    pub fn casual_pop(&mut self) -> Option<T> {
+        unsafe {
+            let state = self.state.get();
+            // NB: Unsynchronized check
+            if (*state).count == 0 { return None; }
+            do (*state).lock.lock {
+                if !(*state).queue.is_empty() {
+                    (*state).count += 1;
+                    Some((*state).queue.shift())
                 } else {
                     None
                 }
@@ -51,10 +84,10 @@ impl<T: Send> MessageQueue<T> {
     }
 }
 
-impl<T> Clone for MessageQueue<T> {
+impl<T: Send> Clone for MessageQueue<T> {
     fn clone(&self) -> MessageQueue<T> {
         MessageQueue {
-            queue: self.queue.clone()
+            state: self.state.clone()
         }
     }
 }
diff --git a/src/libstd/rt/metrics.rs b/src/libstd/rt/metrics.rs
deleted file mode 100644
index b0c0fa5d708..00000000000
--- a/src/libstd/rt/metrics.rs
+++ /dev/null
@@ -1,98 +0,0 @@
-// Copyright 2013 The Rust Project Developers. See the COPYRIGHT
-// file at the top-level directory of this distribution and at
-// http://rust-lang.org/COPYRIGHT.
-//
-// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
-// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
-// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
-// option. This file may not be copied, modified, or distributed
-// except according to those terms.
-
-use to_str::ToStr;
-
-pub struct SchedMetrics {
-    // The number of times executing `run_sched_once`.
-    turns: uint,
-    // The number of turns that received a message.
-    messages_received: uint,
-    // The number of turns that ran a task from the queue.
-    tasks_resumed_from_queue: uint,
-    // The number of turns that found no work to perform.
-    wasted_turns: uint,
-    // The number of times the scheduler went to sleep.
-    sleepy_times: uint,
-    // Context switches from the scheduler into a task.
-    context_switches_sched_to_task: uint,
-    // Context switches from a task into the scheduler.
-    context_switches_task_to_sched: uint,
-    // Context switches from a task to a task.
-    context_switches_task_to_task: uint,
-    // Message sends that unblock the receiver
-    rendezvous_sends: uint,
-    // Message sends that do not unblock the receiver
-    non_rendezvous_sends: uint,
-    // Message receives that do not block the receiver
-    rendezvous_recvs: uint,
-    // Message receives that block the receiver
-    non_rendezvous_recvs: uint,
-    // JoinLatch releases that create tombstones
-    release_tombstone: uint,
-    // JoinLatch releases that do not create tombstones
-    release_no_tombstone: uint,
-}
-
-impl SchedMetrics {
-    pub fn new() -> SchedMetrics {
-        SchedMetrics {
-            turns: 0,
-            messages_received: 0,
-            tasks_resumed_from_queue: 0,
-            wasted_turns: 0,
-            sleepy_times: 0,
-            context_switches_sched_to_task: 0,
-            context_switches_task_to_sched: 0,
-            context_switches_task_to_task: 0,
-            rendezvous_sends: 0,
-            non_rendezvous_sends: 0,
-            rendezvous_recvs: 0,
-            non_rendezvous_recvs: 0,
-            release_tombstone: 0,
-            release_no_tombstone: 0
-        }
-    }
-}
-
-impl ToStr for SchedMetrics {
-    fn to_str(&self) -> ~str {
-        fmt!("turns: %u\n\
-              messages_received: %u\n\
-              tasks_resumed_from_queue: %u\n\
-              wasted_turns: %u\n\
-              sleepy_times: %u\n\
-              context_switches_sched_to_task: %u\n\
-              context_switches_task_to_sched: %u\n\
-              context_switches_task_to_task: %u\n\
-              rendezvous_sends: %u\n\
-              non_rendezvous_sends: %u\n\
-              rendezvous_recvs: %u\n\
-              non_rendezvous_recvs: %u\n\
-              release_tombstone: %u\n\
-              release_no_tombstone: %u\n\
-              ",
-             self.turns,
-             self.messages_received,
-             self.tasks_resumed_from_queue,
-             self.wasted_turns,
-             self.sleepy_times,
-             self.context_switches_sched_to_task,
-             self.context_switches_task_to_sched,
-             self.context_switches_task_to_task,
-             self.rendezvous_sends,
-             self.non_rendezvous_sends,
-             self.rendezvous_recvs,
-             self.non_rendezvous_recvs,
-             self.release_tombstone,
-             self.release_no_tombstone
-        )
-    }
-}
\ No newline at end of file
diff --git a/src/libstd/rt/mod.rs b/src/libstd/rt/mod.rs
index ead0fb63793..0d59d5780cc 100644
--- a/src/libstd/rt/mod.rs
+++ b/src/libstd/rt/mod.rs
@@ -152,8 +152,6 @@ pub mod local_ptr;
 /// Bindings to pthread/windows thread-local storage.
 pub mod thread_local_storage;
 
-pub mod metrics;
-
 // FIXME #5248 shouldn't be pub
 /// Just stuff
 pub mod util;
diff --git a/src/libstd/rt/sched.rs b/src/libstd/rt/sched.rs
index 5ec2df32c48..7d59627ba39 100644
--- a/src/libstd/rt/sched.rs
+++ b/src/libstd/rt/sched.rs
@@ -24,7 +24,6 @@ use rt::kill::BlockedTask;
 use rt::local_ptr;
 use rt::local::Local;
 use rt::rtio::{RemoteCallback, PausibleIdleCallback};
-use rt::metrics::SchedMetrics;
 use borrow::{to_uint};
 use cell::Cell;
 use rand::{XorShiftRng, RngUtil};
@@ -71,7 +70,6 @@ pub struct Scheduler {
     /// An action performed after a context switch on behalf of the
     /// code running before the context switch
     cleanup_job: Option<CleanupJob>,
-    metrics: SchedMetrics,
     /// Should this scheduler run any task, or only pinned tasks?
     run_anything: bool,
     /// If the scheduler shouldn't run some tasks, a friend to send
@@ -83,6 +81,14 @@ pub struct Scheduler {
     idle_callback: Option<~PausibleIdleCallback>
 }
 
+/// An indication of how hard to work on a given operation, the difference
+/// mainly being whether memory is synchronized or not
+#[deriving(Eq)]
+enum EffortLevel {
+    DontTryTooHard,
+    GiveItYourBest
+}
+
 impl Scheduler {
 
     // * Initialization Functions
@@ -118,7 +124,6 @@ impl Scheduler {
             stack_pool: StackPool::new(),
             sched_task: None,
             cleanup_job: None,
-            metrics: SchedMetrics::new(),
             run_anything: run_anything,
             friend_handle: friend,
             rng: XorShiftRng::new(),
@@ -186,7 +191,7 @@ impl Scheduler {
 
         // Should not have any messages
         let message = stask.sched.get_mut_ref().message_queue.pop();
-        assert!(message.is_none());
+        rtassert!(message.is_none());
 
         stask.destroyed = true;
     }
@@ -237,14 +242,21 @@ impl Scheduler {
 
         // First we check for scheduler messages, these are higher
         // priority than regular tasks.
-        let sched = match sched.interpret_message_queue() {
+        let sched = match sched.interpret_message_queue(DontTryTooHard) {
             Some(sched) => sched,
             None => return
         };
 
         // This helper will use a randomized work-stealing algorithm
         // to find work.
-        let mut sched = match sched.do_work() {
+        let sched = match sched.do_work() {
+            Some(sched) => sched,
+            None => return
+        };
+
+        // Now, before sleeping we need to find out if there really
+        // were any messages. Give it your best!
+        let mut sched = match sched.interpret_message_queue(GiveItYourBest) {
             Some(sched) => sched,
             None => return
         };
@@ -252,10 +264,8 @@ impl Scheduler {
         // If we got here then there was no work to do.
         // Generate a SchedHandle and push it to the sleeper list so
         // somebody can wake us up later.
-        sched.metrics.wasted_turns += 1;
         if !sched.sleepy && !sched.no_sleep {
             rtdebug!("scheduler has no work to do, going to sleep");
-            sched.metrics.sleepy_times += 1;
             sched.sleepy = true;
             let handle = sched.make_handle();
             sched.sleeper_list.push(handle);
@@ -277,10 +287,18 @@ impl Scheduler {
     // returns the still-available scheduler. At this point all
     // message-handling will count as a turn of work, and as a result
     // return None.
-    fn interpret_message_queue(~self) -> Option<~Scheduler> {
+    fn interpret_message_queue(~self, effort: EffortLevel) -> Option<~Scheduler> {
 
         let mut this = self;
-        match this.message_queue.pop() {
+
+        let msg = if effort == DontTryTooHard {
+            // Do a cheap check that may miss messages
+            this.message_queue.casual_pop()
+        } else {
+            this.message_queue.pop()
+        };
+
+        match msg {
             Some(PinnedTask(task)) => {
                 let mut task = task;
                 task.give_home(Sched(this.make_handle()));
@@ -469,10 +487,7 @@ impl Scheduler {
         // We've made work available. Notify a
         // sleeping scheduler.
 
-        // XXX: perf. Check for a sleeper without
-        // synchronizing memory.  It's not critical
-        // that we always find it.
-        match this.sleeper_list.pop() {
+        match this.sleeper_list.casual_pop() {
             Some(handle) => {
                         let mut handle = handle;
                 handle.send(Wake)
@@ -505,7 +520,9 @@ impl Scheduler {
         let mut this = self;
 
         // The current task is grabbed from TLS, not taken as an input.
-        let current_task: ~Task = Local::take::<Task>();
+        // Doing an unsafe_take to avoid writing back a null pointer -
+        // We're going to call `put` later to do that.
+        let current_task: ~Task = unsafe { Local::unsafe_take::<Task>() };
 
         // Check that the task is not in an atomically() section (e.g.,
         // holding a pthread mutex, which could deadlock the scheduler).
@@ -563,11 +580,10 @@ impl Scheduler {
         // run the cleanup job, as expected by the previously called
         // swap_contexts function.
         unsafe {
-            let sched = Local::unsafe_borrow::<Scheduler>();
-            (*sched).run_cleanup_job();
+            let task = Local::unsafe_borrow::<Task>();
+            (*task).sched.get_mut_ref().run_cleanup_job();
 
             // Must happen after running the cleanup job (of course).
-            let task = Local::unsafe_borrow::<Task>();
             (*task).death.check_killed((*task).unwinder.unwinding);
         }
     }
diff --git a/src/libstd/rt/sleeper_list.rs b/src/libstd/rt/sleeper_list.rs
index d327023de97..7232afd6594 100644
--- a/src/libstd/rt/sleeper_list.rs
+++ b/src/libstd/rt/sleeper_list.rs
@@ -15,33 +15,68 @@ use container::Container;
 use vec::OwnedVector;
 use option::{Option, Some, None};
 use cell::Cell;
-use unstable::sync::Exclusive;
+use unstable::sync::{UnsafeAtomicRcBox, LittleLock};
 use rt::sched::SchedHandle;
 use clone::Clone;
 
 pub struct SleeperList {
-    priv stack: ~Exclusive<~[SchedHandle]>
+    priv state: UnsafeAtomicRcBox<State>
+}
+
+struct State {
+    count: uint,
+    stack: ~[SchedHandle],
+    lock: LittleLock
 }
 
 impl SleeperList {
     pub fn new() -> SleeperList {
         SleeperList {
-            stack: ~Exclusive::new(~[])
+            state: UnsafeAtomicRcBox::new(State {
+                count: 0,
+                stack: ~[],
+                lock: LittleLock::new()
+            })
         }
     }
 
     pub fn push(&mut self, handle: SchedHandle) {
         let handle = Cell::new(handle);
         unsafe {
-            self.stack.with(|s| s.push(handle.take()));
+            let state = self.state.get();
+            do (*state).lock.lock {
+                (*state).count += 1;
+                (*state).stack.push(handle.take());
+            }
         }
     }
 
     pub fn pop(&mut self) -> Option<SchedHandle> {
         unsafe {
-            do self.stack.with |s| {
-                if !s.is_empty() {
-                    Some(s.pop())
+            let state = self.state.get();
+            do (*state).lock.lock {
+                if !(*state).stack.is_empty() {
+                    (*state).count -= 1;
+                    Some((*state).stack.pop())
+                } else {
+                    None
+                }
+            }
+        }
+    }
+
+    /// A pop that may sometimes miss enqueued elements, but is much faster
+    /// to give up without doing any synchronization
+    pub fn casual_pop(&mut self) -> Option<SchedHandle> {
+        unsafe {
+            let state = self.state.get();
+            // NB: Unsynchronized check
+            if (*state).count == 0 { return None; }
+            do (*state).lock.lock {
+                if !(*state).stack.is_empty() {
+                    // NB: count is also protected by the lock
+                    (*state).count -= 1;
+                    Some((*state).stack.pop())
                 } else {
                     None
                 }
@@ -53,7 +88,7 @@ impl SleeperList {
 impl Clone for SleeperList {
     fn clone(&self) -> SleeperList {
         SleeperList {
-            stack: self.stack.clone()
+            state: self.state.clone()
         }
     }
-}
\ No newline at end of file
+}
diff --git a/src/libstd/rt/task.rs b/src/libstd/rt/task.rs
index 12ba39a6dcd..9c2a6e646d2 100644
--- a/src/libstd/rt/task.rs
+++ b/src/libstd/rt/task.rs
@@ -59,7 +59,7 @@ pub struct Task {
 }
 
 pub enum TaskType {
-    GreenTask(Option<~SchedHome>),
+    GreenTask(Option<SchedHome>),
     SchedTask
 }
 
@@ -173,7 +173,7 @@ impl Task {
             name: None,
             coroutine: Some(Coroutine::new(stack_pool, stack_size, start)),
             sched: None,
-            task_type: GreenTask(Some(~home)),
+            task_type: GreenTask(Some(home)),
             borrow_list: None
         }
     }
@@ -196,7 +196,7 @@ impl Task {
             name: None,
             coroutine: Some(Coroutine::new(stack_pool, stack_size, start)),
             sched: None,
-            task_type: GreenTask(Some(~home)),
+            task_type: GreenTask(Some(home)),
             borrow_list: None
         }
     }
@@ -204,7 +204,7 @@ impl Task {
     pub fn give_home(&mut self, new_home: SchedHome) {
         match self.task_type {
             GreenTask(ref mut home) => {
-                *home = Some(~new_home);
+                *home = Some(new_home);
             }
             SchedTask => {
                 rtabort!("type error: used SchedTask as GreenTask");
@@ -216,7 +216,7 @@ impl Task {
         match self.task_type {
             GreenTask(ref mut home) => {
                 let out = home.take_unwrap();
-                return *out;
+                return out;
             }
             SchedTask => {
                 rtabort!("type error: used SchedTask as GreenTask");
@@ -275,8 +275,8 @@ impl Task {
 
     pub fn is_home_no_tls(&self, sched: &~Scheduler) -> bool {
         match self.task_type {
-            GreenTask(Some(~AnySched)) => { false }
-            GreenTask(Some(~Sched(SchedHandle { sched_id: ref id, _}))) => {
+            GreenTask(Some(AnySched)) => { false }
+            GreenTask(Some(Sched(SchedHandle { sched_id: ref id, _}))) => {
                 *id == sched.sched_id()
             }
             GreenTask(None) => {
@@ -291,8 +291,8 @@ impl Task {
 
     pub fn homed(&self) -> bool {
         match self.task_type {
-            GreenTask(Some(~AnySched)) => { false }
-            GreenTask(Some(~Sched(SchedHandle { _ }))) => { true }
+            GreenTask(Some(AnySched)) => { false }
+            GreenTask(Some(Sched(SchedHandle { _ }))) => { true }
             GreenTask(None) => {
                 rtabort!("task without home");
             }
@@ -309,11 +309,11 @@ impl Task {
             let sched_id = task.sched.get_ref().sched_id();
             let sched_run_anything = task.sched.get_ref().run_anything;
             match task.task_type {
-                GreenTask(Some(~AnySched)) => {
+                GreenTask(Some(AnySched)) => {
                     rtdebug!("anysched task in sched check ****");
                     sched_run_anything
                 }
-                GreenTask(Some(~Sched(SchedHandle { sched_id: ref id, _ }))) => {
+                GreenTask(Some(Sched(SchedHandle { sched_id: ref id, _ }))) => {
                     rtdebug!("homed task in sched check ****");
                     *id == sched_id
                 }
diff --git a/src/libstd/rt/util.rs b/src/libstd/rt/util.rs
index f2ede8872c2..9113f03ffee 100644
--- a/src/libstd/rt/util.rs
+++ b/src/libstd/rt/util.rs
@@ -19,6 +19,10 @@ use unstable::atomics::{AtomicInt, INIT_ATOMIC_INT, SeqCst};
 #[cfg(target_os="macos")]
 use unstable::running_on_valgrind;
 
+// Indicates whether we should perform expensive sanity checks, including rtassert!
+// XXX: Once the runtime matures remove the `true` below to turn off rtassert, etc.
+pub static ENFORCE_SANITY: bool = true || !cfg!(rtopt) || cfg!(rtdebug) || cfg!(rtassert);
+
 /// Get the number of cores available
 pub fn num_cpus() -> uint {
     #[fixed_stack_segment]; #[inline(never)];
diff --git a/src/libstd/unstable/sync.rs b/src/libstd/unstable/sync.rs
index d7f9988edef..6fa0e0eb8c1 100644
--- a/src/libstd/unstable/sync.rs
+++ b/src/libstd/unstable/sync.rs
@@ -281,20 +281,24 @@ impl<T> Drop for UnsafeAtomicRcBox<T>{
  */
 // FIXME(#8140) should not be pub
 pub unsafe fn atomically<U>(f: &fn() -> U) -> U {
-    use rt::task::Task;
+    use rt::task::{Task, GreenTask, SchedTask};
     use rt::local::Local;
-    use rt::in_green_task_context;
-
-    if in_green_task_context() {
-        let t = Local::unsafe_borrow::<Task>();
-        do (|| {
-            (*t).death.inhibit_deschedule();
-            f()
-        }).finally {
-            (*t).death.allow_deschedule();
+
+    match Local::try_unsafe_borrow::<Task>() {
+        Some(t) => {
+            match (*t).task_type {
+                GreenTask(_) => {
+                    do (|| {
+                        (*t).death.inhibit_deschedule();
+                        f()
+                    }).finally {
+                        (*t).death.allow_deschedule();
+                    }
+                }
+                SchedTask => f()
+            }
         }
-    } else {
-        f()
+        None => f()
     }
 }
 
diff --git a/src/libstd/vec.rs b/src/libstd/vec.rs
index f196cf423c1..10dc1b5409e 100644
--- a/src/libstd/vec.rs
+++ b/src/libstd/vec.rs
@@ -1401,7 +1401,7 @@ impl<T> OwnedVector<T> for ~[T] {
         let self_len = self.len();
         let rhs_len = rhs.len();
         let new_len = self_len + rhs_len;
-        self.reserve(new_len);
+        self.reserve_at_least(new_len);
         unsafe { // Note: infallible.
             let self_p = vec::raw::to_mut_ptr(*self);
             let rhs_p = vec::raw::to_ptr(rhs);