rt: Shutdown gracefully on failure

When the kernel fails, kill all tasks and wait for the schedulers to stop
instead of just exiting. I'm sure there are tons of lurking issues here but
this is enough to fail without leaking (at least in the absence of cleanups).

5 files changed, 28 insertions, 2 deletions

diff --git a/src/rt/rust_kernel.cpp b/src/rt/rust_kernel.cpp
index 4197f5dea89..89c06885683 100644
--- a/src/rt/rust_kernel.cpp
+++ b/src/rt/rust_kernel.cpp
@@ -134,6 +134,14 @@ int rust_kernel::start_task_threads()
     return rval;
 }
 
+void
+rust_kernel::fail() {
+    for(size_t i = 0; i < num_threads; ++i) {
+        rust_scheduler *thread = threads[i];
+        thread->kill_all_tasks();
+    }
+}
+
 rust_task_id
 rust_kernel::create_task(rust_task *spawner, const char *name) {
     rust_scheduler *thread = threads[rand(&rctx) % num_threads];
diff --git a/src/rt/rust_kernel.h b/src/rt/rust_kernel.h
index ff5b100445e..774f7f4d0a4 100644
--- a/src/rt/rust_kernel.h
+++ b/src/rt/rust_kernel.h
@@ -52,6 +52,8 @@ public:
     void *realloc(void *mem, size_t size);
     void free(void *mem);
 
+    void fail();
+
     int start_task_threads();
 
 #ifdef __WIN32__
diff --git a/src/rt/rust_scheduler.cpp b/src/rt/rust_scheduler.cpp
index 30504319b30..33e607a85e1 100644
--- a/src/rt/rust_scheduler.cpp
+++ b/src/rt/rust_scheduler.cpp
@@ -71,7 +71,21 @@ rust_scheduler::fail() {
         name, this);
     I(this, kernel->rval == 0);
     kernel->rval = 1;
-    exit(1);
+    kernel->fail();
+}
+
+void
+rust_scheduler::kill_all_tasks() {
+    I(this, !lock.lock_held_by_current_thread());
+    scoped_lock with(lock);
+
+    for (size_t i = 0; i < running_tasks.length(); i++) {
+        running_tasks[i]->kill();
+    }
+
+    for (size_t i = 0; i < blocked_tasks.length(); i++) {
+        blocked_tasks[i]->kill();
+    }
 }
 
 size_t
diff --git a/src/rt/rust_scheduler.h b/src/rt/rust_scheduler.h
index 28669169512..b4a70e51f68 100644
--- a/src/rt/rust_scheduler.h
+++ b/src/rt/rust_scheduler.h
@@ -81,6 +81,8 @@ struct rust_scheduler : public kernel_owned<rust_scheduler>,
 
     void log_state();
 
+    void kill_all_tasks();
+
     rust_task *create_task(rust_task *spawner, const char *name);
 
     virtual void run();
diff --git a/src/rt/rust_upcall.cpp b/src/rt/rust_upcall.cpp
index 5feffe600c3..cd8d9b96e2e 100644
--- a/src/rt/rust_upcall.cpp
+++ b/src/rt/rust_upcall.cpp
@@ -195,8 +195,8 @@ upcall_fail(rust_task *task,
             size_t line) {
     LOG_UPCALL_ENTRY(task);
     LOG_ERR(task, upcall, "upcall fail '%s', %s:%" PRIdPTR, expr, file, line);
-    task->fail();
     task->die();
+    task->fail();
     task->notify_tasks_waiting_to_join();
     task->yield(4);
 }