rt: Shutdown gracefully on failure

When the kernel fails, kill all tasks and wait for the schedulers to stop
instead of just exiting. I'm sure there are tons of lurking issues here but
this is enough to fail without leaking (at least in the absence of cleanups).

src/rt/rust_kernel.cpp

@@ -134,6 +134,14 @@ int rust_kernel::start_task_threads()
     return rval;
 }
 
+void
+rust_kernel::fail() {
+    for(size_t i = 0; i < num_threads; ++i) {
+        rust_scheduler *thread = threads[i];
+        thread->kill_all_tasks();
+    }
+}
+
 rust_task_id
 rust_kernel::create_task(rust_task *spawner, const char *name) {
     rust_scheduler *thread = threads[rand(&rctx) % num_threads];

src/rt/rust_kernel.h

@@ -52,6 +52,8 @@ public:
     void *realloc(void *mem, size_t size);
     void free(void *mem);
 
+    void fail();
+
     int start_task_threads();
 
 #ifdef __WIN32__

src/rt/rust_scheduler.cpp

@@ -71,7 +71,21 @@ rust_scheduler::fail() {
         name, this);
     I(this, kernel->rval == 0);
     kernel->rval = 1;
-    exit(1);
+    kernel->fail();
+}
+
+void
+rust_scheduler::kill_all_tasks() {
+    I(this, !lock.lock_held_by_current_thread());
+    scoped_lock with(lock);
+
+    for (size_t i = 0; i < running_tasks.length(); i++) {
+        running_tasks[i]->kill();
+    }
+
+    for (size_t i = 0; i < blocked_tasks.length(); i++) {
+        blocked_tasks[i]->kill();
+    }
 }
 
 size_t

src/rt/rust_scheduler.h

@@ -81,6 +81,8 @@ struct rust_scheduler : public kernel_owned<rust_scheduler>,
 
     void log_state();
 
+    void kill_all_tasks();
+
     rust_task *create_task(rust_task *spawner, const char *name);
 
     virtual void run();

src/rt/rust_upcall.cpp

@@ -195,8 +195,8 @@ upcall_fail(rust_task *task,
             size_t line) {
     LOG_UPCALL_ENTRY(task);
     LOG_ERR(task, upcall, "upcall fail '%s', %s:%" PRIdPTR, expr, file, line);
-    task->fail();
     task->die();
+    task->fail();
     task->notify_tasks_waiting_to_join();
     task->yield(4);
 }