Allocate rust_ivec buffers out of the kernel pool

The duplication of upcalls is due to the fact that the runtime is
shared between stage0/rustc and stage1/rustc. Once snapshots are
updated, they should be de-duplicated.

3 files changed, 138 insertions, 0 deletions

diff --git a/src/rt/rust_builtin.cpp b/src/rt/rust_builtin.cpp
index 6438c549c55..872a317ca38 100644
--- a/src/rt/rust_builtin.cpp
+++ b/src/rt/rust_builtin.cpp
@@ -649,6 +649,37 @@ ivec_reserve(rust_task *task, type_desc *ty, rust_ivec *v, size_t n_elems)
 }
 
 /**
+ * Preallocates the exact number of bytes in the given interior vector.
+ */
+extern "C" CDECL void
+ivec_reserve_shared(rust_task *task, type_desc *ty, rust_ivec *v,
+                    size_t n_elems)
+{
+    size_t new_alloc = n_elems * ty->size;
+    if (new_alloc <= v->alloc)
+        return;     // Already big enough.
+
+    rust_ivec_heap *heap_part;
+    if (v->fill || !v->payload.ptr) {
+        // On stack; spill to heap.
+        heap_part = (rust_ivec_heap *)task->kernel->malloc(new_alloc +
+                                                           sizeof(size_t));
+        heap_part->fill = v->fill;
+        memcpy(&heap_part->data, v->payload.data, v->fill);
+
+        v->fill = 0;
+        v->payload.ptr = heap_part;
+    } else {
+        // On heap; resize.
+        heap_part = (rust_ivec_heap *)task->kernel->realloc(v->payload.ptr,
+                                                new_alloc + sizeof(size_t));
+        v->payload.ptr = heap_part;
+    }
+
+    v->alloc = new_alloc;
+}
+
+/**
  * Returns true if the given vector is on the heap and false if it's on the
  * stack.
  */
@@ -706,6 +737,35 @@ ivec_copy_from_buf(rust_task *task, type_desc *ty, rust_ivec *v, void *ptr,
     v->payload.ptr->fill = new_size;
 }
 
+/**
+ * Copies elements in an unsafe buffer to the given interior vector. The
+ * vector must have size zero.
+ */
+extern "C" CDECL void
+ivec_copy_from_buf_shared(rust_task *task, type_desc *ty, rust_ivec *v,
+                   void *ptr, size_t count)
+{
+    size_t old_size = get_ivec_size(v);
+    if (old_size) {
+        task->fail(1);
+        return;
+    }
+
+    ivec_reserve_shared(task, ty, v, count);
+
+    size_t new_size = count * ty->size;
+    if (v->fill || !v->payload.ptr) {
+        // On stack.
+        memmove(v->payload.data, ptr, new_size);
+        v->fill = new_size;
+        return;
+    }
+
+    // On heap.
+    memmove(v->payload.ptr->data, ptr, new_size);
+    v->payload.ptr->fill = new_size;
+}
+
 extern "C" CDECL void
 pin_task(rust_task *task) {
     task->pin();
diff --git a/src/rt/rust_upcall.cpp b/src/rt/rust_upcall.cpp
index d8ff9236aa6..a89c01ebdfb 100644
--- a/src/rt/rust_upcall.cpp
+++ b/src/rt/rust_upcall.cpp
@@ -290,6 +290,36 @@ upcall_free(rust_task *task, void* ptr, uintptr_t is_gc) {
 }
 
 extern "C" CDECL uintptr_t
+upcall_shared_malloc(rust_task *task, size_t nbytes, type_desc *td) {
+    LOG_UPCALL_ENTRY(task);
+    scoped_lock with(task->kernel->scheduler_lock);
+
+    LOG(task, mem,
+                   "upcall shared_malloc(%" PRIdPTR ", 0x%" PRIxPTR ")",
+                   nbytes, td);
+    void *p = task->kernel->malloc(nbytes);
+    LOG(task, mem,
+                   "upcall shared_malloc(%" PRIdPTR ", 0x%" PRIxPTR
+                   ") = 0x%" PRIxPTR,
+                   nbytes, td, (uintptr_t)p);
+    return (uintptr_t) p;
+}
+
+/**
+ * Called whenever an object's ref count drops to zero.
+ */
+extern "C" CDECL void
+upcall_shared_free(rust_task *task, void* ptr) {
+    LOG_UPCALL_ENTRY(task);
+    scoped_lock with(task->kernel->scheduler_lock);
+    rust_scheduler *sched = task->sched;
+    DLOG(sched, mem,
+             "upcall shared_free(0x%" PRIxPTR")",
+             (uintptr_t)ptr);
+    task->kernel->free(ptr);
+}
+
+extern "C" CDECL uintptr_t
 upcall_mark(rust_task *task, void* ptr) {
     LOG_UPCALL_ENTRY(task);
     scoped_lock with(task->kernel->scheduler_lock);
@@ -537,6 +567,7 @@ extern "C" CDECL void
 upcall_ivec_resize(rust_task *task,
                    rust_ivec *v,
                    size_t newsz) {
+    LOG_UPCALL_ENTRY(task);
     scoped_lock with(task->kernel->scheduler_lock);
     I(task->sched, !v->fill);
 
@@ -556,6 +587,7 @@ extern "C" CDECL void
 upcall_ivec_spill(rust_task *task,
                   rust_ivec *v,
                   size_t newsz) {
+    LOG_UPCALL_ENTRY(task);
     scoped_lock with(task->kernel->scheduler_lock);
     size_t new_alloc = next_power_of_two(newsz);
 
@@ -569,6 +601,46 @@ upcall_ivec_spill(rust_task *task,
     v->payload.ptr = heap_part;
 }
 
+/**
+ * Resizes an interior vector that has been spilled to the heap.
+ */
+extern "C" CDECL void
+upcall_ivec_resize_shared(rust_task *task,
+                          rust_ivec *v,
+                          size_t newsz) {
+    LOG_UPCALL_ENTRY(task);
+    scoped_lock with(task->kernel->scheduler_lock);
+    I(task->sched, !v->fill);
+
+    size_t new_alloc = next_power_of_two(newsz);
+    rust_ivec_heap *new_heap_part = (rust_ivec_heap *)
+        task->kernel->realloc(v->payload.ptr, new_alloc + sizeof(size_t));
+
+    new_heap_part->fill = newsz;
+    v->alloc = new_alloc;
+    v->payload.ptr = new_heap_part;
+}
+
+/**
+ * Spills an interior vector to the heap.
+ */
+extern "C" CDECL void
+upcall_ivec_spill_shared(rust_task *task,
+                         rust_ivec *v,
+                         size_t newsz) {
+    LOG_UPCALL_ENTRY(task);
+    scoped_lock with(task->kernel->scheduler_lock);
+    size_t new_alloc = next_power_of_two(newsz);
+
+    rust_ivec_heap *heap_part = (rust_ivec_heap *)
+        task->kernel->malloc(new_alloc + sizeof(size_t));
+    heap_part->fill = newsz;
+    memcpy(&heap_part->data, v->payload.data, v->fill);
+
+    v->fill = 0;
+    v->alloc = new_alloc;
+    v->payload.ptr = heap_part;
+}
 //
 // Local Variables:
 // mode: C++
diff --git a/src/rt/rustrt.def.in b/src/rt/rustrt.def.in
index 444faccef26..481936eee69 100644
--- a/src/rt/rustrt.def.in
+++ b/src/rt/rustrt.def.in
@@ -11,8 +11,10 @@ debug_tydesc
 do_gc
 get_time
 ivec_copy_from_buf
+ivec_copy_from_buf_shared
 ivec_on_heap
 ivec_reserve
+ivec_reserve_shared
 ivec_to_ptr
 last_os_error
 nano_time
@@ -59,7 +61,9 @@ upcall_free
 upcall_get_type_desc
 upcall_grow_task
 upcall_ivec_resize
+upcall_ivec_resize_shared
 upcall_ivec_spill
+upcall_ivec_spill_shared
 upcall_kill
 upcall_log_double
 upcall_log_float
@@ -74,6 +78,8 @@ upcall_new_task
 upcall_new_vec
 upcall_recv
 upcall_send
+upcall_shared_malloc
+upcall_shared_free
 upcall_sleep
 upcall_start_task
 upcall_trace_str