Lots of changes around memory managment in the Runtime. Added memory regions and fixed race caused by calling rust_srv::malloc() from multiple threads when sending messages.

12 files changed, 298 insertions, 124 deletions

diff --git a/src/rt/memory.h b/src/rt/memory.h
new file mode 100644
index 00000000000..22bc15d32b6
--- /dev/null
+++ b/src/rt/memory.h
@@ -0,0 +1,60 @@
+/*
+ *
+ */
+
+#ifndef MEMORY_H
+#define MEMORY_H
+
+
+inline void *operator new(size_t size, void *mem) {
+    return mem;
+}
+
+inline void *operator new(size_t size, rust_dom *dom) {
+    return dom->malloc(size, memory_region::LOCAL);
+}
+
+inline void *operator new[](size_t size, rust_dom *dom) {
+    return dom->malloc(size, memory_region::LOCAL);
+}
+
+inline void *operator new(size_t size, rust_dom &dom) {
+    return dom.malloc(size, memory_region::LOCAL);
+}
+
+inline void *operator new[](size_t size, rust_dom &dom) {
+    return dom.malloc(size, memory_region::LOCAL);
+}
+
+inline void *operator new(size_t size, rust_dom *dom,
+    memory_region::memory_region_type type) {
+    return dom->malloc(size, type);
+}
+
+inline void *operator new[](size_t size, rust_dom *dom,
+    memory_region::memory_region_type type) {
+    return dom->malloc(size, type);
+}
+
+inline void *operator new(size_t size, rust_dom &dom,
+    memory_region::memory_region_type type) {
+    return dom.malloc(size, type);
+}
+
+inline void *operator new[](size_t size, rust_dom &dom,
+    memory_region::memory_region_type type) {
+    return dom.malloc(size, type);
+}
+
+inline void operator delete(void *mem, rust_dom *dom) {
+    dom->free(mem, memory_region::LOCAL);
+    return;
+}
+
+inline void operator delete(void *mem, rust_dom *dom,
+    memory_region::memory_region_type type) {
+    dom->free(mem, type);
+    return;
+}
+
+#endif /* MEMORY_H */
diff --git a/src/rt/memory_region.cpp b/src/rt/memory_region.cpp
new file mode 100644
index 00000000000..797a7c1d669
--- /dev/null
+++ b/src/rt/memory_region.cpp
@@ -0,0 +1,100 @@
+/*
+ *
+ */
+
+#include "rust_internal.h"
+#include "memory_region.h"
+
+#define TRACK_ALLOCATIONS
+
+memory_region::memory_region(rust_srv *srv, bool synchronized) :
+    _srv(srv), _parent(NULL), _live_allocations(0),
+    _synchronized(synchronized) {
+    // Nop.
+}
+
+memory_region::memory_region(memory_region *parent) :
+    _srv(parent->_srv), _parent(parent), _live_allocations(0),
+    _synchronized(parent->_synchronized) {
+    // Nop.
+}
+
+void memory_region::free(void *mem) {
+    if (_synchronized) { _lock.lock(); }
+#ifdef TRACK_ALLOCATIONS
+    if (_allocation_list.replace(mem, NULL) == false) {
+        printf("free: ptr 0x%" PRIxPTR " is not in allocation_list\n",
+            (uintptr_t) mem);
+        _srv->fatal("not in allocation_list", __FILE__, __LINE__, "");
+    }
+#endif
+    if (_live_allocations < 1) {
+        _srv->fatal("live_allocs < 1", __FILE__, __LINE__, "");
+    }
+    _live_allocations--;
+    _srv->free(mem);
+    if (_synchronized) { _lock.unlock(); }
+
+}
+
+void *
+memory_region::realloc(void *mem, size_t size) {
+    if (_synchronized) { _lock.lock(); }
+    if (!mem) {
+        _live_allocations++;
+    }
+    void *newMem = _srv->realloc(mem, size);
+#ifdef TRACK_ALLOCATIONS
+    if (_allocation_list.replace(mem, newMem) == false) {
+        printf("realloc: ptr 0x%" PRIxPTR " is not in allocation_list\n",
+            (uintptr_t) mem);
+        _srv->fatal("not in allocation_list", __FILE__, __LINE__, "");
+    }
+#endif
+    if (_synchronized) { _lock.unlock(); }
+    return newMem;
+}
+
+void *
+memory_region::malloc(size_t size) {
+    if (_synchronized) { _lock.lock(); }
+    _live_allocations++;
+    void *mem = _srv->malloc(size);
+#ifdef TRACK_ALLOCATIONS
+    _allocation_list.append(mem);
+#endif
+    if (_synchronized) { _lock.unlock(); }
+    return mem;
+}
+
+void *
+memory_region::calloc(size_t size) {
+    if (_synchronized) { _lock.lock(); }
+    _live_allocations++;
+    void *mem = _srv->malloc(size);
+    memset(mem, 0, size);
+#ifdef TRACK_ALLOCATIONS
+    _allocation_list.append(mem);
+#endif
+    if (_synchronized) { _lock.unlock(); }
+    return mem;
+}
+
+memory_region::~memory_region() {
+    if (_live_allocations == 0) {
+        return;
+    }
+    char msg[128];
+    snprintf(msg, sizeof(msg),
+        "leaked memory in rust main loop (%" PRIuPTR " objects)",
+        _live_allocations);
+#ifdef TRACK_ALLOCATIONS
+    for (size_t i = 0; i < _allocation_list.size(); i++) {
+        if (_allocation_list[i] != NULL) {
+            printf("allocation 0x%" PRIxPTR " was not freed\n",
+                (uintptr_t) _allocation_list[i]);
+        }
+    }
+#endif
+    _srv->fatal(msg, __FILE__, __LINE__, "%d objects", _live_allocations);
+}
diff --git a/src/rt/memory_region.h b/src/rt/memory_region.h
new file mode 100644
index 00000000000..3411d867ebf
--- /dev/null
+++ b/src/rt/memory_region.h
@@ -0,0 +1,37 @@
+/*
+ * The Rust runtime uses memory regions to provide a primitive level of
+ * memory management and isolation between tasks, and domains.
+ *
+ * TODO: Implement a custom lock-free malloc / free instead of relying solely
+ *       on the standard malloc / free.
+ */
+
+#ifndef MEMORY_REGION_H
+#define MEMORY_REGION_H
+
+#include "sync/spin_lock.h"
+
+class rust_srv;
+
+class memory_region {
+private:
+    rust_srv *_srv;
+    memory_region *_parent;
+    size_t _live_allocations;
+    array_list<void *> _allocation_list;
+    const bool _synchronized;
+    spin_lock _lock;
+public:
+    enum memory_region_type {
+        LOCAL = 0x1, SYNCHRONIZED = 0x2
+    };
+    memory_region(rust_srv *srv, bool synchronized);
+    memory_region(memory_region *parent);
+    void *malloc(size_t size);
+    void *calloc(size_t size);
+    void *realloc(void *mem, size_t size);
+    void free(void *mem);
+    virtual ~memory_region();
+};
+
+#endif /* MEMORY_REGION_H */
diff --git a/src/rt/rust.h b/src/rt/rust.h
index 97f5c0d448d..3c534c9444b 100644
--- a/src/rt/rust.h
+++ b/src/rt/rust.h
@@ -21,11 +21,6 @@
 
 #include "rust_srv.h"
 
-inline void *operator new(size_t size, rust_srv *srv)
-{
-    return srv->malloc(size);
-}
-
 /*
  * Local Variables:
  * fill-column: 78;
diff --git a/src/rt/rust_builtin.cpp b/src/rt/rust_builtin.cpp
index 6b1f7d198c4..276ea5582d0 100644
--- a/src/rt/rust_builtin.cpp
+++ b/src/rt/rust_builtin.cpp
@@ -38,7 +38,7 @@ last_os_error(rust_task *task) {
 #endif
     size_t fill = strlen(buf) + 1;
     size_t alloc = next_power_of_two(sizeof(rust_str) + fill);
-    void *mem = dom->malloc(alloc);
+    void *mem = dom->malloc(alloc, memory_region::LOCAL);
     if (!mem) {
         task->fail(1);
         return NULL;
@@ -134,7 +134,7 @@ str_alloc_with_data(rust_task *task,
 {
     rust_dom *dom = task->dom;
     size_t alloc = next_power_of_two(sizeof(rust_str) + n_bytes);
-    void *mem = dom->malloc(alloc);
+    void *mem = dom->malloc(alloc, memory_region::LOCAL);
     if (!mem)
         return NULL;
     rust_str *st = new (mem) rust_str(dom, alloc, fill, d);
diff --git a/src/rt/rust_dom.cpp b/src/rt/rust_dom.cpp
index c41e26881fc..67a1be3644d 100644
--- a/src/rt/rust_dom.cpp
+++ b/src/rt/rust_dom.cpp
@@ -13,6 +13,8 @@ rust_dom::rust_dom(rust_srv *srv, rust_crate const *root_crate,
     root_crate(root_crate),
     _log(srv, this),
     srv(srv),
+    local_region(&srv->local_region),
+    synchronized_region(&srv->synchronized_region),
     name(name),
     running_tasks(this),
     blocked_tasks(this),
@@ -144,36 +146,65 @@ rust_dom::fail() {
 }
 
 void *
-rust_dom::malloc(size_t sz) {
-    void *p = srv->malloc(sz);
-    I(this, p);
-    log(rust_log::MEM,
-        "%s @0x%" PRIxPTR " rust_dom::malloc(%d) -> 0x%" PRIxPTR,
-        name, (uintptr_t) this, sz, p);
-    return p;
+rust_dom::malloc(size_t size) {
+    return malloc(size, memory_region::LOCAL);
 }
 
 void *
-rust_dom::calloc(size_t sz) {
-    void *p = this->malloc(sz);
-    memset(p, 0, sz);
-    return p;
+rust_dom::malloc(size_t size, memory_region::memory_region_type type) {
+    if (type == memory_region::LOCAL) {
+        return local_region.malloc(size);
+    } else if (type == memory_region::SYNCHRONIZED) {
+        return synchronized_region.malloc(size);
+    }
+    return NULL;
+}
+
+void *
+rust_dom::calloc(size_t size) {
+    return calloc(size, memory_region::LOCAL);
+}
+
+void *
+rust_dom::calloc(size_t size, memory_region::memory_region_type type) {
+    if (type == memory_region::LOCAL) {
+        return local_region.calloc(size);
+    } else if (type == memory_region::SYNCHRONIZED) {
+        return synchronized_region.calloc(size);
+    }
+    return NULL;
 }
 
 void *
-rust_dom::realloc(void *p, size_t sz) {
-    void *p1 = srv->realloc(p, sz);
-    I(this, p1);
-    log(rust_log::MEM, "rust_dom::realloc(0x%" PRIxPTR ", %d) -> 0x%" PRIxPTR,
-        p, sz, p1);
-    return p1;
+rust_dom::realloc(void *mem, size_t size) {
+    return realloc(mem, size, memory_region::LOCAL);
+}
+
+void *
+rust_dom::realloc(void *mem, size_t size,
+    memory_region::memory_region_type type) {
+    if (type == memory_region::LOCAL) {
+        return local_region.realloc(mem, size);
+    } else if (type == memory_region::SYNCHRONIZED) {
+        return synchronized_region.realloc(mem, size);
+    }
+    return NULL;
 }
 
 void
-rust_dom::free(void *p) {
-    log(rust_log::MEM, "rust_dom::free(0x%" PRIxPTR ")", p);
-    I(this, p);
-    srv->free(p);
+rust_dom::free(void *mem) {
+    free(mem, memory_region::LOCAL);
+}
+
+void
+rust_dom::free(void *mem, memory_region::memory_region_type type) {
+    log(rust_log::MEM, "rust_dom::free(0x%" PRIxPTR ")", mem);
+    if (type == memory_region::LOCAL) {
+        local_region.free(mem);
+    } else if (type == memory_region::SYNCHRONIZED) {
+        synchronized_region.free(mem);
+    }
+    return;
 }
 
 #ifdef __WIN32__
@@ -264,7 +295,6 @@ void rust_dom::send_message(rust_message *message) {
                         message,
                         &_incoming_message_queue,
                         this);
-    A(this, message->dom == this, "Message owned by non-local domain.");
     _incoming_message_queue.enqueue(message);
 }
 
@@ -277,7 +307,8 @@ void rust_dom::drain_incoming_message_queue() {
         log(rust_log::COMM, "<== processing incoming message \"%s\" 0x%"
             PRIxPTR, message->label, message);
         message->process();
-        delete message;
+        message->~rust_message();
+        this->synchronized_region.free(message);
     }
 }
 
@@ -322,8 +353,7 @@ rust_dom::get_port_proxy_synchronized(rust_port *port) {
  * Returns NULL if no tasks can be scheduled.
  */
 rust_task *
-rust_dom::schedule_task()
-{
+rust_dom::schedule_task() {
     I(this, this);
     // FIXME: in the face of failing tasks, this is not always right.
     // I(this, n_live_tasks() > 0);
diff --git a/src/rt/rust_dom.h b/src/rt/rust_dom.h
index 3aaf635a622..d2b6e57033b 100644
--- a/src/rt/rust_dom.h
+++ b/src/rt/rust_dom.h
@@ -25,6 +25,8 @@ struct rust_dom
     rust_crate const *root_crate;
     rust_log _log;
     rust_srv *srv;
+    memory_region local_region;
+    memory_region synchronized_region;
     const char *const name;
     ptr_vec<rust_task> running_tasks;
     ptr_vec<rust_task> blocked_tasks;
@@ -58,10 +60,15 @@ struct rust_dom
     template<typename T>
     void logptr(char const *msg, T* ptrval);
     void fail();
-    void *malloc(size_t sz);
-    void *calloc(size_t sz);
-    void *realloc(void *data, size_t sz);
-    void free(void *p);
+    void *malloc(size_t size);
+    void *malloc(size_t size, memory_region::memory_region_type type);
+    void *calloc(size_t size);
+    void *calloc(size_t size, memory_region::memory_region_type type);
+    void *realloc(void *mem, size_t size);
+    void *realloc(void *mem, size_t size,
+        memory_region::memory_region_type type);
+    void free(void *mem);
+    void free(void *mem, memory_region::memory_region_type type);
 
     void send_message(rust_message *message);
     void drain_incoming_message_queue();
diff --git a/src/rt/rust_internal.h b/src/rt/rust_internal.h
index 67787719444..d6618873dff 100644
--- a/src/rt/rust_internal.h
+++ b/src/rt/rust_internal.h
@@ -173,25 +173,7 @@ check_null(rust_dom *dom, T value, char const *expr,
 
 #define CHECK_NULL(dom, e) (check_null(dom, e, #e, __FILE__, __LINE__))
 
-inline void *operator new(size_t sz, void *mem) {
-    return mem;
-}
-
-inline void *operator new(size_t sz, rust_dom *dom) {
-    return dom->malloc(sz);
-}
-
-inline void *operator new[](size_t sz, rust_dom *dom) {
-    return dom->malloc(sz);
-}
-
-inline void *operator new(size_t sz, rust_dom &dom) {
-    return dom.malloc(sz);
-}
-
-inline void *operator new[](size_t sz, rust_dom &dom) {
-    return dom.malloc(sz);
-}
+#include "memory.h"
 
 struct
 rust_timer
diff --git a/src/rt/rust_message.cpp b/src/rt/rust_message.cpp
index 1de804c9dec..b6b7fbf07e0 100644
--- a/src/rt/rust_message.cpp
+++ b/src/rt/rust_message.cpp
@@ -3,7 +3,8 @@
 
 rust_message::
 rust_message(const char* label, rust_task *source, rust_task *target) :
-             dom(target->dom), label(label),
+             label(label),
+             _dom(target->dom),
              _source(source),
              _target(target) {
 }
@@ -12,12 +13,12 @@ rust_message::~rust_message() {
 }
 
 void rust_message::process() {
-    I(dom, false);
+    I(_dom, false);
 }
 
 rust_proxy<rust_task> *
 rust_message::get_source_proxy() {
-    return dom->get_task_proxy(_source);
+    return _dom->get_task_proxy(_source);
 }
 
 notify_message::
@@ -50,8 +51,9 @@ send(notification_type type, const char* label, rust_task *source,
      rust_proxy<rust_task> *target) {
     rust_task *target_task = target->delegate();
     rust_dom *target_domain = target_task->dom;
-    notify_message *message = new (target_domain)
-        notify_message(type, label, source, target_task);
+    notify_message *message =
+        new (target_domain, memory_region::SYNCHRONIZED) notify_message(type,
+            label, source, target_task);
     target_domain->send_message(message);
 }
 
@@ -83,9 +85,10 @@ send(uint8_t *buffer, size_t buffer_sz, const char* label, rust_task *source,
     rust_task *target_task = target->delegate();
     rust_port *target_port = port->delegate();
     rust_dom *target_domain = target_task->dom;
-    data_message *message = new (target_domain)
-        data_message(buffer, buffer_sz, label, source,
-                     target_task, target_port);
+    data_message *message =
+        new (target_domain, memory_region::SYNCHRONIZED)
+            data_message(buffer, buffer_sz, label, source,
+                target_task, target_port);
     target_domain->send_message(message);
 }
 
diff --git a/src/rt/rust_message.h b/src/rt/rust_message.h
index b7b8568a628..6d986acf5d2 100644
--- a/src/rt/rust_message.h
+++ b/src/rt/rust_message.h
@@ -9,12 +9,11 @@
 /**
  * Abstract base class for all message types.
  */
-class rust_message : public lock_free_queue_node,
-                     public dom_owned<rust_message> {
+class rust_message : public lock_free_queue_node {
 public:
-    rust_dom *dom;
     const char* label;
 private:
+    rust_dom *_dom;
     rust_task *_source;
 protected:
     rust_task *_target;
@@ -70,7 +69,7 @@ public:
 
     data_message(uint8_t *buffer, size_t buffer_sz, const char* label,
                  rust_task *source, rust_task *target, rust_port *port);
-    ~data_message();
+    virtual ~data_message();
     void process();
 
     /**
diff --git a/src/rt/rust_srv.cpp b/src/rt/rust_srv.cpp
index 9239f643ebe..f2dfef63ab6 100644
--- a/src/rt/rust_srv.cpp
+++ b/src/rt/rust_srv.cpp
@@ -5,70 +5,29 @@
 #include "rust_internal.h"
 #include "rust_srv.h"
 
-#define TRACK_ALLOCATIONS
-
-rust_srv::rust_srv() : _live_allocations(0) {
+rust_srv::rust_srv() :
+    local_region(this, false),
+    synchronized_region(this, true) {
     // Nop.
 }
 
 rust_srv::~rust_srv() {
-    if (_live_allocations != 0) {
-        char msg[128];
-        snprintf(msg, sizeof(msg),
-                 "leaked memory in rust main loop (%" PRIuPTR " objects)",
-                 _live_allocations);
-#ifdef TRACK_ALLOCATIONS
-        for (size_t i = 0; i < _allocation_list.size(); i++) {
-            if (_allocation_list[i] != NULL) {
-                printf("allocation 0x%" PRIxPTR " was not freed\n",
-                        (uintptr_t) _allocation_list[i]);
-            }
-        }
-#endif
-        fatal(msg, __FILE__, __LINE__, "");
-    }
+    // Nop.
+}
+
+void
+rust_srv::free(void *p) {
+    ::free(p);
 }
 
 void *
 rust_srv::malloc(size_t bytes) {
-    ++_live_allocations;
-    void * val = ::malloc(bytes);
-#ifdef TRACK_ALLOCATIONS
-    _allocation_list.append(val);
-#endif
-    return val;
+    return ::malloc(bytes);
 }
 
 void *
 rust_srv::realloc(void *p, size_t bytes) {
-    if (!p) {
-        _live_allocations++;
-    }
-    void * val = ::realloc(p, bytes);
-#ifdef TRACK_ALLOCATIONS
-    if (_allocation_list.replace(p, val) == false) {
-        printf("realloc: ptr 0x%" PRIxPTR " is not in allocation_list\n",
-               (uintptr_t) p);
-        fatal("not in allocation_list", __FILE__, __LINE__, "");
-    }
-#endif
-    return val;
-}
-
-void
-rust_srv::free(void *p) {
-#ifdef TRACK_ALLOCATIONS
-    if (_allocation_list.replace(p, NULL) == false) {
-        printf("free: ptr 0x%" PRIxPTR " is not in allocation_list\n",
-               (uintptr_t) p);
-        fatal("not in allocation_list", __FILE__, __LINE__, "");
-    }
-#endif
-    if (_live_allocations < 1) {
-        fatal("live_allocs < 1", __FILE__, __LINE__, "");
-    }
-    _live_allocations--;
-    ::free(p);
+    return ::realloc(p, bytes);
 }
 
 void
diff --git a/src/rt/rust_srv.h b/src/rt/rust_srv.h
index b25e5e75f4e..e617c002576 100644
--- a/src/rt/rust_srv.h
+++ b/src/rt/rust_srv.h
@@ -5,11 +5,13 @@
 #ifndef RUST_SRV_H
 #define RUST_SRV_H
 
+#include "sync/spin_lock.h"
+#include "memory_region.h"
+
 class rust_srv {
-private:
-    size_t _live_allocations;
-    array_list<void *> _allocation_list;
 public:
+    memory_region local_region;
+    memory_region synchronized_region;
     virtual void log(char const *msg);
     virtual void fatal(char const *expression,
         char const *file,
@@ -21,12 +23,12 @@ public:
         size_t line,
         char const *format,
         ...);
+    virtual void free(void *);
     virtual void *malloc(size_t);
     virtual void *realloc(void *, size_t);
-    virtual void free(void *);
-    virtual rust_srv *clone();
     rust_srv();
     virtual ~rust_srv();
+    virtual rust_srv *clone();
 };
 
 #endif /* RUST_SRV_H */