rt: Introduce "end_dp" bailouts in order to avoid marching past the end of oddly aligned vectors

2 files changed, 32 insertions, 19 deletions

diff --git a/src/rt/rust_shape.cpp b/src/rt/rust_shape.cpp
index 90915e549fc..a693684fa89 100644
--- a/src/rt/rust_shape.cpp
+++ b/src/rt/rust_shape.cpp
@@ -280,7 +280,7 @@ private:
     }
 
     inline void cmp_two_pointers() {
-        if (align) dp = align_to(dp, alignof<uint8_t *>() * 2);
+        ALIGN_TO(alignof<void *>() * 2);
         data_pair<uint8_t *> fst = bump_dp<uint8_t *>(dp);
         data_pair<uint8_t *> snd = bump_dp<uint8_t *>(dp);
         cmp_number(fst);
@@ -289,7 +289,7 @@ private:
     }
 
     inline void cmp_pointer() {
-        if (align) dp = align_to(dp, alignof<uint8_t *>());
+        ALIGN_TO(alignof<void *>());
         cmp_number(bump_dp<uint8_t *>(dp));
     }
 
@@ -369,7 +369,7 @@ void cmp::cmp_number<int32_t>(const data_pair<int32_t> &nums) {
 void
 cmp::walk_vec(bool is_pod, const std::pair<ptr_pair,ptr_pair> &data_range) {
     cmp sub(*this, data_range.first);
-    ptr_pair data_end = data_range.second;
+    ptr_pair data_end = sub.end_dp = data_range.second;
     while (!result && sub.dp < data_end) {
         sub.walk_reset();
         result = sub.result;
@@ -467,6 +467,7 @@ log::walk_vec(bool is_pod, const std::pair<ptr,ptr> &data) {
     out << "[";
 
     log sub(*this, data.first);
+    sub.end_dp = data.second;
 
     bool first = true;
     while (sub.dp < data.second) {
diff --git a/src/rt/rust_shape.h b/src/rt/rust_shape.h
index af22d46d337..dacfddba049 100644
--- a/src/rt/rust_shape.h
+++ b/src/rt/rust_shape.h
@@ -622,11 +622,9 @@ public:
     template<typename T>
     struct data { typedef T t; };
 
-    ptr(uint8_t *in_p)
-    : p(in_p) {}
-
-    ptr(uintptr_t in_p)
-    : p((uint8_t *)in_p) {}
+    ptr() : p(NULL) {}
+    ptr(uint8_t *in_p) : p(in_p) {}
+    ptr(uintptr_t in_p) : p((uint8_t *)in_p) {}
 
     inline ptr operator+(const size_t amount) const {
         return make(p + amount);
@@ -639,7 +637,8 @@ public:
     template<typename T>
     inline operator T *() { return (T *)p; }
 
-    inline operator uintptr_t() { return (uintptr_t)p; }
+    inline operator bool() const { return p != NULL; }
+    inline operator uintptr_t() const { return (uintptr_t)p; }
 
     static inline ptr make(uint8_t *in_p) {
         ptr self(in_p);
@@ -687,12 +686,14 @@ public:
     template<typename T>
     struct data { typedef data_pair<T> t; };
 
+    ptr_pair() : fst(NULL), snd(NULL) {}
     ptr_pair(uint8_t *in_fst, uint8_t *in_snd) : fst(in_fst), snd(in_snd) {}
-
     ptr_pair(data_pair<uint8_t *> &other) : fst(other.fst), snd(other.snd) {}
 
     inline void operator=(uint8_t *rhs) { fst = snd = rhs; }
 
+    inline operator bool() const { return fst != NULL && snd != NULL; }
+
     inline ptr_pair operator+(size_t n) const {
         return make(fst + n, snd + n);
     }
@@ -754,15 +755,27 @@ namespace shape {
 // An abstract class (again using the curiously recurring template pattern)
 // for methods that actually manipulate the data involved.
 
+#define ALIGN_TO(alignment) \
+    if (this->align) { \
+        dp = align_to(dp, (alignment)); \
+        if (this->end_dp && !(dp < this->end_dp)) \
+            return; \
+    }
+
 #define DATA_SIMPLE(ty, call) \
-    if (this->align) dp = align_to(dp, alignof<ty>()); \
+    ALIGN_TO(alignof<ty>()); \
     U end_dp = dp + sizeof(ty); \
     static_cast<T *>(this)->call; \
     dp = end_dp;
 
 template<typename T,typename U>
 class data : public ctxt< data<T,U> > {
+public:
+    U dp;
+
 protected:
+    U end_dp;
+
     void walk_box_contents();
     void walk_fn_contents(ptr &dp);
     void walk_obj_contents(ptr &dp);
@@ -774,8 +787,6 @@ protected:
     static std::pair<ptr_pair,ptr_pair> get_vec_data_range(ptr_pair &dp);
 
 public:
-    U dp;
-
     data(rust_task *in_task,
          bool in_align,
          const uint8_t *in_sp,
@@ -783,7 +794,8 @@ public:
          const rust_shape_tables *in_tables,
          U const &in_dp)
     : ctxt< data<T,U> >(in_task, in_align, in_sp, in_params, in_tables),
-      dp(in_dp) {}
+      dp(in_dp),
+      end_dp() {}
 
     void walk_tag(tag_info &tinfo);
 
@@ -801,14 +813,14 @@ public:
     void walk_box()     { DATA_SIMPLE(void *, walk_box()); }
 
     void walk_fn() {
-        if (this->align) dp = align_to(dp, sizeof(void *));
+        ALIGN_TO(alignof<void *>());
         U next_dp = dp + sizeof(void *) * 2;
         static_cast<T *>(this)->walk_fn();
         dp = next_dp;
     }
 
     void walk_obj() {
-        if (this->align) dp = align_to(dp, sizeof(void *));
+        ALIGN_TO(alignof<void *>());
         U next_dp = dp + sizeof(void *) * 2;
         static_cast<T *>(this)->walk_obj();
         dp = next_dp;
@@ -893,8 +905,8 @@ void
 data<T,U>::walk_tag(tag_info &tinfo) {
     size_of::compute_tag_size(*this, tinfo);
 
-    if (tinfo.variant_count > 1 && this->align)
-        dp = align_to(dp, alignof<uint32_t>());
+    if (tinfo.variant_count > 1)
+        ALIGN_TO(alignof<uint32_t>());
 
     U end_dp = dp + tinfo.tag_sa.size;
 
@@ -1029,7 +1041,7 @@ private:
     void walk_subcontext(log &sub) { sub.walk(); }
 
     void walk_box_contents(log &sub, ptr &ref_count_dp) {
-        if (ref_count_dp == 0) {
+        if (!ref_count_dp) {
             out << "(null)";
         } else {
             sub.align = true;