add serialization library; convert ebml lib to use u64 internally

3 files changed, 168 insertions, 122 deletions

diff --git a/src/libstd/ebml.rs b/src/libstd/ebml.rs
index 41cec77851d..45faf62cf7d 100644
--- a/src/libstd/ebml.rs
+++ b/src/libstd/ebml.rs
@@ -19,35 +19,42 @@ type doc = {data: @[u8], start: uint, end: uint};
 
 type tagged_doc = {tag: uint, doc: doc};
 
-fn vint_at(data: [u8], start: uint) -> {val: uint, next: uint} {
+fn vu64_at(data: [u8], start: uint) -> {val: u64, next: uint} {
     let a = data[start];
     if a & 0x80u8 != 0u8 {
-        ret {val: (a & 0x7fu8) as uint, next: start + 1u};
+        ret {val: (a & 0x7fu8) as u64, next: start + 1u};
     }
     if a & 0x40u8 != 0u8 {
-        ret {val: ((a & 0x3fu8) as uint) << 8u | (data[start + 1u] as uint),
+        ret {val: ((a & 0x3fu8) as u64) << 8u64 |
+                 (data[start + 1u] as u64),
              next: start + 2u};
     } else if a & 0x20u8 != 0u8 {
-        ret {val: ((a & 0x1fu8) as uint) << 16u |
-                 (data[start + 1u] as uint) << 8u |
-                 (data[start + 2u] as uint),
+        ret {val: ((a & 0x1fu8) as u64) << 16u64 |
+                 (data[start + 1u] as u64) << 8u64 |
+                 (data[start + 2u] as u64),
              next: start + 3u};
     } else if a & 0x10u8 != 0u8 {
-        ret {val: ((a & 0x0fu8) as uint) << 24u |
-                 (data[start + 1u] as uint) << 16u |
-                 (data[start + 2u] as uint) << 8u |
-                 (data[start + 3u] as uint),
+        ret {val: ((a & 0x0fu8) as u64) << 24u64 |
+                 (data[start + 1u] as u64) << 16u64 |
+                 (data[start + 2u] as u64) << 8u64 |
+                 (data[start + 3u] as u64),
              next: start + 4u};
     } else { #error("vint too big"); fail; }
 }
 
+fn vuint_at(data: [u8], start: uint) -> {val: uint, next: uint} {
+    let {val, next} = vu64_at(data, start);
+    ret {val: val as uint, next: next};
+}
+
+
 fn new_doc(data: @[u8]) -> doc {
     ret {data: data, start: 0u, end: vec::len::<u8>(*data)};
 }
 
 fn doc_at(data: @[u8], start: uint) -> tagged_doc {
-    let elt_tag = vint_at(*data, start);
-    let elt_size = vint_at(*data, elt_tag.next);
+    let elt_tag = vuint_at(*data, start);
+    let elt_size = vuint_at(*data, elt_tag.next);
     let end = elt_size.next + elt_size.val;
     ret {tag: elt_tag.val,
          doc: {data: data, start: elt_size.next, end: end}};
@@ -56,8 +63,8 @@ fn doc_at(data: @[u8], start: uint) -> tagged_doc {
 fn maybe_get_doc(d: doc, tg: uint) -> option<doc> {
     let pos = d.start;
     while pos < d.end {
-        let elt_tag = vint_at(*d.data, pos);
-        let elt_size = vint_at(*d.data, elt_tag.next);
+        let elt_tag = vuint_at(*d.data, pos);
+        let elt_size = vuint_at(*d.data, elt_tag.next);
         pos = elt_size.next + elt_size.val;
         if elt_tag.val == tg {
             ret some::<doc>({data: d.data, start: elt_size.next, end: pos});
@@ -79,8 +86,8 @@ fn get_doc(d: doc, tg: uint) -> doc {
 fn docs(d: doc, it: fn(uint, doc)) {
     let pos = d.start;
     while pos < d.end {
-        let elt_tag = vint_at(*d.data, pos);
-        let elt_size = vint_at(*d.data, elt_tag.next);
+        let elt_tag = vuint_at(*d.data, pos);
+        let elt_size = vuint_at(*d.data, elt_tag.next);
         pos = elt_size.next + elt_size.val;
         it(elt_tag.val, {data: d.data, start: elt_size.next, end: pos});
     }
@@ -89,8 +96,8 @@ fn docs(d: doc, it: fn(uint, doc)) {
 fn tagged_docs(d: doc, tg: uint, it: fn(doc)) {
     let pos = d.start;
     while pos < d.end {
-        let elt_tag = vint_at(*d.data, pos);
-        let elt_size = vint_at(*d.data, elt_tag.next);
+        let elt_tag = vuint_at(*d.data, pos);
+        let elt_size = vuint_at(*d.data, elt_tag.next);
         pos = elt_size.next + elt_size.val;
         if elt_tag.val == tg {
             it({data: d.data, start: elt_size.next, end: pos});
@@ -102,28 +109,36 @@ fn doc_data(d: doc) -> [u8] { ret vec::slice::<u8>(*d.data, d.start, d.end); }
 
 fn doc_str(d: doc) -> str { ret str::from_bytes(doc_data(d)); }
 
-fn be_uint_from_bytes(data: @[u8], start: uint, size: uint) -> uint {
+fn be_u64_from_bytes(data: @[u8], start: uint, size: uint) -> u64 {
     let sz = size;
     assert (sz <= 4u);
-    let val = 0u;
+    let val = 0_u64;
     let pos = start;
     while sz > 0u {
         sz -= 1u;
-        val += (data[pos] as uint) << sz * 8u;
+        val += (data[pos] as u64) << ((sz * 8u) as u64);
         pos += 1u;
     }
     ret val;
 }
 
-fn doc_as_uint(d: doc) -> uint {
-    ret be_uint_from_bytes(d.data, d.start, d.end - d.start);
+fn doc_as_u8(d: doc) -> u8 {
+    assert d.end == d.start + 1u;
+    ret (*d.data)[d.start];
+}
+
+fn doc_as_vu64(d: doc) -> u64 {
+    ret vu64_at(*d.data, d.start).val;
 }
 
+fn doc_as_vuint(d: doc) -> uint {
+    ret vuint_at(*d.data, d.start).val;
+}
 
 // ebml writing
 type writer = {writer: io::writer, mutable size_positions: [uint]};
 
-fn write_sized_vint(w: io::writer, n: u64, size: uint) {
+fn write_sized_vu64(w: io::writer, n: u64, size: uint) {
     let buf: [u8];
     alt size {
       1u { buf = [0x80u8 | (n as u8)]; }
@@ -141,11 +156,11 @@ fn write_sized_vint(w: io::writer, n: u64, size: uint) {
     w.write(buf);
 }
 
-fn write_vint(w: io::writer, n: u64) {
-    if n < 0x7f_u64 { write_sized_vint(w, n, 1u); ret; }
-    if n < 0x4000_u64 { write_sized_vint(w, n, 2u); ret; }
-    if n < 0x200000_u64 { write_sized_vint(w, n, 3u); ret; }
-    if n < 0x10000000_u64 { write_sized_vint(w, n, 4u); ret; }
+fn write_vu64(w: io::writer, n: u64) {
+    if n < 0x7f_u64 { write_sized_vu64(w, n, 1u); ret; }
+    if n < 0x4000_u64 { write_sized_vu64(w, n, 2u); ret; }
+    if n < 0x200000_u64 { write_sized_vu64(w, n, 3u); ret; }
+    if n < 0x10000000_u64 { write_sized_vu64(w, n, 4u); ret; }
     #error("vint to write too big");
     fail;
 }
@@ -158,8 +173,10 @@ fn create_writer(w: io::writer) -> writer {
 
 // TODO: Provide a function to write the standard ebml header.
 fn start_tag(w: writer, tag_id: uint) {
+    #debug["Start tag %u", tag_id];
+
     // Write the enum ID:
-    write_vint(w.writer, tag_id as u64);
+    write_vu64(w.writer, tag_id as u64);
 
     // Write a placeholder four-byte size.
     w.size_positions += [w.writer.tell()];
@@ -171,8 +188,11 @@ fn end_tag(w: writer) {
     let last_size_pos = vec::pop::<uint>(w.size_positions);
     let cur_pos = w.writer.tell();
     w.writer.seek(last_size_pos as int, io::seek_set);
-    write_sized_vint(w.writer, (cur_pos - last_size_pos - 4u) as u64, 4u);
+    let size = (cur_pos - last_size_pos - 4u);
+    write_sized_vu64(w.writer, size as u64, 4u);
     w.writer.seek(cur_pos as int, io::seek_set);
+
+    #debug["End tag (size = %u)", size];
 }
 
 impl writer_util for writer {
@@ -182,20 +202,26 @@ impl writer_util for writer {
         end_tag(self);
     }
 
-    fn wr_u64(id: u64) {
-        write_vint(self.writer, id);
+    fn wr_vu64(id: u64) {
+        #debug["Write u64 0x%02x%02x",
+               (id >> 32u64) as uint,
+               (id & 0xFFFFFFFFu64) as uint];
+        write_vu64(self.writer, id);
     }
 
-    fn wr_uint(id: uint) {
-        self.wr_u64(id as u64);
+    fn wr_vuint(id: uint) {
+        #debug["Write uint: %u", id];
+        write_vu64(self.writer, id as u64);
     }
 
     fn wr_bytes(b: [u8]) {
+        #debug["Write %u bytes", vec::len(b)];
         self.writer.write(b);
     }
 
     fn wr_str(s: str) {
-        self.wr_bytes(str::bytes(s));
+        #debug["Write str: %?", s];
+        self.writer.write(str::bytes(s));
     }
 }
 
diff --git a/src/libstd/serialization.rs b/src/libstd/serialization.rs
index b71e057a5ad..1b91b34fe11 100644
--- a/src/libstd/serialization.rs
+++ b/src/libstd/serialization.rs
@@ -5,12 +5,13 @@ Support code for serialization.
 */
 
 import list::list;
+import ebml::writer_util;
 
 iface serializer {
     // Primitive types:
     fn emit_nil();
     fn emit_u64(v: u64);
-    fn emit_i64(v: u64);
+    fn emit_i64(v: i64);
     fn emit_bool(v: bool);
     fn emit_f64(v: f64);
     fn emit_str(v: str);
@@ -52,54 +53,8 @@ iface deserializer {
     fn read_tup_elt<T:copy>(idx: uint, f: fn() -> T) -> T;
 }
 
-/*
-type ppserializer = {
-    writer: io::writer
-};
-
-impl serializer for ppserializer {
-    fn emit_nil() { self.writer.write_str("()") }
-
-    fn emit_u64(v: u64) { self.writer.write_str(#fmt["%lu", v]); }
-    fn emit_i64(v: u64) { ebml::write_vint(self, v as uint) }
-    fn emit_bool(v: bool) { ebml::write_vint(self, v as uint) }
-    fn emit_f64(v: f64) { fail "float serialization not impl"; }
-    fn emit_str(v: str) {
-        self.wr_tag(es_str as uint) {|| self.wr_str(v) }
-    }
-
-    fn emit_enum(name: str, f: fn()) {
-        self.wr_tag(es_enum as uint) {|| f() }
-    }
-    fn emit_enum_variant(v_name: str, v_id: uint, f: fn()) {
-        self.wr_tag(es_enum_vid as uint) {|| self.write_vint(v_id) }
-        self.wr_tag(es_enum_body as uint) {|| f() }
-    }
-
-    fn emit_vec(len: uint, f: fn()) {
-        self.wr_tag(es_vec as uint) {||
-            self.wr_tag(es_vec_len as uint) {|| self.write_vint(len) }
-            f()
-        }
-    }
-
-    fn emit_vec_elt(idx: uint, f: fn()) {
-        self.wr_tag(es_vec_elt as uint) {|| f() }
-    }
-
-    fn emit_vec_elt(idx: uint, f: fn()) {
-        self.wr_tag(es_vec_elt as uint) {|| f() }
-    }
-
-    fn emit_box(f: fn()) { f() }
-    fn emit_uniq(f: fn()) { f() }
-    fn emit_rec_field(f_name: str, f_idx: uint, f: fn()) { f() }
-    fn emit_tup(sz: uint, f: fn()) { f() }
-    fn emit_tup_elt(idx: uint, f: fn()) { f() }
-}
-*/
-
-enum ebml_serializer_tags {
+enum ebml_serializer_tag {
+    es_u64, es_i64, es_bool,
     es_str,
     es_enum, es_enum_vid, es_enum_body,
     es_vec, es_vec_len, es_vec_elt
@@ -108,100 +63,108 @@ enum ebml_serializer_tags {
 impl of serializer for ebml::writer {
     fn emit_nil() {}
 
-    fn emit_u64(v: u64) { ebml::write_vint(self, v) }
-    fn emit_i64(v: u64) { ebml::write_vint(self, v as uint) }
-    fn emit_bool(v: bool) { ebml::write_vint(self, v as uint) }
-    fn emit_f64(v: f64) { fail "float serialization not impl"; }
-    fn emit_str(v: str) {
-        self.wr_tag(es_str as uint) {|| self.wr_str(v) }
+    fn emit_num(tag: ebml_serializer_tag, v: u64) {
+        self.wr_tag(tag as uint) {|| self.wr_vu64(v) }
     }
 
-    fn emit_enum(name: str, f: fn()) {
+    fn emit_u64(v: u64) { self.emit_num(es_u64, v) }
+    fn emit_i64(v: i64) { self.emit_num(es_i64, v as u64) }
+    fn emit_bool(v: bool) { self.emit_num(es_bool, v as u64) }
+    fn emit_f64(_v: f64) { fail "TODO"; }
+    fn emit_str(v: str) { self.wr_tag(es_str as uint) {|| self.wr_str(v) } }
+
+    fn emit_enum(_name: str, f: fn()) {
         self.wr_tag(es_enum as uint) {|| f() }
     }
-    fn emit_enum_variant(v_name: str, v_id: uint, f: fn()) {
-        self.wr_tag(es_enum_vid as uint) {|| self.write_vint(v_id) }
+    fn emit_enum_variant(_v_name: str, v_id: uint, _cnt: uint, f: fn()) {
+        self.emit_num(es_enum_vid, v_id as u64);
         self.wr_tag(es_enum_body as uint) {|| f() }
     }
-    fn emit_enum_variant_arg(idx: uint, f: fn()) { f() }
+    fn emit_enum_variant_arg(_idx: uint, f: fn()) { f() }
 
     fn emit_vec(len: uint, f: fn()) {
         self.wr_tag(es_vec as uint) {||
-            self.wr_tag(es_vec_len as uint) {|| self.write_vint(len) }
+            self.emit_num(es_vec_len, len as u64);
             f()
         }
     }
 
-    fn emit_vec_elt(idx: uint, f: fn()) {
+    fn emit_vec_elt(_idx: uint, f: fn()) {
         self.wr_tag(es_vec_elt as uint) {|| f() }
     }
 
-    fn emit_vec_elt(idx: uint, f: fn()) {
+    fn emit_vec_elt(_idx: uint, f: fn()) {
         self.wr_tag(es_vec_elt as uint) {|| f() }
     }
 
     fn emit_box(f: fn()) { f() }
     fn emit_uniq(f: fn()) { f() }
     fn emit_rec(f: fn()) { f() }
-    fn emit_rec_field(f_name: str, f_idx: uint, f: fn()) { f() }
-    fn emit_tup(sz: uint, f: fn()) { f() }
-    fn emit_tup_elt(idx: uint, f: fn()) { f() }
+    fn emit_rec_field(_f_name: str, _f_idx: uint, f: fn()) { f() }
+    fn emit_tup(_sz: uint, f: fn()) { f() }
+    fn emit_tup_elt(_idx: uint, f: fn()) { f() }
 }
 
 type ebml_deserializer = {mutable parent: ebml::doc,
                           mutable pos: uint};
 
 fn mk_ebml_deserializer(d: ebml::doc) -> ebml_deserializer {
-    {mutable parent: d, mutable pos: 0u}
+    {mutable parent: d, mutable pos: d.start}
 }
 
 impl of deserializer for ebml_deserializer {
-    fn next_doc(exp_tag: uint) -> ebml::doc {
+    fn next_doc(exp_tag: ebml_serializer_tag) -> ebml::doc {
         if self.pos >= self.parent.end {
             fail "no more documents in current node!";
         }
-        let (r_tag, r_doc) = ebml::doc_at(self.parent.data, self.pos);
-        if r_tag != exp_tag {
-            fail #fmt["expected EMBL doc with tag %u but found tag %u",
+        let {tag: r_tag, doc: r_doc} =
+            ebml::doc_at(self.parent.data, self.pos);
+        #debug["self.parent=%?-%? self.pos=%? r_tag=%? r_doc=%?-%?",
+               self.parent.start, self.parent.end, self.pos,
+               r_tag, r_doc.start, r_doc.end];
+        if r_tag != (exp_tag as uint) {
+            fail #fmt["expected EMBL doc with tag %? but found tag %?",
                       exp_tag, r_tag];
         }
-        if r_doc.end >= self.parent.end {
+        if r_doc.end > self.parent.end {
             fail #fmt["invalid EBML, child extends to 0x%x, parent to 0x%x",
                       r_doc.end, self.parent.end];
         }
-        self.pos = result.end;
-        ret result;
+        self.pos = r_doc.end;
+        ret r_doc;
     }
 
     fn push_doc<T: copy>(d: ebml::doc, f: fn() -> T) -> T{
         let old_parent = self.parent;
         let old_pos = self.pos;
         self.parent = d;
-        self.pos = 0u;
+        self.pos = d.start;
         let r = f();
         self.parent = old_parent;
         self.pos = old_pos;
         ret r;
     }
 
-    fn next_u64(exp_tag: uint) {
-        ebml::doc_as_uint(self.next_doc(exp_tag))
+    fn next_u64(exp_tag: ebml_serializer_tag) -> u64 {
+        let r = ebml::doc_as_vu64(self.next_doc(exp_tag));
+        #debug["next_u64 exp_tag=%? result=%?", exp_tag, r];
+        ret r;
     }
 
     fn read_nil() -> () { () }
-    fn read_u64() -> u64 { next_u64(es_u64) }
-    fn read_i64() -> i64 { next_u64(es_u64) as i64 }
-    fn read_bool() -> bool { next_u64(es_u64) as bool }
+    fn read_u64() -> u64 { self.next_u64(es_u64) }
+    fn read_i64() -> i64 { self.next_u64(es_i64) as i64 }
+    fn read_bool() -> bool { self.next_u64(es_bool) as bool }
     fn read_f64() -> f64 { fail "Float"; }
     fn read_str() -> str { ebml::doc_str(self.next_doc(es_str)) }
 
     // Compound types:
-    fn read_enum<T:copy>(name: str, f: fn() -> T) -> T {
+    fn read_enum<T:copy>(_name: str, f: fn() -> T) -> T {
         self.push_doc(self.next_doc(es_enum), f)
     }
 
     fn read_enum_variant<T:copy>(f: fn(uint) -> T) -> T {
-        let idx = self.next_u64(es_enum_vid);
+        let idx = self.next_u64(es_enum_vid) as uint;
         self.push_doc(self.next_doc(es_enum_body)) {||
             f(idx)
         }
@@ -218,7 +181,7 @@ impl of deserializer for ebml_deserializer {
         }
     }
 
-    fn read_vec_elt<T:copy>(idx: uint, f: fn() -> T) -> T {
+    fn read_vec_elt<T:copy>(_idx: uint, f: fn() -> T) -> T {
         self.push_doc(self.next_doc(es_vec_elt), f)
     }
 
@@ -234,15 +197,15 @@ impl of deserializer for ebml_deserializer {
         f()
     }
 
-    fn read_rec_field<T:copy>(f_name: str, f_idx: uint, f: fn() -> T) -> T {
+    fn read_rec_field<T:copy>(_f_name: str, _f_idx: uint, f: fn() -> T) -> T {
         f()
     }
 
-    fn read_tup<T:copy>(sz: uint, f: fn() -> T) -> T {
+    fn read_tup<T:copy>(_sz: uint, f: fn() -> T) -> T {
         f()
     }
 
-    fn read_tup_elt<T:copy>(idx: uint, f: fn() -> T) -> T {
+    fn read_tup_elt<T:copy>(_idx: uint, f: fn() -> T) -> T {
         f()
     }
 }
@@ -250,3 +213,60 @@ impl of deserializer for ebml_deserializer {
 // ___________________________________________________________________________
 // Testing
 
+#[test]
+fn test_option_int() {
+    fn serialize_1<S: serializer>(s: S, v: int) {
+        s.emit_i64(v as i64);
+    }
+
+    fn serialize_0<S: serializer>(s: S, v: option<int>) {
+        s.emit_enum("core::option::t") {||
+            alt v {
+              none {
+                s.emit_enum_variant("core::option::none", 0u, 0u) {||}
+              }
+              some(v0) {
+                s.emit_enum_variant("core::option::some", 1u, 1u) {||
+                    s.emit_enum_variant_arg(0u) {|| serialize_1(s, v0) }
+                }
+              }
+            }
+        }
+    }
+
+    fn deserialize_1<S: deserializer>(s: S) -> int {
+        s.read_i64() as int
+    }
+
+    fn deserialize_0<S: deserializer>(s: S) -> option<int> {
+        s.read_enum("option") {||
+            s.read_enum_variant {|i|
+                alt i {
+                  0u { none }
+                  1u {
+                    let v0 = s.read_enum_variant_arg(0u) {||
+                        deserialize_1(s)
+                    };
+                    some(v0)
+                  }
+                }
+            }
+        }
+    }
+
+    fn test_v(v: option<int>) {
+        #debug["v == %?", v];
+        let mbuf = io::mk_mem_buffer();
+        let ebml_w = ebml::create_writer(io::mem_buffer_writer(mbuf));
+        serialize_0(ebml_w, v);
+        let ebml_doc = ebml::new_doc(@io::mem_buffer_buf(mbuf));
+        let deser = mk_ebml_deserializer(ebml_doc);
+        let v1 = deserialize_0(deser);
+        #debug["v1 == %?", v1];
+        assert v == v1;
+    }
+
+    test_v(some(22));
+    test_v(none);
+    test_v(some(3));
+}
diff --git a/src/libstd/std.rc b/src/libstd/std.rc
index bcfe108eae5..695a45f9be7 100644
--- a/src/libstd/std.rc
+++ b/src/libstd/std.rc
@@ -70,7 +70,7 @@ mod unicode;
 // Compiler support modules
 
 mod test;
-
+mod serialization;
 
 // Target-os module.