Change literal representation to not truncate

Also shuffles around the organization of numeric literals and types,
separating by int/uint/float instead of machine-vs-non-machine types.
This simplifies some code.

Closes #974
Closes #1252

3 files changed, 87 insertions, 140 deletions

diff --git a/src/comp/syntax/parse/lexer.rs b/src/comp/syntax/parse/lexer.rs
index 385e0df34fe..aa35635f178 100644
--- a/src/comp/syntax/parse/lexer.rs
+++ b/src/comp/syntax/parse/lexer.rs
@@ -1,5 +1,5 @@
 
-import std::{io, vec, str, option};
+import std::{io, vec, str, option, either};
 import std::option::{some, none};
 import util::interner;
 import util::interner::intern;
@@ -161,103 +161,89 @@ fn scan_exponent(rdr: reader) -> option::t<str> {
     let c = rdr.curr();
     let rslt = "";
     if c == 'e' || c == 'E' {
-        rslt += str::unsafe_from_bytes([c as u8]);
+        str::push_byte(rslt, c as u8);
         rdr.bump();
         c = rdr.curr();
         if c == '-' || c == '+' {
-            rslt += str::unsafe_from_bytes([c as u8]);
+            str::push_byte(rslt, c as u8);
             rdr.bump();
         }
-        let exponent = scan_dec_digits(rdr);
+        let exponent = scan_digits(rdr, 10u);
         if str::byte_len(exponent) > 0u {
             ret some(rslt + exponent);
         } else { rdr.err("scan_exponent: bad fp literal"); fail; }
     } else { ret none::<str>; }
 }
 
-fn scan_dec_digits(rdr: reader) -> str {
-    let c = rdr.curr();
-    let rslt: str = "";
-    while is_dec_digit(c) || c == '_' {
-        if c != '_' { rslt += str::unsafe_from_bytes([c as u8]); }
-        rdr.bump();
-        c = rdr.curr();
+fn scan_digits(rdr: reader, radix: uint) -> str {
+    radix; // FIXME work around issue #1265
+    let rslt = "";
+    while true {
+        let c = rdr.curr();
+        if c == '_' { rdr.bump(); cont; }
+        alt std::char::maybe_digit(c) {
+          some(d) when (d as uint) < radix {
+            str::push_byte(rslt, c as u8);
+            rdr.bump();
+          }
+          _ { break; }
+        }
     }
     ret rslt;
 }
 
 fn scan_number(c: char, rdr: reader) -> token::token {
-    let accum_int = 0, c = c;
-    let num_str: str = "";
-    let n = rdr.next();
+    let num_str, base = 10u, c = c, n = rdr.next();
     if c == '0' && n == 'x' {
         rdr.bump();
         rdr.bump();
-        c = rdr.curr();
-        while is_hex_digit(c) || c == '_' {
-            if c != '_' { accum_int *= 16; accum_int += hex_digit_val(c); }
-            rdr.bump();
-            c = rdr.curr();
-        }
+        base = 16u;
     } else if c == '0' && n == 'b' {
         rdr.bump();
         rdr.bump();
-        c = rdr.curr();
-        while is_bin_digit(c) || c == '_' {
-            if c != '_' { accum_int *= 2; accum_int += bin_digit_value(c); }
-            rdr.bump();
-            c = rdr.curr();
-        }
-    } else {
-        num_str = scan_dec_digits(rdr);
-        accum_int = std::int::from_str(num_str);
+        base = 2u;
     }
+    num_str = scan_digits(rdr, base);
     c = rdr.curr();
     n = rdr.next();
     if c == 'u' || c == 'i' {
-        let signed: bool = c == 'i';
+        let signed = c == 'i', tp = signed ? either::left(ast::ty_i)
+                                           : either::right(ast::ty_u);
         rdr.bump();
         c = rdr.curr();
         if c == '8' {
             rdr.bump();
-            if signed {
-                ret token::LIT_MACH_INT(ast::ty_i8, accum_int);
-            } else { ret token::LIT_MACH_INT(ast::ty_u8, accum_int); }
+            tp = signed ? either::left(ast::ty_i8)
+                        : either::right(ast::ty_u8);
         }
         n = rdr.next();
         if c == '1' && n == '6' {
             rdr.bump();
             rdr.bump();
-            if signed {
-                ret token::LIT_MACH_INT(ast::ty_i16, accum_int);
-            } else { ret token::LIT_MACH_INT(ast::ty_u16, accum_int); }
-        }
-        if c == '3' && n == '2' {
+            tp = signed ? either::left(ast::ty_i16)
+                        : either::right(ast::ty_u16);
+        } else if c == '3' && n == '2' {
             rdr.bump();
             rdr.bump();
-            if signed {
-                ret token::LIT_MACH_INT(ast::ty_i32, accum_int);
-            } else { ret token::LIT_MACH_INT(ast::ty_u32, accum_int); }
-        }
-        if c == '6' && n == '4' {
+            tp = signed ? either::left(ast::ty_i32)
+                        : either::right(ast::ty_u32);
+        } else if c == '6' && n == '4' {
             rdr.bump();
             rdr.bump();
-            if signed {
-                ret token::LIT_MACH_INT(ast::ty_i64, accum_int);
-            } else { ret token::LIT_MACH_INT(ast::ty_u64, accum_int); }
+            tp = signed ? either::left(ast::ty_i64)
+                        : either::right(ast::ty_u64);
         }
-        if signed {
-            ret token::LIT_INT(accum_int);
-        } else {
-            // FIXME: should cast in the target bit-width.
-            ret token::LIT_UINT(accum_int as uint);
+        let parsed = std::u64::from_str(num_str, base as u64);
+        alt tp {
+          either::left(t) { ret token::LIT_INT(parsed as i64, t); }
+          either::right(t) { ret token::LIT_UINT(parsed, t); }
         }
     }
     let is_float = false;
     if rdr.curr() == '.' {
         is_float = true;
         rdr.bump();
-        let dec_part = scan_dec_digits(rdr);
+        let dec_part = scan_digits(rdr, 10u);
         num_str += "." + dec_part;
     }
     alt scan_exponent(rdr) {
@@ -274,15 +260,13 @@ fn scan_number(c: char, rdr: reader) -> token::token {
         if c == '3' && n == '2' {
             rdr.bump();
             rdr.bump();
-            ret token::LIT_MACH_FLOAT(ast::ty_f32,
-                                      intern(*rdr.get_interner(),
-                                             num_str));
+            ret token::LIT_FLOAT(intern(*rdr.get_interner(), num_str),
+                                 ast::ty_f32);
         } else if c == '6' && n == '4' {
             rdr.bump();
             rdr.bump();
-            ret token::LIT_MACH_FLOAT(ast::ty_f64,
-                                      intern(*rdr.get_interner(),
-                                             num_str));
+            ret token::LIT_FLOAT(intern(*rdr.get_interner(), num_str),
+                                 ast::ty_f64);
             /* FIXME: if this is out of range for either a 32-bit or
             64-bit float, it won't be noticed till the back-end */
         } else {
@@ -290,10 +274,11 @@ fn scan_number(c: char, rdr: reader) -> token::token {
         }
     }
     if is_float {
-        ret token::LIT_FLOAT(interner::intern::<str>(*rdr.get_interner(),
-                                                     num_str));
+        ret token::LIT_FLOAT(interner::intern(*rdr.get_interner(), num_str),
+                             ast::ty_f);
     } else {
-        ret token::LIT_INT(accum_int);
+        let parsed = std::u64::from_str(num_str, base as u64);
+        ret token::LIT_INT(parsed as i64, ast::ty_i);
     }
 }
 
@@ -463,8 +448,7 @@ fn next_token_inner(rdr: reader) -> token::token {
             fail;
         }
         rdr.bump(); // advance curr past token
-
-        ret token::LIT_CHAR(c2);
+        ret token::LIT_INT(c2 as i64, ast::ty_char);
       }
       '"' {
         let n = rdr.get_chpos();
@@ -685,13 +669,10 @@ fn consume_comment(rdr: reader, code_to_the_left: bool, &comments: [cmnt]) {
 
 fn is_lit(t: token::token) -> bool {
     ret alt t {
-          token::LIT_INT(_) { true }
-          token::LIT_UINT(_) { true }
-          token::LIT_MACH_INT(_, _) { true }
-          token::LIT_FLOAT(_) { true }
-          token::LIT_MACH_FLOAT(_, _) { true }
+          token::LIT_INT(_, _) { true }
+          token::LIT_UINT(_, _) { true }
+          token::LIT_FLOAT(_, _) { true }
           token::LIT_STR(_) { true }
-          token::LIT_CHAR(_) { true }
           token::LIT_BOOL(_) { true }
           _ { false }
         }
diff --git a/src/comp/syntax/parse/parser.rs b/src/comp/syntax/parse/parser.rs
index 7523ec26238..23ed2485f69 100644
--- a/src/comp/syntax/parse/parser.rs
+++ b/src/comp/syntax/parse/parser.rs
@@ -459,39 +459,35 @@ fn parse_ty(p: parser, colons_before_params: bool) -> @ast::ty {
     if eat_word(p, "bool") {
         t = ast::ty_bool;
     } else if eat_word(p, "int") {
-        t = ast::ty_int;
+        t = ast::ty_int(ast::ty_i);
     } else if eat_word(p, "uint") {
-        t = ast::ty_uint;
+        t = ast::ty_uint(ast::ty_u);
     } else if eat_word(p, "float") {
-        t = ast::ty_float;
+        t = ast::ty_float(ast::ty_f);
     } else if eat_word(p, "str") {
         t = ast::ty_str;
     } else if eat_word(p, "char") {
-        t = ast::ty_char;
-        /*
-            } else if (eat_word(p, "task")) {
-                t = ast::ty_task;
-        */
+        t = ast::ty_int(ast::ty_char);
     } else if eat_word(p, "i8") {
-        t = ast::ty_machine(ast::ty_i8);
+        t = ast::ty_int(ast::ty_i8);
     } else if eat_word(p, "i16") {
-        t = ast::ty_machine(ast::ty_i16);
+        t = ast::ty_int(ast::ty_i16);
     } else if eat_word(p, "i32") {
-        t = ast::ty_machine(ast::ty_i32);
+        t = ast::ty_int(ast::ty_i32);
     } else if eat_word(p, "i64") {
-        t = ast::ty_machine(ast::ty_i64);
+        t = ast::ty_int(ast::ty_i64);
     } else if eat_word(p, "u8") {
-        t = ast::ty_machine(ast::ty_u8);
+        t = ast::ty_uint(ast::ty_u8);
     } else if eat_word(p, "u16") {
-        t = ast::ty_machine(ast::ty_u16);
+        t = ast::ty_uint(ast::ty_u16);
     } else if eat_word(p, "u32") {
-        t = ast::ty_machine(ast::ty_u32);
+        t = ast::ty_uint(ast::ty_u32);
     } else if eat_word(p, "u64") {
-        t = ast::ty_machine(ast::ty_u64);
+        t = ast::ty_uint(ast::ty_u64);
     } else if eat_word(p, "f32") {
-        t = ast::ty_machine(ast::ty_f32);
+        t = ast::ty_float(ast::ty_f32);
     } else if eat_word(p, "f64") {
-        t = ast::ty_machine(ast::ty_f64);
+        t = ast::ty_float(ast::ty_f64);
     } else if p.peek() == token::LPAREN {
         p.bump();
         if p.peek() == token::RPAREN {
@@ -662,12 +658,9 @@ fn parse_seq<copy T>(bra: token::token, ket: token::token,
 
 fn lit_from_token(p: parser, tok: token::token) -> ast::lit_ {
     alt tok {
-      token::LIT_INT(i) { ast::lit_int(i) }
-      token::LIT_UINT(u) { ast::lit_uint(u) }
-      token::LIT_FLOAT(s) { ast::lit_float(p.get_str(s)) }
-      token::LIT_MACH_INT(tm, i) { ast::lit_mach_int(tm, i) }
-      token::LIT_MACH_FLOAT(tm, s) { ast::lit_mach_float(tm, p.get_str(s)) }
-      token::LIT_CHAR(c) { ast::lit_char(c) }
+      token::LIT_INT(i, it) { ast::lit_int(i, it) }
+      token::LIT_UINT(u, ut) { ast::lit_uint(u, ut) }
+      token::LIT_FLOAT(s, ft) { ast::lit_float(p.get_str(s), ft) }
       token::LIT_STR(s) { ast::lit_str(p.get_str(s)) }
       token::LPAREN. { expect(p, token::RPAREN); ast::lit_nil }
       _ { unexpected(p, tok); }
diff --git a/src/comp/syntax/parse/token.rs b/src/comp/syntax/parse/token.rs
index bb1b7bd6e57..6740ba74c00 100644
--- a/src/comp/syntax/parse/token.rs
+++ b/src/comp/syntax/parse/token.rs
@@ -1,6 +1,4 @@
 
-import ast::ty_mach;
-import ast_util::ty_mach_to_str;
 import util::interner;
 import std::{int, uint, str};
 
@@ -21,8 +19,6 @@ tag binop {
 }
 
 tag token {
-
-
     /* Expression-operator symbols. */
     EQ;
     LT;
@@ -38,7 +34,6 @@ tag token {
     BINOP(binop);
     BINOPEQ(binop);
 
-
     /* Structural symbols */
     AT;
     DOT;
@@ -61,18 +56,13 @@ tag token {
     POUND_LBRACE;
     POUND_LT;
 
-
     /* Literals */
-    LIT_INT(int);
-    LIT_UINT(uint);
-    LIT_MACH_INT(ty_mach, int);
-    LIT_FLOAT(str_num);
-    LIT_MACH_FLOAT(ty_mach, str_num);
+    LIT_INT(i64, ast::int_ty);
+    LIT_UINT(u64, ast::uint_ty);
+    LIT_FLOAT(str_num, ast::float_ty);
     LIT_STR(str_num);
-    LIT_CHAR(char);
     LIT_BOOL(bool);
 
-
     /* Name components */
     IDENT(str_num, bool);
     IDX(int);
@@ -113,10 +103,6 @@ fn to_str(r: lexer::reader, t: token) -> str {
       BINOP(op) { ret binop_to_str(op); }
       BINOPEQ(op) { ret binop_to_str(op) + "="; }
 
-
-
-
-
       /* Structural symbols */
       AT. {
         ret "@";
@@ -141,39 +127,29 @@ fn to_str(r: lexer::reader, t: token) -> str {
       POUND_LBRACE. { ret "#{"; }
       POUND_LT. { ret "#<"; }
 
-
-
-
-
       /* Literals */
-      LIT_INT(i) {
-        ret int::to_str(i, 10u);
+      LIT_INT(c, ast::ty_char.) {
+        // FIXME: escape.
+        let tmp = "'";
+        str::push_char(tmp, c as char);
+        str::push_byte(tmp, '\'' as u8);
+        ret tmp;
+      }
+      LIT_INT(i, t) {
+        ret int::to_str(i as int, 10u) + ast_util::int_ty_to_str(t);
       }
-      LIT_UINT(u) { ret uint::to_str(u, 10u); }
-      LIT_MACH_INT(tm, i) {
-        ret int::to_str(i, 10u) + "_" + ty_mach_to_str(tm);
+      LIT_UINT(u, t) {
+        ret uint::to_str(u as uint, 10u) + ast_util::uint_ty_to_str(t);
       }
-      LIT_MACH_FLOAT(tm, s) {
-        ret interner::get::<str>(*r.get_interner(), s) + "_" +
-                ty_mach_to_str(tm);
+      LIT_FLOAT(s, t) {
+        ret interner::get::<str>(*r.get_interner(), s) +
+            ast_util::float_ty_to_str(t);
       }
-      LIT_FLOAT(s) { ret interner::get::<str>(*r.get_interner(), s); }
       LIT_STR(s) { // FIXME: escape.
         ret "\"" + interner::get::<str>(*r.get_interner(), s) + "\"";
       }
-      LIT_CHAR(c) {
-        // FIXME: escape.
-        let tmp = "'";
-        str::push_char(tmp, c);
-        str::push_byte(tmp, '\'' as u8);
-        ret tmp;
-      }
       LIT_BOOL(b) { if b { ret "true"; } else { ret "false"; } }
 
-
-
-
-
       /* Name components */
       IDENT(s, _) {
         ret interner::get::<str>(*r.get_interner(), s);
@@ -194,13 +170,10 @@ pure fn can_begin_expr(t: token) -> bool {
       IDENT(_, _) { true }
       UNDERSCORE. { true }
       TILDE. { true }
-      LIT_INT(_) { true }
-      LIT_UINT(_) { true }
-      LIT_MACH_INT(_, _) { true }
-      LIT_FLOAT(_) { true }
-      LIT_MACH_FLOAT(_, _) { true }
+      LIT_INT(_, _) { true }
+      LIT_UINT(_, _) { true }
+      LIT_FLOAT(_, _) { true }
       LIT_STR(_) { true }
-      LIT_CHAR(_) { true }
       POUND. { true }
       AT. { true }
       NOT. { true }