Add a b"xx" byte string literal of type &'static [u8].

4 files changed, 128 insertions, 66 deletions

diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs
index 7e4cb195cea..59bcf059fcd 100644
--- a/src/libsyntax/parse/lexer/mod.rs
+++ b/src/libsyntax/parse/lexer/mod.rs
@@ -654,7 +654,8 @@ impl<'a> StringReader<'a> {
             // Note: r as in r" or r#" is part of a raw string literal,
             // b as in b' is part of a byte literal.
             // They are not identifiers, and are handled further down.
-           ('r', Some('"')) | ('r', Some('#')) | ('b', Some('\'')) => false,
+           ('r', Some('"')) | ('r', Some('#')) |
+           ('b', Some('"')) | ('b', Some('\'')) => false,
            _ => true
         } {
             let start = self.last_pos;
@@ -859,62 +860,124 @@ impl<'a> StringReader<'a> {
           }
           'b' => {
             self.bump();
-            assert!(self.curr_is('\''), "Should have been a token::IDENT");
-            self.bump();
-            let start = self.last_pos;
-
-            // the eof will be picked up by the final `'` check below
-            let mut c2 = self.curr.unwrap_or('\x00');
-            self.bump();
+            return match self.curr {
+                Some('\'') => parse_byte(self),
+                Some('"') => parse_byte_string(self),
+                _ => unreachable!()  // Should have been a token::IDENT above.
+            };
 
-            match c2 {
-                '\\' => {
-                    // '\X' for some X must be a character constant:
-                    let escaped = self.curr;
-                    let escaped_pos = self.last_pos;
-                    self.bump();
-                    match escaped {
-                        None => {}
-                        Some(e) => {
-                            c2 = match e {
-                                'n' => '\n',
-                                'r' => '\r',
-                                't' => '\t',
-                                '\\' => '\\',
-                                '\'' => '\'',
-                                '"' => '"',
-                                '0' => '\x00',
-                                'x' => self.scan_numeric_escape(2u, '\''),
-                                c2 => {
-                                    self.err_span_char(escaped_pos, self.last_pos,
-                                                       "unknown byte escape", c2);
-                                    c2
+            fn parse_byte(self_: &mut StringReader) -> token::Token {
+                self_.bump();
+                let start = self_.last_pos;
+
+                // the eof will be picked up by the final `'` check below
+                let mut c2 = self_.curr.unwrap_or('\x00');
+                self_.bump();
+
+                match c2 {
+                    '\\' => {
+                        // '\X' for some X must be a character constant:
+                        let escaped = self_.curr;
+                        let escaped_pos = self_.last_pos;
+                        self_.bump();
+                        match escaped {
+                            None => {}
+                            Some(e) => {
+                                c2 = match e {
+                                    'n' => '\n',
+                                    'r' => '\r',
+                                    't' => '\t',
+                                    '\\' => '\\',
+                                    '\'' => '\'',
+                                    '"' => '"',
+                                    '0' => '\x00',
+                                    'x' => self_.scan_numeric_escape(2u, '\''),
+                                    c2 => {
+                                        self_.err_span_char(
+                                            escaped_pos, self_.last_pos,
+                                            "unknown byte escape", c2);
+                                        c2
+                                    }
                                 }
                             }
                         }
                     }
+                    '\t' | '\n' | '\r' | '\'' => {
+                        self_.err_span_char( start, self_.last_pos,
+                            "byte constant must be escaped", c2);
+                    }
+                    _ => if c2 > '\x7F' {
+                        self_.err_span_char( start, self_.last_pos,
+                            "byte constant must be ASCII. \
+                             Use a \\xHH escape for a non-ASCII byte", c2);
+                    }
                 }
-                '\t' | '\n' | '\r' | '\'' => {
-                    self.err_span_char( start, self.last_pos,
-                        "byte constant must be escaped", c2);
-                }
-                _ if c2 > '\x7F' => {
-                    self.err_span_char( start, self.last_pos,
-                        "byte constant must be ASCII. \
-                         Use a \\xHH escape for a non-ASCII byte", c2);
+                if !self_.curr_is('\'') {
+                    // Byte offsetting here is okay because the
+                    // character before position `start` are an
+                    // ascii single quote and ascii 'b'.
+                    self_.fatal_span_verbose(
+                        start - BytePos(2), self_.last_pos,
+                        "unterminated byte constant".to_string());
                 }
-                _ => {}
+                self_.bump(); // advance curr past token
+                return token::LIT_BYTE(c2 as u8);
             }
-            if !self.curr_is('\'') {
-                self.fatal_span_verbose(
-                                   // Byte offsetting here is okay because the
-                                   // character before position `start` are an
-                                   // ascii single quote and ascii 'b'.
-                                   start - BytePos(2), self.last_pos,
-                                   "unterminated byte constant".to_string());
+
+            fn parse_byte_string(self_: &mut StringReader) -> token::Token {
+                self_.bump();
+                let start = self_.last_pos;
+                let mut value = Vec::new();
+                while !self_.curr_is('"') {
+                    if self_.is_eof() {
+                        self_.fatal_span(start, self_.last_pos,
+                                         "unterminated double quote byte string");
+                    }
+
+                    let ch = self_.curr.unwrap();
+                    self_.bump();
+                    match ch {
+                      '\\' => {
+                        if self_.is_eof() {
+                            self_.fatal_span(start, self_.last_pos,
+                                             "unterminated double quote byte string");
+                        }
+
+                        let escaped = self_.curr.unwrap();
+                        let escaped_pos = self_.last_pos;
+                        self_.bump();
+                        match escaped {
+                          'n' => value.push('\n' as u8),
+                          'r' => value.push('\r' as u8),
+                          't' => value.push('\t' as u8),
+                          '\\' => value.push('\\' as u8),
+                          '\'' => value.push('\'' as u8),
+                          '"' => value.push('"' as u8),
+                          '\n' => self_.consume_whitespace(),
+                          '0' => value.push(0),
+                          'x' => {
+                            value.push(self_.scan_numeric_escape(2u, '"') as u8);
+                          }
+                          c2 => {
+                            self_.err_span_char(escaped_pos, self_.last_pos,
+                                                "unknown byte string escape", c2);
+                          }
+                        }
+                      }
+                      _ => {
+                        if ch <= '\x7F' {
+                            value.push(ch as u8)
+                        } else {
+                            self_.err_span_char(self_.last_pos, self_.last_pos,
+                                "byte string must be ASCII. \
+                                 Use a \\xHH escape for a non-ASCII byte", ch);
+                        }
+                      }
+                    }
+                }
+                self_.bump();
+                return token::LIT_BINARY(Rc::new(value));
             }
-            self.bump(); // advance curr past token
-            return token::LIT_BYTE(c2 as u8);
           }
           '"' => {
             let mut accum_str = String::new();
diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs
index 0bd47ede214..826d28ef3ff 100644
--- a/src/libsyntax/parse/parser.rs
+++ b/src/libsyntax/parse/parser.rs
@@ -33,7 +33,7 @@ use ast::{ForeignItem, ForeignItemStatic, ForeignItemFn, ForeignMod};
 use ast::{Ident, NormalFn, Inherited, Item, Item_, ItemStatic};
 use ast::{ItemEnum, ItemFn, ItemForeignMod, ItemImpl};
 use ast::{ItemMac, ItemMod, ItemStruct, ItemTrait, ItemTy, Lit, Lit_};
-use ast::{LitBool, LitFloat, LitFloatUnsuffixed, LitInt, LitChar, LitByte};
+use ast::{LitBool, LitFloat, LitFloatUnsuffixed, LitInt, LitChar, LitByte, LitBinary};
 use ast::{LitIntUnsuffixed, LitNil, LitStr, LitUint, Local, LocalLet};
 use ast::{MutImmutable, MutMutable, Mac_, MacInvocTT, Matcher, MatchNonterminal};
 use ast::{MatchSeq, MatchTok, Method, MutTy, BiMul, Mutability};
@@ -1529,6 +1529,7 @@ impl<'a> Parser<'a> {
             token::LIT_STR_RAW(s, n) => {
                 LitStr(self.id_to_interned_str(s), ast::RawStr(n))
             }
+            token::LIT_BINARY(ref v) => LitBinary(v.clone()),
             token::LPAREN => { self.expect(&token::RPAREN); LitNil },
             _ => { self.unexpected_last(tok); }
         }
diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs
index b8f13624a32..b76dcaf0b94 100644
--- a/src/libsyntax/parse/token.rs
+++ b/src/libsyntax/parse/token.rs
@@ -87,6 +87,7 @@ pub enum Token {
     LIT_FLOAT_UNSUFFIXED(ast::Ident),
     LIT_STR(ast::Ident),
     LIT_STR_RAW(ast::Ident, uint), /* raw str delimited by n hash symbols */
+    LIT_BINARY(Rc<Vec<u8>>),
 
     /* Name components */
     // an identifier contains an "is_mod_name" boolean,
@@ -231,17 +232,22 @@ pub fn to_str(t: &Token) -> String {
         body
       }
       LIT_STR(s) => {
-          (format!("\"{}\"", get_ident(s).get().escape_default())).to_string()
+          format!("\"{}\"", get_ident(s).get().escape_default())
       }
       LIT_STR_RAW(s, n) => {
-          (format!("r{delim}\"{string}\"{delim}",
-                  delim="#".repeat(n), string=get_ident(s))).to_string()
+        format!("r{delim}\"{string}\"{delim}",
+                 delim="#".repeat(n), string=get_ident(s))
+      }
+      LIT_BINARY(ref v) => {
+          format!(
+            "b\"{}\"",
+            v.iter().map(|&b| b as char).collect::<String>().escape_default())
       }
 
       /* Name components */
       IDENT(s, _) => get_ident(s).get().to_string(),
       LIFETIME(s) => {
-          (format!("{}", get_ident(s))).to_string()
+          format!("{}", get_ident(s))
       }
       UNDERSCORE => "_".to_string(),
 
@@ -291,6 +297,7 @@ pub fn can_begin_expr(t: &Token) -> bool {
       LIT_FLOAT_UNSUFFIXED(_) => true,
       LIT_STR(_) => true,
       LIT_STR_RAW(_, _) => true,
+      LIT_BINARY(_) => true,
       POUND => true,
       AT => true,
       NOT => true,
@@ -330,6 +337,7 @@ pub fn is_lit(t: &Token) -> bool {
       LIT_FLOAT_UNSUFFIXED(_) => true,
       LIT_STR(_) => true,
       LIT_STR_RAW(_, _) => true,
+      LIT_BINARY(_) => true,
       _ => false
     }
 }
diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs
index 6ea2eed293e..fafebd3c5dc 100644
--- a/src/libsyntax/print/pprust.rs
+++ b/src/libsyntax/print/pprust.rs
@@ -2342,19 +2342,9 @@ impl<'a> State<'a> {
             ast::LitBool(val) => {
                 if val { word(&mut self.s, "true") } else { word(&mut self.s, "false") }
             }
-            ast::LitBinary(ref arr) => {
-                try!(self.ibox(indent_unit));
-                try!(word(&mut self.s, "["));
-                try!(self.commasep_cmnt(Inconsistent,
-                                        arr.as_slice(),
-                                        |s, u| {
-                                            word(&mut s.s,
-                                                 format!("{}",
-                                                         *u).as_slice())
-                                        },
-                                        |_| lit.span));
-                try!(word(&mut self.s, "]"));
-                self.end()
+            ast::LitBinary(ref v) => {
+                let escaped: String = v.iter().map(|&b| b as char).collect();
+                word(&mut self.s, format!("b\"{}\"", escaped.escape_default()).as_slice())
             }
         }
     }