diff options
Diffstat (limited to 'src/libsyntax/parse')
| -rw-r--r-- | src/libsyntax/parse/attr.rs | 48 | ||||
| -rw-r--r-- | src/libsyntax/parse/classify.rs | 20 | ||||
| -rw-r--r-- | src/libsyntax/parse/common.rs | 4 | ||||
| -rw-r--r-- | src/libsyntax/parse/lexer/comments.rs | 26 | ||||
| -rw-r--r-- | src/libsyntax/parse/lexer/mod.rs | 803 | ||||
| -rw-r--r-- | src/libsyntax/parse/mod.rs | 354 | ||||
| -rw-r--r-- | src/libsyntax/parse/obsolete.rs | 8 | ||||
| -rw-r--r-- | src/libsyntax/parse/parser.rs | 517 | ||||
| -rw-r--r-- | src/libsyntax/parse/token.rs | 196 |
9 files changed, 1185 insertions, 791 deletions
diff --git a/src/libsyntax/parse/attr.rs b/src/libsyntax/parse/attr.rs index 53489e32837..55ad1b77123 100644 --- a/src/libsyntax/parse/attr.rs +++ b/src/libsyntax/parse/attr.rs @@ -18,7 +18,7 @@ use parse::token::INTERPOLATED; use std::gc::{Gc, GC}; -// a parser that can parse attributes. +/// A parser that can parse attributes. pub trait ParserAttr { fn parse_outer_attributes(&mut self) -> Vec<ast::Attribute>; fn parse_attribute(&mut self, permit_inner: bool) -> ast::Attribute; @@ -30,11 +30,11 @@ pub trait ParserAttr { } impl<'a> ParserAttr for Parser<'a> { - // Parse attributes that appear before an item + /// Parse attributes that appear before an item fn parse_outer_attributes(&mut self) -> Vec<ast::Attribute> { let mut attrs: Vec<ast::Attribute> = Vec::new(); loop { - debug!("parse_outer_attributes: self.token={:?}", + debug!("parse_outer_attributes: self.token={}", self.token); match self.token { token::POUND => { @@ -43,7 +43,7 @@ impl<'a> ParserAttr for Parser<'a> { token::DOC_COMMENT(s) => { let attr = ::attr::mk_sugared_doc_attr( attr::mk_attr_id(), - self.id_to_interned_str(s), + self.id_to_interned_str(s.ident()), self.span.lo, self.span.hi ); @@ -59,10 +59,10 @@ impl<'a> ParserAttr for Parser<'a> { return attrs; } - // matches attribute = # ! [ meta_item ] - // - // if permit_inner is true, then a leading `!` indicates an inner - // attribute + /// Matches `attribute = # ! [ meta_item ]` + /// + /// If permit_inner is true, then a leading `!` indicates an inner + /// attribute fn parse_attribute(&mut self, permit_inner: bool) -> ast::Attribute { debug!("parse_attributes: permit_inner={:?} self.token={:?}", permit_inner, self.token); @@ -114,17 +114,17 @@ impl<'a> ParserAttr for Parser<'a> { }; } - // Parse attributes that appear after the opening of an item. These should - // be preceded by an exclamation mark, but we accept and warn about one - // terminated by a semicolon. In addition to a vector of inner attributes, - // this function also returns a vector that may contain the first outer - // attribute of the next item (since we can't know whether the attribute - // is an inner attribute of the containing item or an outer attribute of - // the first contained item until we see the semi). - - // matches inner_attrs* outer_attr? - // you can make the 'next' field an Option, but the result is going to be - // more useful as a vector. + /// Parse attributes that appear after the opening of an item. These should + /// be preceded by an exclamation mark, but we accept and warn about one + /// terminated by a semicolon. In addition to a vector of inner attributes, + /// this function also returns a vector that may contain the first outer + /// attribute of the next item (since we can't know whether the attribute + /// is an inner attribute of the containing item or an outer attribute of + /// the first contained item until we see the semi). + + /// matches inner_attrs* outer_attr? + /// you can make the 'next' field an Option, but the result is going to be + /// more useful as a vector. fn parse_inner_attrs_and_next(&mut self) -> (Vec<ast::Attribute> , Vec<ast::Attribute> ) { let mut inner_attrs: Vec<ast::Attribute> = Vec::new(); @@ -139,7 +139,7 @@ impl<'a> ParserAttr for Parser<'a> { let Span { lo, hi, .. } = self.span; self.bump(); attr::mk_sugared_doc_attr(attr::mk_attr_id(), - self.id_to_interned_str(s), + self.id_to_interned_str(s.ident()), lo, hi) } @@ -157,9 +157,9 @@ impl<'a> ParserAttr for Parser<'a> { (inner_attrs, next_outer_attrs) } - // matches meta_item = IDENT - // | IDENT = lit - // | IDENT meta_seq + /// matches meta_item = IDENT + /// | IDENT = lit + /// | IDENT meta_seq fn parse_meta_item(&mut self) -> Gc<ast::MetaItem> { match self.token { token::INTERPOLATED(token::NtMeta(e)) => { @@ -201,7 +201,7 @@ impl<'a> ParserAttr for Parser<'a> { } } - // matches meta_seq = ( COMMASEP(meta_item) ) + /// matches meta_seq = ( COMMASEP(meta_item) ) fn parse_meta_seq(&mut self) -> Vec<Gc<ast::MetaItem>> { self.parse_seq(&token::LPAREN, &token::RPAREN, diff --git a/src/libsyntax/parse/classify.rs b/src/libsyntax/parse/classify.rs index 8d9cc305c26..516f22cdf4d 100644 --- a/src/libsyntax/parse/classify.rs +++ b/src/libsyntax/parse/classify.rs @@ -15,13 +15,13 @@ use ast; use std::gc::Gc; -// does this expression require a semicolon to be treated -// as a statement? The negation of this: 'can this expression -// be used as a statement without a semicolon' -- is used -// as an early-bail-out in the parser so that, for instance, -// 'if true {...} else {...} -// |x| 5 ' -// isn't parsed as (if true {...} else {...} | x) | 5 +/// Does this expression require a semicolon to be treated +/// as a statement? The negation of this: 'can this expression +/// be used as a statement without a semicolon' -- is used +/// as an early-bail-out in the parser so that, for instance, +/// if true {...} else {...} +/// |x| 5 +/// isn't parsed as (if true {...} else {...} | x) | 5 pub fn expr_requires_semi_to_be_stmt(e: Gc<ast::Expr>) -> bool { match e.node { ast::ExprIf(..) @@ -41,9 +41,9 @@ pub fn expr_is_simple_block(e: Gc<ast::Expr>) -> bool { } } -// this statement requires a semicolon after it. -// note that in one case (stmt_semi), we've already -// seen the semicolon, and thus don't need another. +/// this statement requires a semicolon after it. +/// note that in one case (stmt_semi), we've already +/// seen the semicolon, and thus don't need another. pub fn stmt_ends_with_semi(stmt: &ast::Stmt) -> bool { return match stmt.node { ast::StmtDecl(d, _) => { diff --git a/src/libsyntax/parse/common.rs b/src/libsyntax/parse/common.rs index 3c3f0c7a820..3842170d677 100644 --- a/src/libsyntax/parse/common.rs +++ b/src/libsyntax/parse/common.rs @@ -12,8 +12,8 @@ use parse::token; -// SeqSep : a sequence separator (token) -// and whether a trailing separator is allowed. +/// SeqSep : a sequence separator (token) +/// and whether a trailing separator is allowed. pub struct SeqSep { pub sep: Option<token::Token>, pub trailing_sep_allowed: bool diff --git a/src/libsyntax/parse/lexer/comments.rs b/src/libsyntax/parse/lexer/comments.rs index 73e5bb97f51..3f3a8a723f1 100644 --- a/src/libsyntax/parse/lexer/comments.rs +++ b/src/libsyntax/parse/lexer/comments.rs @@ -13,7 +13,7 @@ use codemap::{BytePos, CharPos, CodeMap, Pos}; use diagnostic; use parse::lexer::{is_whitespace, Reader}; use parse::lexer::{StringReader, TokenAndSpan}; -use parse::lexer::{is_line_non_doc_comment, is_block_non_doc_comment}; +use parse::lexer::is_block_doc_comment; use parse::lexer; use parse::token; @@ -24,10 +24,14 @@ use std::uint; #[deriving(Clone, PartialEq)] pub enum CommentStyle { - Isolated, // No code on either side of each line of the comment - Trailing, // Code exists to the left of the comment - Mixed, // Code before /* foo */ and after the comment - BlankLine, // Just a manual blank line "\n\n", for layout + /// No code on either side of each line of the comment + Isolated, + /// Code exists to the left of the comment + Trailing, + /// Code before /* foo */ and after the comment + Mixed, + /// Just a manual blank line "\n\n", for layout + BlankLine, } #[deriving(Clone)] @@ -38,9 +42,9 @@ pub struct Comment { } pub fn is_doc_comment(s: &str) -> bool { - (s.starts_with("///") && !is_line_non_doc_comment(s)) || + (s.starts_with("///") && super::is_doc_comment(s)) || s.starts_with("//!") || - (s.starts_with("/**") && !is_block_non_doc_comment(s)) || + (s.starts_with("/**") && is_block_doc_comment(s)) || s.starts_with("/*!") } @@ -198,9 +202,9 @@ fn read_line_comments(rdr: &mut StringReader, code_to_the_left: bool, } } -// Returns None if the first col chars of s contain a non-whitespace char. -// Otherwise returns Some(k) where k is first char offset after that leading -// whitespace. Note k may be outside bounds of s. +/// Returns None if the first col chars of s contain a non-whitespace char. +/// Otherwise returns Some(k) where k is first char offset after that leading +/// whitespace. Note k may be outside bounds of s. fn all_whitespace(s: &str, col: CharPos) -> Option<uint> { let len = s.len(); let mut col = col.to_uint(); @@ -256,7 +260,7 @@ fn read_block_comment(rdr: &mut StringReader, rdr.bump(); rdr.bump(); } - if !is_block_non_doc_comment(curr_line.as_slice()) { + if is_block_doc_comment(curr_line.as_slice()) { return } assert!(!curr_line.as_slice().contains_char('\n')); diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs index 1e72b2de20f..0aaddacfab6 100644 --- a/src/libsyntax/parse/lexer/mod.rs +++ b/src/libsyntax/parse/lexer/mod.rs @@ -18,7 +18,6 @@ use parse::token::{str_to_ident}; use std::char; use std::mem::replace; -use std::num::from_str_radix; use std::rc::Rc; use std::str; @@ -44,13 +43,13 @@ pub struct TokenAndSpan { pub struct StringReader<'a> { pub span_diagnostic: &'a SpanHandler, - // The absolute offset within the codemap of the next character to read + /// The absolute offset within the codemap of the next character to read pub pos: BytePos, - // The absolute offset within the codemap of the last character read(curr) + /// The absolute offset within the codemap of the last character read(curr) pub last_pos: BytePos, - // The column of the next character to read + /// The column of the next character to read pub col: CharPos, - // The last character to be read + /// The last character to be read pub curr: Option<char>, pub filemap: Rc<codemap::FileMap>, /* cached: */ @@ -60,7 +59,7 @@ pub struct StringReader<'a> { impl<'a> Reader for StringReader<'a> { fn is_eof(&self) -> bool { self.curr.is_none() } - // return the next token. EFFECT: advances the string_reader. + /// Return the next token. EFFECT: advances the string_reader. fn next_token(&mut self) -> TokenAndSpan { let ret_val = TokenAndSpan { tok: replace(&mut self.peek_tok, token::UNDERSCORE), @@ -90,7 +89,7 @@ impl<'a> Reader for TtReader<'a> { } fn next_token(&mut self) -> TokenAndSpan { let r = tt_next_token(self); - debug!("TtReader: r={:?}", r); + debug!("TtReader: r={}", r); r } fn fatal(&self, m: &str) -> ! { @@ -188,7 +187,7 @@ impl<'a> StringReader<'a> { /// Advance peek_tok and peek_span to refer to the next token, and /// possibly update the interner. fn advance_token(&mut self) { - match self.consume_whitespace_and_comments() { + match self.scan_whitespace_or_comment() { Some(comment) => { self.peek_span = comment.sp; self.peek_tok = comment.tok; @@ -217,6 +216,20 @@ impl<'a> StringReader<'a> { self.with_str_from_to(start, self.last_pos, f) } + /// Create a Name from a given offset to the current offset, each + /// adjusted 1 towards each other (assumes that on either side there is a + /// single-byte delimiter). + pub fn name_from(&self, start: BytePos) -> ast::Name { + debug!("taking an ident from {} to {}", start, self.last_pos); + self.with_str_from(start, token::intern) + } + + /// As name_from, with an explicit endpoint. + pub fn name_from_to(&self, start: BytePos, end: BytePos) -> ast::Name { + debug!("taking an ident from {} to {}", start, end); + self.with_str_from_to(start, end, token::intern) + } + /// Calls `f` with a string slice of the source text spanning from `start` /// up to but excluding `end`. fn with_str_from_to<T>(&self, start: BytePos, end: BytePos, f: |s: &str| -> T) -> T { @@ -326,8 +339,7 @@ impl<'a> StringReader<'a> { /// PRECONDITION: self.curr is not whitespace /// Eats any kind of comment. - /// Returns a Some(sugared-doc-attr) if one exists, None otherwise - fn consume_any_line_comment(&mut self) -> Option<TokenAndSpan> { + fn scan_comment(&mut self) -> Option<TokenAndSpan> { match self.curr { Some(c) => { if c.is_whitespace() { @@ -362,28 +374,32 @@ impl<'a> StringReader<'a> { } self.bump(); } - let ret = self.with_str_from(start_bpos, |string| { + return self.with_str_from(start_bpos, |string| { // but comments with only more "/"s are not - if !is_line_non_doc_comment(string) { - Some(TokenAndSpan{ - tok: token::DOC_COMMENT(str_to_ident(string)), - sp: codemap::mk_sp(start_bpos, self.last_pos) - }) + let tok = if is_doc_comment(string) { + token::DOC_COMMENT(token::intern(string)) } else { - None - } - }); + token::COMMENT + }; - if ret.is_some() { - return ret; - } + return Some(TokenAndSpan{ + tok: tok, + sp: codemap::mk_sp(start_bpos, self.last_pos) + }); + }); } else { + let start_bpos = self.last_pos - BytePos(2); while !self.curr_is('\n') && !self.is_eof() { self.bump(); } + return Some(TokenAndSpan { + tok: token::COMMENT, + sp: codemap::mk_sp(start_bpos, self.last_pos) + }); } - // Restart whitespace munch. - self.consume_whitespace_and_comments() } - Some('*') => { self.bump(); self.bump(); self.consume_block_comment() } + Some('*') => { + self.bump(); self.bump(); + self.scan_block_comment() + } _ => None } } else if self.curr_is('#') { @@ -399,9 +415,15 @@ impl<'a> StringReader<'a> { let cmap = CodeMap::new(); cmap.files.borrow_mut().push(self.filemap.clone()); let loc = cmap.lookup_char_pos_adj(self.last_pos); + debug!("Skipping a shebang"); if loc.line == 1u && loc.col == CharPos(0u) { + // FIXME: Add shebang "token", return it + let start = self.last_pos; while !self.curr_is('\n') && !self.is_eof() { self.bump(); } - return self.consume_whitespace_and_comments(); + return Some(TokenAndSpan { + tok: token::SHEBANG(self.name_from(start)), + sp: codemap::mk_sp(start, self.last_pos) + }); } } None @@ -410,15 +432,33 @@ impl<'a> StringReader<'a> { } } - /// EFFECT: eats whitespace and comments. - /// Returns a Some(sugared-doc-attr) if one exists, None otherwise. - fn consume_whitespace_and_comments(&mut self) -> Option<TokenAndSpan> { - while is_whitespace(self.curr) { self.bump(); } - return self.consume_any_line_comment(); + /// If there is whitespace, shebang, or a comment, scan it. Otherwise, + /// return None. + fn scan_whitespace_or_comment(&mut self) -> Option<TokenAndSpan> { + match self.curr.unwrap_or('\0') { + // # to handle shebang at start of file -- this is the entry point + // for skipping over all "junk" + '/' | '#' => { + let c = self.scan_comment(); + debug!("scanning a comment {}", c); + c + }, + c if is_whitespace(Some(c)) => { + let start_bpos = self.last_pos; + while is_whitespace(self.curr) { self.bump(); } + let c = Some(TokenAndSpan { + tok: token::WS, + sp: codemap::mk_sp(start_bpos, self.last_pos) + }); + debug!("scanning whitespace: {}", c); + c + }, + _ => None + } } - // might return a sugared-doc-attr - fn consume_block_comment(&mut self) -> Option<TokenAndSpan> { + /// Might return a sugared-doc-attr + fn scan_block_comment(&mut self) -> Option<TokenAndSpan> { // block comments starting with "/**" or "/*!" are doc-comments let is_doc_comment = self.curr_is('*') || self.curr_is('!'); let start_bpos = self.last_pos - BytePos(2); @@ -453,228 +493,132 @@ impl<'a> StringReader<'a> { self.bump(); } - let res = if is_doc_comment { - self.with_str_from(start_bpos, |string| { - // but comments with only "*"s between two "/"s are not - if !is_block_non_doc_comment(string) { - let string = if has_cr { - self.translate_crlf(start_bpos, string, - "bare CR not allowed in block doc-comment") - } else { string.into_maybe_owned() }; - Some(TokenAndSpan{ - tok: token::DOC_COMMENT(str_to_ident(string.as_slice())), - sp: codemap::mk_sp(start_bpos, self.last_pos) - }) - } else { - None - } - }) - } else { - None - }; - - // restart whitespace munch. - if res.is_some() { res } else { self.consume_whitespace_and_comments() } - } - - fn scan_exponent(&mut self, start_bpos: BytePos) -> Option<String> { - // \x00 hits the `return None` case immediately, so this is fine. - let mut c = self.curr.unwrap_or('\x00'); - let mut rslt = String::new(); - if c == 'e' || c == 'E' { - rslt.push_char(c); - self.bump(); - c = self.curr.unwrap_or('\x00'); - if c == '-' || c == '+' { - rslt.push_char(c); - self.bump(); - } - let exponent = self.scan_digits(10u); - if exponent.len() > 0u { - rslt.push_str(exponent.as_slice()); - return Some(rslt); + self.with_str_from(start_bpos, |string| { + // but comments with only "*"s between two "/"s are not + let tok = if is_block_doc_comment(string) { + let string = if has_cr { + self.translate_crlf(start_bpos, string, + "bare CR not allowed in block doc-comment") + } else { string.into_maybe_owned() }; + token::DOC_COMMENT(token::intern(string.as_slice())) } else { - let last_bpos = self.last_pos; - self.err_span_(start_bpos, last_bpos, "scan_exponent: bad fp literal"); - rslt.push_str("1"); // arbitrary placeholder exponent - return Some(rslt); - } - } else { - return None::<String>; - } + token::COMMENT + }; + + Some(TokenAndSpan{ + tok: tok, + sp: codemap::mk_sp(start_bpos, self.last_pos) + }) + }) } - fn scan_digits(&mut self, radix: uint) -> String { - let mut rslt = String::new(); + /// Scan through any digits (base `radix`) or underscores, and return how + /// many digits there were. + fn scan_digits(&mut self, radix: uint) -> uint { + let mut len = 0u; loop { let c = self.curr; - if c == Some('_') { self.bump(); continue; } + if c == Some('_') { debug!("skipping a _"); self.bump(); continue; } match c.and_then(|cc| char::to_digit(cc, radix)) { - Some(_) => { - rslt.push_char(c.unwrap()); - self.bump(); - } - _ => return rslt + Some(_) => { + debug!("{} in scan_digits", c); + len += 1; + self.bump(); + } + _ => return len } }; } - fn check_float_base(&mut self, start_bpos: BytePos, last_bpos: BytePos, base: uint) { - match base { - 16u => self.err_span_(start_bpos, last_bpos, - "hexadecimal float literal is not supported"), - 8u => self.err_span_(start_bpos, last_bpos, "octal float literal is not supported"), - 2u => self.err_span_(start_bpos, last_bpos, "binary float literal is not supported"), - _ => () - } - } - + /// Lex a LIT_INTEGER or a LIT_FLOAT fn scan_number(&mut self, c: char) -> token::Token { - let mut num_str; - let mut base = 10u; - let mut c = c; - let mut n = self.nextch().unwrap_or('\x00'); + let mut num_digits; + let mut base = 10; let start_bpos = self.last_pos; - if c == '0' && n == 'x' { - self.bump(); - self.bump(); - base = 16u; - } else if c == '0' && n == 'o' { - self.bump(); - self.bump(); - base = 8u; - } else if c == '0' && n == 'b' { - self.bump(); - self.bump(); - base = 2u; - } - num_str = self.scan_digits(base); - c = self.curr.unwrap_or('\x00'); - self.nextch(); - if c == 'u' || c == 'i' { - enum Result { Signed(ast::IntTy), Unsigned(ast::UintTy) } - let signed = c == 'i'; - let mut tp = { - if signed { Signed(ast::TyI) } - else { Unsigned(ast::TyU) } - }; - self.bump(); - c = self.curr.unwrap_or('\x00'); - if c == '8' { - self.bump(); - tp = if signed { Signed(ast::TyI8) } - else { Unsigned(ast::TyU8) }; - } - n = self.nextch().unwrap_or('\x00'); - if c == '1' && n == '6' { - self.bump(); - self.bump(); - tp = if signed { Signed(ast::TyI16) } - else { Unsigned(ast::TyU16) }; - } else if c == '3' && n == '2' { - self.bump(); - self.bump(); - tp = if signed { Signed(ast::TyI32) } - else { Unsigned(ast::TyU32) }; - } else if c == '6' && n == '4' { - self.bump(); - self.bump(); - tp = if signed { Signed(ast::TyI64) } - else { Unsigned(ast::TyU64) }; - } - if num_str.len() == 0u { - let last_bpos = self.last_pos; - self.err_span_(start_bpos, last_bpos, "no valid digits found for number"); - num_str = "1".to_string(); - } - let parsed = match from_str_radix::<u64>(num_str.as_slice(), - base as uint) { - Some(p) => p, - None => { - let last_bpos = self.last_pos; - self.err_span_(start_bpos, last_bpos, "int literal is too large"); - 1 - } - }; - match tp { - Signed(t) => return token::LIT_INT(parsed as i64, t), - Unsigned(t) => return token::LIT_UINT(parsed, t) + self.bump(); + + if c == '0' { + match self.curr.unwrap_or('\0') { + 'b' => { self.bump(); base = 2; num_digits = self.scan_digits(2); } + 'o' => { self.bump(); base = 8; num_digits = self.scan_digits(8); } + 'x' => { self.bump(); base = 16; num_digits = self.scan_digits(16); } + '0'..'9' | '_' | '.' => { + num_digits = self.scan_digits(10) + 1; + } + 'u' | 'i' => { + self.scan_int_suffix(); + return token::LIT_INTEGER(self.name_from(start_bpos)); + }, + 'f' => { + let last_pos = self.last_pos; + self.scan_float_suffix(); + self.check_float_base(start_bpos, last_pos, base); + return token::LIT_FLOAT(self.name_from(start_bpos)); + } + _ => { + // just a 0 + return token::LIT_INTEGER(self.name_from(start_bpos)); + } } + } else if c.is_digit_radix(10) { + num_digits = self.scan_digits(10) + 1; + } else { + num_digits = 0; } - let mut is_float = false; - if self.curr_is('.') && !(ident_start(self.nextch()) || self.nextch_is('.')) { - is_float = true; - self.bump(); - let dec_part = self.scan_digits(10u); - num_str.push_char('.'); - num_str.push_str(dec_part.as_slice()); - } - match self.scan_exponent(start_bpos) { - Some(ref s) => { - is_float = true; - num_str.push_str(s.as_slice()); - } - None => () + + if num_digits == 0 { + self.err_span_(start_bpos, self.last_pos, "no valid digits found for number"); + // eat any suffix + self.scan_int_suffix(); + return token::LIT_INTEGER(token::intern("0")); } - if self.curr_is('f') { + // might be a float, but don't be greedy if this is actually an + // integer literal followed by field/method access or a range pattern + // (`0..2` and `12.foo()`) + if self.curr_is('.') && !self.nextch_is('.') && !self.nextch().unwrap_or('\0') + .is_XID_start() { + // might have stuff after the ., and if it does, it needs to start + // with a number self.bump(); - c = self.curr.unwrap_or('\x00'); - n = self.nextch().unwrap_or('\x00'); - if c == '3' && n == '2' { - self.bump(); - self.bump(); - let last_bpos = self.last_pos; - self.check_float_base(start_bpos, last_bpos, base); - return token::LIT_FLOAT(str_to_ident(num_str.as_slice()), - ast::TyF32); - } else if c == '6' && n == '4' { - self.bump(); - self.bump(); - let last_bpos = self.last_pos; - self.check_float_base(start_bpos, last_bpos, base); - return token::LIT_FLOAT(str_to_ident(num_str.as_slice()), - ast::TyF64); - /* FIXME (#2252): if this is out of range for either a - 32-bit or 64-bit float, it won't be noticed till the - back-end. */ + if self.curr.unwrap_or('\0').is_digit_radix(10) { + self.scan_digits(10); + self.scan_float_exponent(); + self.scan_float_suffix(); } - let last_bpos = self.last_pos; - self.err_span_(start_bpos, last_bpos, "expected `f32` or `f64` suffix"); - } - if is_float { - let last_bpos = self.last_pos; - self.check_float_base(start_bpos, last_bpos, base); - return token::LIT_FLOAT_UNSUFFIXED(str_to_ident( - num_str.as_slice())); + let last_pos = self.last_pos; + self.check_float_base(start_bpos, last_pos, base); + return token::LIT_FLOAT(self.name_from(start_bpos)); + } else if self.curr_is('f') { + // or it might be an integer literal suffixed as a float + self.scan_float_suffix(); + let last_pos = self.last_pos; + self.check_float_base(start_bpos, last_pos, base); + return token::LIT_FLOAT(self.name_from(start_bpos)); } else { - if num_str.len() == 0u { - let last_bpos = self.last_pos; - self.err_span_(start_bpos, last_bpos, "no valid digits found for number"); - num_str = "1".to_string(); + // it might be a float if it has an exponent + if self.curr_is('e') || self.curr_is('E') { + self.scan_float_exponent(); + self.scan_float_suffix(); + let last_pos = self.last_pos; + self.check_float_base(start_bpos, last_pos, base); + return token::LIT_FLOAT(self.name_from(start_bpos)); } - let parsed = match from_str_radix::<u64>(num_str.as_slice(), - base as uint) { - Some(p) => p, - None => { - let last_bpos = self.last_pos; - self.err_span_(start_bpos, last_bpos, "int literal is too large"); - 1 - } - }; - - debug!("lexing {} as an unsuffixed integer literal", - num_str.as_slice()); - return token::LIT_INT_UNSUFFIXED(parsed as i64); + // but we certainly have an integer! + self.scan_int_suffix(); + return token::LIT_INTEGER(self.name_from(start_bpos)); } } - - fn scan_numeric_escape(&mut self, n_hex_digits: uint, delim: char) -> char { - let mut accum_int = 0u32; + /// Scan over `n_digits` hex digits, stopping at `delim`, reporting an + /// error if too many or too few digits are encountered. + fn scan_hex_digits(&mut self, n_digits: uint, delim: char) -> bool { + debug!("scanning {} digits until {}", n_digits, delim); let start_bpos = self.last_pos; - for _ in range(0, n_hex_digits) { + let mut accum_int = 0; + + for _ in range(0, n_digits) { if self.is_eof() { let last_bpos = self.last_pos; self.fatal_span_(start_bpos, last_bpos, "unterminated numeric character escape"); @@ -695,11 +639,11 @@ impl<'a> StringReader<'a> { } match char::from_u32(accum_int) { - Some(x) => x, + Some(_) => true, None => { let last_bpos = self.last_pos; self.err_span_(start_bpos, last_bpos, "illegal numeric character escape"); - '?' + false } } } @@ -707,8 +651,10 @@ impl<'a> StringReader<'a> { /// Scan for a single (possibly escaped) byte or char /// in a byte, (non-raw) byte string, char, or (non-raw) string literal. /// `start` is the position of `first_source_char`, which is already consumed. + /// + /// Returns true if there was a valid char/byte, false otherwise. fn scan_char_or_byte(&mut self, start: BytePos, first_source_char: char, - ascii_only: bool, delim: char) -> Option<char> { + ascii_only: bool, delim: char) -> bool { match first_source_char { '\\' => { // '\X' for some X must be a character constant: @@ -718,24 +664,18 @@ impl<'a> StringReader<'a> { match escaped { None => {}, // EOF here is an error that will be checked later. Some(e) => { - return Some(match e { - 'n' => '\n', - 'r' => '\r', - 't' => '\t', - '\\' => '\\', - '\'' => '\'', - '"' => '"', - '0' => '\x00', - 'x' => self.scan_numeric_escape(2u, delim), - 'u' if !ascii_only => self.scan_numeric_escape(4u, delim), - 'U' if !ascii_only => self.scan_numeric_escape(8u, delim), + return match e { + 'n' | 'r' | 't' | '\\' | '\'' | '"' | '0' => true, + 'x' => self.scan_hex_digits(2u, delim), + 'u' if !ascii_only => self.scan_hex_digits(4u, delim), + 'U' if !ascii_only => self.scan_hex_digits(8u, delim), '\n' if delim == '"' => { self.consume_whitespace(); - return None + true }, '\r' if delim == '"' && self.curr_is('\n') => { self.consume_whitespace(); - return None + true } c => { let last_pos = self.last_pos; @@ -744,9 +684,9 @@ impl<'a> StringReader<'a> { if ascii_only { "unknown byte escape" } else { "unknown character escape" }, c); - c + false } - }) + } } } } @@ -757,14 +697,16 @@ impl<'a> StringReader<'a> { if ascii_only { "byte constant must be escaped" } else { "character constant must be escaped" }, first_source_char); + return false; } '\r' => { if self.curr_is('\n') { self.bump(); - return Some('\n'); + return true; } else { self.err_span_(start, self.last_pos, "bare CR not allowed in string, use \\r instead"); + return false; } } _ => if ascii_only && first_source_char > '\x7F' { @@ -773,9 +715,84 @@ impl<'a> StringReader<'a> { start, last_pos, "byte constant must be ASCII. \ Use a \\xHH escape for a non-ASCII byte", first_source_char); + return false; + } + } + true + } + + /// Scan over an int literal suffix. + fn scan_int_suffix(&mut self) { + match self.curr { + Some('i') | Some('u') => { + self.bump(); + + if self.curr_is('8') { + self.bump(); + } else if self.curr_is('1') { + if !self.nextch_is('6') { + self.err_span_(self.last_pos, self.pos, + "illegal int suffix"); + } else { + self.bump(); self.bump(); + } + } else if self.curr_is('3') { + if !self.nextch_is('2') { + self.err_span_(self.last_pos, self.pos, + "illegal int suffix"); + } else { + self.bump(); self.bump(); + } + } else if self.curr_is('6') { + if !self.nextch_is('4') { + self.err_span_(self.last_pos, self.pos, + "illegal int suffix"); + } else { + self.bump(); self.bump(); + } + } + }, + _ => { } + } + } + + /// Scan over a float literal suffix + fn scan_float_suffix(&mut self) { + if self.curr_is('f') { + if (self.nextch_is('3') && self.nextnextch_is('2')) + || (self.nextch_is('6') && self.nextnextch_is('4')) { + self.bump(); + self.bump(); + self.bump(); + } else { + self.err_span_(self.last_pos, self.pos, "illegal float suffix"); } } - Some(first_source_char) + } + + /// Scan over a float exponent. + fn scan_float_exponent(&mut self) { + if self.curr_is('e') || self.curr_is('E') { + self.bump(); + if self.curr_is('-') || self.curr_is('+') { + self.bump(); + } + if self.scan_digits(10) == 0 { + self.err_span_(self.last_pos, self.pos, "expected at least one digit in exponent") + } + } + } + + /// Check that a base is valid for a floating literal, emitting a nice + /// error if it isn't. + fn check_float_base(&mut self, start_bpos: BytePos, last_bpos: BytePos, base: uint) { + match base { + 16u => self.err_span_(start_bpos, last_bpos, "hexadecimal float literal is not \ + supported"), + 8u => self.err_span_(start_bpos, last_bpos, "octal float literal is not supported"), + 2u => self.err_span_(start_bpos, last_bpos, "binary float literal is not supported"), + _ => () + } } fn binop(&mut self, op: token::BinOp) -> token::Token { @@ -910,7 +927,7 @@ impl<'a> StringReader<'a> { let start = self.last_pos; // the eof will be picked up by the final `'` check below - let mut c2 = self.curr.unwrap_or('\x00'); + let c2 = self.curr.unwrap_or('\x00'); self.bump(); // If the character is an ident start not followed by another single @@ -953,7 +970,7 @@ impl<'a> StringReader<'a> { } // Otherwise it is a character constant: - c2 = self.scan_char_or_byte(start, c2, /* ascii_only = */ false, '\'').unwrap(); + let valid = self.scan_char_or_byte(start, c2, /* ascii_only = */ false, '\''); if !self.curr_is('\'') { let last_bpos = self.last_pos; self.fatal_span_verbose( @@ -963,118 +980,23 @@ impl<'a> StringReader<'a> { start - BytePos(1), last_bpos, "unterminated character constant".to_string()); } + let id = if valid { self.name_from(start) } else { token::intern("0") }; self.bump(); // advance curr past token - return token::LIT_CHAR(c2); + return token::LIT_CHAR(id); } 'b' => { self.bump(); return match self.curr { - Some('\'') => parse_byte(self), - Some('"') => parse_byte_string(self), - Some('r') => parse_raw_byte_string(self), + Some('\'') => self.scan_byte(), + Some('"') => self.scan_byte_string(), + Some('r') => self.scan_raw_byte_string(), _ => unreachable!() // Should have been a token::IDENT above. }; - fn parse_byte(self_: &mut StringReader) -> token::Token { - self_.bump(); - let start = self_.last_pos; - - // the eof will be picked up by the final `'` check below - let mut c2 = self_.curr.unwrap_or('\x00'); - self_.bump(); - - c2 = self_.scan_char_or_byte(start, c2, /* ascii_only = */ true, '\'').unwrap(); - if !self_.curr_is('\'') { - // Byte offsetting here is okay because the - // character before position `start` are an - // ascii single quote and ascii 'b'. - let last_pos = self_.last_pos; - self_.fatal_span_verbose( - start - BytePos(2), last_pos, - "unterminated byte constant".to_string()); - } - self_.bump(); // advance curr past token - return token::LIT_BYTE(c2 as u8); - } - - fn parse_byte_string(self_: &mut StringReader) -> token::Token { - self_.bump(); - let start = self_.last_pos; - let mut value = Vec::new(); - while !self_.curr_is('"') { - if self_.is_eof() { - let last_pos = self_.last_pos; - self_.fatal_span_(start, last_pos, - "unterminated double quote byte string"); - } - - let ch_start = self_.last_pos; - let ch = self_.curr.unwrap(); - self_.bump(); - self_.scan_char_or_byte(ch_start, ch, /* ascii_only = */ true, '"') - .map(|ch| value.push(ch as u8)); - } - self_.bump(); - return token::LIT_BINARY(Rc::new(value)); - } - - fn parse_raw_byte_string(self_: &mut StringReader) -> token::Token { - let start_bpos = self_.last_pos; - self_.bump(); - let mut hash_count = 0u; - while self_.curr_is('#') { - self_.bump(); - hash_count += 1; - } - - if self_.is_eof() { - let last_pos = self_.last_pos; - self_.fatal_span_(start_bpos, last_pos, "unterminated raw string"); - } else if !self_.curr_is('"') { - let last_pos = self_.last_pos; - let ch = self_.curr.unwrap(); - self_.fatal_span_char(start_bpos, last_pos, - "only `#` is allowed in raw string delimitation; \ - found illegal character", - ch); - } - self_.bump(); - let content_start_bpos = self_.last_pos; - let mut content_end_bpos; - 'outer: loop { - match self_.curr { - None => { - let last_pos = self_.last_pos; - self_.fatal_span_(start_bpos, last_pos, "unterminated raw string") - }, - Some('"') => { - content_end_bpos = self_.last_pos; - for _ in range(0, hash_count) { - self_.bump(); - if !self_.curr_is('#') { - continue 'outer; - } - } - break; - }, - Some(c) => if c > '\x7F' { - let last_pos = self_.last_pos; - self_.err_span_char( - last_pos, last_pos, "raw byte string must be ASCII", c); - } - } - self_.bump(); - } - self_.bump(); - let bytes = self_.with_str_from_to(content_start_bpos, - content_end_bpos, - |s| s.as_bytes().to_owned()); - return token::LIT_BINARY_RAW(Rc::new(bytes), hash_count); - } } '"' => { - let mut accum_str = String::new(); let start_bpos = self.last_pos; + let mut valid = true; self.bump(); while !self.curr_is('"') { if self.is_eof() { @@ -1085,11 +1007,13 @@ impl<'a> StringReader<'a> { let ch_start = self.last_pos; let ch = self.curr.unwrap(); self.bump(); - self.scan_char_or_byte(ch_start, ch, /* ascii_only = */ false, '"') - .map(|ch| accum_str.push_char(ch)); + valid &= self.scan_char_or_byte(ch_start, ch, /* ascii_only = */ false, '"'); } + // adjust for the ACSII " at the start of the literal + let id = if valid { self.name_from(start_bpos + BytePos(1)) } + else { token::intern("??") }; self.bump(); - return token::LIT_STR(str_to_ident(accum_str.as_slice())); + return token::LIT_STR(id); } 'r' => { let start_bpos = self.last_pos; @@ -1114,7 +1038,7 @@ impl<'a> StringReader<'a> { self.bump(); let content_start_bpos = self.last_pos; let mut content_end_bpos; - let mut has_cr = false; + let mut valid = true; 'outer: loop { if self.is_eof() { let last_bpos = self.last_pos; @@ -1137,23 +1061,26 @@ impl<'a> StringReader<'a> { } } break; - } + }, '\r' => { - has_cr = true; + if !self.nextch_is('\n') { + let last_bpos = self.last_pos; + self.err_span_(start_bpos, last_bpos, "bare CR not allowed in raw \ + string, use \\r instead"); + valid = false; + } } _ => () } self.bump(); } self.bump(); - let str_content = self.with_str_from_to(content_start_bpos, content_end_bpos, |string| { - let string = if has_cr { - self.translate_crlf(content_start_bpos, string, - "bare CR not allowed in raw string") - } else { string.into_maybe_owned() }; - str_to_ident(string.as_slice()) - }); - return token::LIT_STR_RAW(str_content, hash_count); + let id = if valid { + self.name_from_to(content_start_bpos, content_end_bpos) + } else { + token::intern("??") + }; + return token::LIT_STR_RAW(id, hash_count); } '-' => { if self.nextch_is('>') { @@ -1221,6 +1148,104 @@ impl<'a> StringReader<'a> { // consider shebangs comments, but not inner attributes || (self.curr_is('#') && self.nextch_is('!') && !self.nextnextch_is('[')) } + + fn scan_byte(&mut self) -> token::Token { + self.bump(); + let start = self.last_pos; + + // the eof will be picked up by the final `'` check below + let c2 = self.curr.unwrap_or('\x00'); + self.bump(); + + let valid = self.scan_char_or_byte(start, c2, /* ascii_only = */ true, '\''); + if !self.curr_is('\'') { + // Byte offsetting here is okay because the + // character before position `start` are an + // ascii single quote and ascii 'b'. + let last_pos = self.last_pos; + self.fatal_span_verbose( + start - BytePos(2), last_pos, + "unterminated byte constant".to_string()); + } + + let id = if valid { self.name_from(start) } else { token::intern("??") }; + self.bump(); // advance curr past token + return token::LIT_BYTE(id); + } + + fn scan_byte_string(&mut self) -> token::Token { + self.bump(); + let start = self.last_pos; + let mut valid = true; + + while !self.curr_is('"') { + if self.is_eof() { + let last_pos = self.last_pos; + self.fatal_span_(start, last_pos, + "unterminated double quote byte string"); + } + + let ch_start = self.last_pos; + let ch = self.curr.unwrap(); + self.bump(); + valid &= self.scan_char_or_byte(ch_start, ch, /* ascii_only = */ true, '"'); + } + let id = if valid { self.name_from(start) } else { token::intern("??") }; + self.bump(); + return token::LIT_BINARY(id); + } + + fn scan_raw_byte_string(&mut self) -> token::Token { + let start_bpos = self.last_pos; + self.bump(); + let mut hash_count = 0u; + while self.curr_is('#') { + self.bump(); + hash_count += 1; + } + + if self.is_eof() { + let last_pos = self.last_pos; + self.fatal_span_(start_bpos, last_pos, "unterminated raw string"); + } else if !self.curr_is('"') { + let last_pos = self.last_pos; + let ch = self.curr.unwrap(); + self.fatal_span_char(start_bpos, last_pos, + "only `#` is allowed in raw string delimitation; \ + found illegal character", + ch); + } + self.bump(); + let content_start_bpos = self.last_pos; + let mut content_end_bpos; + 'outer: loop { + match self.curr { + None => { + let last_pos = self.last_pos; + self.fatal_span_(start_bpos, last_pos, "unterminated raw string") + }, + Some('"') => { + content_end_bpos = self.last_pos; + for _ in range(0, hash_count) { + self.bump(); + if !self.curr_is('#') { + continue 'outer; + } + } + break; + }, + Some(c) => if c > '\x7F' { + let last_pos = self.last_pos; + self.err_span_char( + last_pos, last_pos, "raw byte string must be ASCII", c); + } + } + self.bump(); + } + self.bump(); + return token::LIT_BINARY_RAW(self.name_from_to(content_start_bpos, content_end_bpos), + hash_count); + } } pub fn is_whitespace(c: Option<char>) -> bool { @@ -1239,12 +1264,18 @@ fn in_range(c: Option<char>, lo: char, hi: char) -> bool { fn is_dec_digit(c: Option<char>) -> bool { return in_range(c, '0', '9'); } -pub fn is_line_non_doc_comment(s: &str) -> bool { - s.starts_with("////") +pub fn is_doc_comment(s: &str) -> bool { + let res = (s.starts_with("///") && *s.as_bytes().get(3).unwrap_or(&b' ') != b'/') + || s.starts_with("//!"); + debug!("is `{}` a doc comment? {}", s, res); + res } -pub fn is_block_non_doc_comment(s: &str) -> bool { - s.starts_with("/***") +pub fn is_block_doc_comment(s: &str) -> bool { + let res = (s.starts_with("/**") && *s.as_bytes().get(3).unwrap_or(&b' ') != b'*') + || s.starts_with("/*!"); + debug!("is `{}` a doc comment? {}", s, res); + res } fn ident_start(c: Option<char>) -> bool { @@ -1295,11 +1326,14 @@ mod test { "/* my source file */ \ fn main() { println!(\"zebra\"); }\n".to_string()); let id = str_to_ident("fn"); + assert_eq!(string_reader.next_token().tok, token::COMMENT); + assert_eq!(string_reader.next_token().tok, token::WS); let tok1 = string_reader.next_token(); let tok2 = TokenAndSpan{ tok:token::IDENT(id, false), sp:Span {lo:BytePos(21),hi:BytePos(23),expn_info: None}}; assert_eq!(tok1,tok2); + assert_eq!(string_reader.next_token().tok, token::WS); // the 'main' id is already read: assert_eq!(string_reader.last_pos.clone(), BytePos(28)); // read another token: @@ -1328,6 +1362,7 @@ mod test { #[test] fn doublecolonparsing () { check_tokenization(setup(&mk_sh(), "a b".to_string()), vec!(mk_ident("a",false), + token::WS, mk_ident("b",false))); } @@ -1341,6 +1376,7 @@ mod test { #[test] fn dcparsing_3 () { check_tokenization(setup(&mk_sh(), "a ::b".to_string()), vec!(mk_ident("a",false), + token::WS, token::MOD_SEP, mk_ident("b",false))); } @@ -1349,22 +1385,23 @@ mod test { check_tokenization(setup(&mk_sh(), "a:: b".to_string()), vec!(mk_ident("a",true), token::MOD_SEP, + token::WS, mk_ident("b",false))); } #[test] fn character_a() { assert_eq!(setup(&mk_sh(), "'a'".to_string()).next_token().tok, - token::LIT_CHAR('a')); + token::LIT_CHAR(token::intern("a"))); } #[test] fn character_space() { assert_eq!(setup(&mk_sh(), "' '".to_string()).next_token().tok, - token::LIT_CHAR(' ')); + token::LIT_CHAR(token::intern(" "))); } #[test] fn character_escaped() { assert_eq!(setup(&mk_sh(), "'\\n'".to_string()).next_token().tok, - token::LIT_CHAR('\n')); + token::LIT_CHAR(token::intern("\\n"))); } #[test] fn lifetime_name() { @@ -1376,19 +1413,23 @@ mod test { assert_eq!(setup(&mk_sh(), "r###\"\"#a\\b\x00c\"\"###".to_string()).next_token() .tok, - token::LIT_STR_RAW(token::str_to_ident("\"#a\\b\x00c\""), 3)); + token::LIT_STR_RAW(token::intern("\"#a\\b\x00c\""), 3)); } #[test] fn line_doc_comments() { - assert!(!is_line_non_doc_comment("///")); - assert!(!is_line_non_doc_comment("/// blah")); - assert!(is_line_non_doc_comment("////")); + assert!(is_doc_comment("///")); + assert!(is_doc_comment("/// blah")); + assert!(!is_doc_comment("////")); } #[test] fn nested_block_comments() { - assert_eq!(setup(&mk_sh(), - "/* /* */ */'a'".to_string()).next_token().tok, - token::LIT_CHAR('a')); + let sh = mk_sh(); + let mut lexer = setup(&sh, "/* /* */ */'a'".to_string()); + match lexer.next_token().tok { + token::COMMENT => { }, + _ => fail!("expected a comment!") + } + assert_eq!(lexer.next_token().tok, token::LIT_CHAR(token::intern("a"))); } } diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index fb4a23cf326..37c84c95af6 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -10,7 +10,6 @@ //! The main parser interface - use ast; use codemap::{Span, CodeMap, FileMap}; use diagnostic::{SpanHandler, mk_span_handler, default_handler, Auto}; @@ -32,7 +31,7 @@ pub mod common; pub mod classify; pub mod obsolete; -// info about a parsing session. +/// Info about a parsing session. pub struct ParseSess { pub span_diagnostic: SpanHandler, // better be the same as the one in the reader! /// Used to determine and report recursive mod inclusions @@ -241,14 +240,14 @@ pub fn file_to_filemap(sess: &ParseSess, path: &Path, spanopt: Option<Span>) unreachable!() } -// given a session and a string, add the string to -// the session's codemap and return the new filemap +/// Given a session and a string, add the string to +/// the session's codemap and return the new filemap pub fn string_to_filemap(sess: &ParseSess, source: String, path: String) -> Rc<FileMap> { sess.span_diagnostic.cm.new_filemap(path, source) } -// given a filemap, produce a sequence of token-trees +/// Given a filemap, produce a sequence of token-trees pub fn filemap_to_tts(sess: &ParseSess, filemap: Rc<FileMap>) -> Vec<ast::TokenTree> { // it appears to me that the cfg doesn't matter here... indeed, @@ -259,7 +258,7 @@ pub fn filemap_to_tts(sess: &ParseSess, filemap: Rc<FileMap>) p1.parse_all_token_trees() } -// given tts and cfg, produce a parser +/// Given tts and cfg, produce a parser pub fn tts_to_parser<'a>(sess: &'a ParseSess, tts: Vec<ast::TokenTree>, cfg: ast::CrateConfig) -> Parser<'a> { @@ -267,13 +266,354 @@ pub fn tts_to_parser<'a>(sess: &'a ParseSess, Parser::new(sess, cfg, box trdr) } -// abort if necessary +/// Abort if necessary pub fn maybe_aborted<T>(result: T, mut p: Parser) -> T { p.abort_if_errors(); result } +/// Parse a string representing a character literal into its final form. +/// Rather than just accepting/rejecting a given literal, unescapes it as +/// well. Can take any slice prefixed by a character escape. Returns the +/// character and the number of characters consumed. +pub fn char_lit(lit: &str) -> (char, int) { + use std::{num, char}; + + let mut chars = lit.chars(); + let c = match (chars.next(), chars.next()) { + (Some(c), None) if c != '\\' => return (c, 1), + (Some('\\'), Some(c)) => match c { + '"' => Some('"'), + 'n' => Some('\n'), + 'r' => Some('\r'), + 't' => Some('\t'), + '\\' => Some('\\'), + '\'' => Some('\''), + '0' => Some('\0'), + _ => { None } + }, + _ => fail!("lexer accepted invalid char escape `{}`", lit) + }; + + match c { + Some(x) => return (x, 2), + None => { } + } + + let msg = format!("lexer should have rejected a bad character escape {}", lit); + let msg2 = msg.as_slice(); + + let esc: |uint| -> Option<(char, int)> = |len| + num::from_str_radix(lit.slice(2, len), 16) + .and_then(char::from_u32) + .map(|x| (x, len as int)); + + // Unicode escapes + return match lit.as_bytes()[1] as char { + 'x' | 'X' => esc(4), + 'u' => esc(6), + 'U' => esc(10), + _ => None, + }.expect(msg2); +} + +/// Parse a string representing a string literal into its final form. Does +/// unescaping. +pub fn str_lit(lit: &str) -> String { + debug!("parse_str_lit: given {}", lit.escape_default()); + let mut res = String::with_capacity(lit.len()); + + // FIXME #8372: This could be a for-loop if it didn't borrow the iterator + let error = |i| format!("lexer should have rejected {} at {}", lit, i); + + /// Eat everything up to a non-whitespace + fn eat<'a>(it: &mut ::std::iter::Peekable<(uint, char), ::std::str::CharOffsets<'a>>) { + loop { + match it.peek().map(|x| x.val1()) { + Some(' ') | Some('\n') | Some('\r') | Some('\t') => { + it.next(); + }, + _ => { break; } + } + } + } + + let mut chars = lit.char_indices().peekable(); + loop { + match chars.next() { + Some((i, c)) => { + let em = error(i); + match c { + '\\' => { + if chars.peek().expect(em.as_slice()).val1() == '\n' { + eat(&mut chars); + } else if chars.peek().expect(em.as_slice()).val1() == '\r' { + chars.next(); + if chars.peek().expect(em.as_slice()).val1() != '\n' { + fail!("lexer accepted bare CR"); + } + eat(&mut chars); + } else { + // otherwise, a normal escape + let (c, n) = char_lit(lit.slice_from(i)); + for _ in range(0, n - 1) { // we don't need to move past the first \ + chars.next(); + } + res.push_char(c); + } + }, + '\r' => { + if chars.peek().expect(em.as_slice()).val1() != '\n' { + fail!("lexer accepted bare CR"); + } + chars.next(); + res.push_char('\n'); + } + c => res.push_char(c), + } + }, + None => break + } + } + + res.shrink_to_fit(); // probably not going to do anything, unless there was an escape. + debug!("parse_str_lit: returning {}", res); + res +} + +/// Parse a string representing a raw string literal into its final form. The +/// only operation this does is convert embedded CRLF into a single LF. +pub fn raw_str_lit(lit: &str) -> String { + debug!("raw_str_lit: given {}", lit.escape_default()); + let mut res = String::with_capacity(lit.len()); + + // FIXME #8372: This could be a for-loop if it didn't borrow the iterator + let mut chars = lit.chars().peekable(); + loop { + match chars.next() { + Some(c) => { + if c == '\r' { + if *chars.peek().unwrap() != '\n' { + fail!("lexer accepted bare CR"); + } + chars.next(); + res.push_char('\n'); + } else { + res.push_char(c); + } + }, + None => break + } + } + + res.shrink_to_fit(); + res +} + +pub fn float_lit(s: &str) -> ast::Lit_ { + debug!("float_lit: {}", s); + // FIXME #2252: bounds checking float literals is defered until trans + let s2 = s.chars().filter(|&c| c != '_').collect::<String>(); + let s = s2.as_slice(); + + let mut ty = None; + + if s.ends_with("f32") { + ty = Some(ast::TyF32); + } else if s.ends_with("f64") { + ty = Some(ast::TyF64); + } + + + match ty { + Some(t) => { + ast::LitFloat(token::intern_and_get_ident(s.slice_to(s.len() - t.suffix_len())), t) + }, + None => ast::LitFloatUnsuffixed(token::intern_and_get_ident(s)) + } +} + +/// Parse a string representing a byte literal into its final form. Similar to `char_lit` +pub fn byte_lit(lit: &str) -> (u8, uint) { + let err = |i| format!("lexer accepted invalid byte literal {} step {}", lit, i); + + if lit.len() == 1 { + (lit.as_bytes()[0], 1) + } else { + assert!(lit.as_bytes()[0] == b'\\', err(0i)); + let b = match lit.as_bytes()[1] { + b'"' => b'"', + b'n' => b'\n', + b'r' => b'\r', + b't' => b'\t', + b'\\' => b'\\', + b'\'' => b'\'', + b'0' => b'\0', + _ => { + match ::std::num::from_str_radix::<u64>(lit.slice(2, 4), 16) { + Some(c) => + if c > 0xFF { + fail!(err(2)) + } else { + return (c as u8, 4) + }, + None => fail!(err(3)) + } + } + }; + return (b, 2); + } +} + +pub fn binary_lit(lit: &str) -> Rc<Vec<u8>> { + let mut res = Vec::with_capacity(lit.len()); + + // FIXME #8372: This could be a for-loop if it didn't borrow the iterator + let error = |i| format!("lexer should have rejected {} at {}", lit, i); + + // binary literals *must* be ASCII, but the escapes don't have to be + let mut chars = lit.as_bytes().iter().enumerate().peekable(); + loop { + match chars.next() { + Some((i, &c)) => { + if c == b'\\' { + if *chars.peek().expect(error(i).as_slice()).val1() == b'\n' { + loop { + // eat everything up to a non-whitespace + match chars.peek().map(|x| *x.val1()) { + Some(b' ') | Some(b'\n') | Some(b'\r') | Some(b'\t') => { + chars.next(); + }, + _ => { break; } + } + } + } else { + // otherwise, a normal escape + let (c, n) = byte_lit(lit.slice_from(i)); + for _ in range(0, n - 1) { // we don't need to move past the first \ + chars.next(); + } + res.push(c); + } + } else { + res.push(c); + } + }, + None => { break; } + } + } + + Rc::new(res) +} + +pub fn integer_lit(s: &str, sd: &SpanHandler, sp: Span) -> ast::Lit_ { + // s can only be ascii, byte indexing is fine + + let s2 = s.chars().filter(|&c| c != '_').collect::<String>(); + let mut s = s2.as_slice(); + + debug!("parse_integer_lit: {}", s); + + if s.len() == 1 { + return ast::LitIntUnsuffixed((s.char_at(0)).to_digit(10).unwrap() as i64); + } + let mut base = 10; + let orig = s; + + #[deriving(Show)] + enum Result { + Nothing, + Signed(ast::IntTy), + Unsigned(ast::UintTy) + } + + impl Result { + fn suffix_len(&self) -> uint { + match *self { + Nothing => 0, + Signed(s) => s.suffix_len(), + Unsigned(u) => u.suffix_len() + } + } + } + + let mut ty = Nothing; + + + if s.char_at(0) == '0' { + match s.char_at(1) { + 'x' => base = 16, + 'o' => base = 8, + 'b' => base = 2, + _ => { } + } + } + + if base != 10 { + s = s.slice_from(2); + } + + let last = s.len() - 1; + match s.char_at(last) { + 'i' => ty = Signed(ast::TyI), + 'u' => ty = Unsigned(ast::TyU), + '8' => { + if s.len() > 2 { + match s.char_at(last - 1) { + 'i' => ty = Signed(ast::TyI8), + 'u' => ty = Unsigned(ast::TyU8), + _ => { } + } + } + }, + '6' => { + if s.len() > 3 && s.char_at(last - 1) == '1' { + match s.char_at(last - 2) { + 'i' => ty = Signed(ast::TyI16), + 'u' => ty = Unsigned(ast::TyU16), + _ => { } + } + } + }, + '2' => { + if s.len() > 3 && s.char_at(last - 1) == '3' { + match s.char_at(last - 2) { + 'i' => ty = Signed(ast::TyI32), + 'u' => ty = Unsigned(ast::TyU32), + _ => { } + } + } + }, + '4' => { + if s.len() > 3 && s.char_at(last - 1) == '6' { + match s.char_at(last - 2) { + 'i' => ty = Signed(ast::TyI64), + 'u' => ty = Unsigned(ast::TyU64), + _ => { } + } + } + }, + _ => { } + } + + + s = s.slice_to(s.len() - ty.suffix_len()); + + debug!("The suffix is {}, base {}, the new string is {}, the original \ + string was {}", ty, base, s, orig); + + let res: u64 = match ::std::num::from_str_radix(s, base) { + Some(r) => r, + None => { sd.span_err(sp, "int literal is too large"); 0 } + }; + + match ty { + Nothing => ast::LitIntUnsuffixed(res as i64), + Signed(t) => ast::LitInt(res as i64, t), + Unsigned(t) => ast::LitUint(res, t) + } +} #[cfg(test)] mod test { diff --git a/src/libsyntax/parse/obsolete.rs b/src/libsyntax/parse/obsolete.rs index 025684ae71e..cadae7ef12f 100644 --- a/src/libsyntax/parse/obsolete.rs +++ b/src/libsyntax/parse/obsolete.rs @@ -38,8 +38,8 @@ pub enum ObsoleteSyntax { pub trait ParserObsoleteMethods { /// Reports an obsolete syntax non-fatal error. fn obsolete(&mut self, sp: Span, kind: ObsoleteSyntax); - // Reports an obsolete syntax non-fatal error, and returns - // a placeholder expression + /// Reports an obsolete syntax non-fatal error, and returns + /// a placeholder expression fn obsolete_expr(&mut self, sp: Span, kind: ObsoleteSyntax) -> Gc<Expr>; fn report(&mut self, sp: Span, @@ -83,8 +83,8 @@ impl<'a> ParserObsoleteMethods for parser::Parser<'a> { self.report(sp, kind, kind_str, desc); } - // Reports an obsolete syntax non-fatal error, and returns - // a placeholder expression + /// Reports an obsolete syntax non-fatal error, and returns + /// a placeholder expression fn obsolete_expr(&mut self, sp: Span, kind: ObsoleteSyntax) -> Gc<Expr> { self.obsolete(sp, kind); self.mk_expr(sp.lo, sp.hi, ExprLit(box(GC) respan(sp, LitNil))) diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index ac4cbf3aa8e..743eeed9da5 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -33,8 +33,8 @@ use ast::{ForeignItem, ForeignItemStatic, ForeignItemFn, ForeignMod}; use ast::{Ident, NormalFn, Inherited, Item, Item_, ItemStatic}; use ast::{ItemEnum, ItemFn, ItemForeignMod, ItemImpl}; use ast::{ItemMac, ItemMod, ItemStruct, ItemTrait, ItemTy, Lit, Lit_}; -use ast::{LitBool, LitFloat, LitFloatUnsuffixed, LitInt, LitChar, LitByte, LitBinary}; -use ast::{LitIntUnsuffixed, LitNil, LitStr, LitUint, Local, LocalLet}; +use ast::{LitBool, LitChar, LitByte, LitBinary}; +use ast::{LitNil, LitStr, LitUint, Local, LocalLet}; use ast::{MutImmutable, MutMutable, Mac_, MacInvocTT, Matcher, MatchNonterminal}; use ast::{MatchSeq, MatchTok, Method, MutTy, BiMul, Mutability}; use ast::{NamedField, UnNeg, NoReturn, UnNot, P, Pat, PatEnum}; @@ -61,6 +61,7 @@ use ast_util::{as_prec, ident_to_path, lit_is_str, operator_prec}; use ast_util; use codemap::{Span, BytePos, Spanned, spanned, mk_sp}; use codemap; +use parse; use parse::attr::ParserAttr; use parse::classify; use parse::common::{SeqSep, seq_sep_none}; @@ -117,8 +118,8 @@ pub struct PathAndBounds { } enum ItemOrViewItem { - // Indicates a failure to parse any kind of item. The attributes are - // returned. + /// Indicates a failure to parse any kind of item. The attributes are + /// returned. IoviNone(Vec<Attribute>), IoviItem(Gc<Item>), IoviForeignItem(Gc<ForeignItem>), @@ -126,12 +127,12 @@ enum ItemOrViewItem { } -// Possibly accept an `INTERPOLATED` expression (a pre-parsed expression -// dropped into the token stream, which happens while parsing the -// result of macro expansion) -/* Placement of these is not as complex as I feared it would be. -The important thing is to make sure that lookahead doesn't balk -at INTERPOLATED tokens */ +/// Possibly accept an `INTERPOLATED` expression (a pre-parsed expression +/// dropped into the token stream, which happens while parsing the +/// result of macro expansion) +/// Placement of these is not as complex as I feared it would be. +/// The important thing is to make sure that lookahead doesn't balk +/// at INTERPOLATED tokens macro_rules! maybe_whole_expr ( ($p:expr) => ( { @@ -166,7 +167,7 @@ macro_rules! maybe_whole_expr ( ) ) -// As above, but for things other than expressions +/// As maybe_whole_expr, but for things other than expressions macro_rules! maybe_whole ( ($p:expr, $constructor:ident) => ( { @@ -287,14 +288,14 @@ struct ParsedItemsAndViewItems { pub struct Parser<'a> { pub sess: &'a ParseSess, - // the current token: + /// the current token: pub token: token::Token, - // the span of the current token: + /// the span of the current token: pub span: Span, - // the span of the prior token: + /// the span of the prior token: pub last_span: Span, pub cfg: CrateConfig, - // the previous token or None (only stashed sometimes). + /// the previous token or None (only stashed sometimes). pub last_token: Option<Box<token::Token>>, pub buffer: [TokenAndSpan, ..4], pub buffer_start: int, @@ -324,10 +325,24 @@ fn is_plain_ident_or_underscore(t: &token::Token) -> bool { is_plain_ident(t) || *t == token::UNDERSCORE } +/// Get a token the parser cares about +fn real_token(rdr: &mut Reader) -> TokenAndSpan { + let mut t = rdr.next_token(); + loop { + match t.tok { + token::WS | token::COMMENT | token::SHEBANG(_) => { + t = rdr.next_token(); + }, + _ => break + } + } + t +} + impl<'a> Parser<'a> { pub fn new(sess: &'a ParseSess, cfg: ast::CrateConfig, mut rdr: Box<Reader>) -> Parser<'a> { - let tok0 = rdr.next_token(); + let tok0 = real_token(rdr); let span = tok0.sp; let placeholder = TokenAndSpan { tok: token::UNDERSCORE, @@ -361,12 +376,13 @@ impl<'a> Parser<'a> { root_module_name: None, } } - // convert a token to a string using self's reader + + /// Convert a token to a string using self's reader pub fn token_to_string(token: &token::Token) -> String { token::to_string(token) } - // convert the current token to a string using self's reader + /// Convert the current token to a string using self's reader pub fn this_token_to_string(&mut self) -> String { Parser::token_to_string(&self.token) } @@ -383,8 +399,8 @@ impl<'a> Parser<'a> { self.fatal(format!("unexpected token: `{}`", this_token).as_slice()); } - // expect and consume the token t. Signal an error if - // the next token is not t. + /// Expect and consume the token t. Signal an error if + /// the next token is not t. pub fn expect(&mut self, t: &token::Token) { if self.token == *t { self.bump(); @@ -397,9 +413,9 @@ impl<'a> Parser<'a> { } } - // Expect next token to be edible or inedible token. If edible, - // then consume it; if inedible, then return without consuming - // anything. Signal a fatal error if next token is unexpected. + /// Expect next token to be edible or inedible token. If edible, + /// then consume it; if inedible, then return without consuming + /// anything. Signal a fatal error if next token is unexpected. pub fn expect_one_of(&mut self, edible: &[token::Token], inedible: &[token::Token]) { @@ -437,9 +453,9 @@ impl<'a> Parser<'a> { } } - // Check for erroneous `ident { }`; if matches, signal error and - // recover (without consuming any expected input token). Returns - // true if and only if input was consumed for recovery. + /// Check for erroneous `ident { }`; if matches, signal error and + /// recover (without consuming any expected input token). Returns + /// true if and only if input was consumed for recovery. pub fn check_for_erroneous_unit_struct_expecting(&mut self, expected: &[token::Token]) -> bool { if self.token == token::LBRACE && expected.iter().all(|t| *t != token::LBRACE) @@ -456,9 +472,9 @@ impl<'a> Parser<'a> { } } - // Commit to parsing a complete expression `e` expected to be - // followed by some token from the set edible + inedible. Recover - // from anticipated input errors, discarding erroneous characters. + /// Commit to parsing a complete expression `e` expected to be + /// followed by some token from the set edible + inedible. Recover + /// from anticipated input errors, discarding erroneous characters. pub fn commit_expr(&mut self, e: Gc<Expr>, edible: &[token::Token], inedible: &[token::Token]) { debug!("commit_expr {:?}", e); @@ -479,9 +495,9 @@ impl<'a> Parser<'a> { self.commit_expr(e, &[edible], &[]) } - // Commit to parsing a complete statement `s`, which expects to be - // followed by some token from the set edible + inedible. Check - // for recoverable input errors, discarding erroneous characters. + /// Commit to parsing a complete statement `s`, which expects to be + /// followed by some token from the set edible + inedible. Check + /// for recoverable input errors, discarding erroneous characters. pub fn commit_stmt(&mut self, s: Gc<Stmt>, edible: &[token::Token], inedible: &[token::Token]) { debug!("commit_stmt {:?}", s); @@ -526,8 +542,8 @@ impl<'a> Parser<'a> { id: ast::DUMMY_NODE_ID }) } - // consume token 'tok' if it exists. Returns true if the given - // token was present, false otherwise. + /// Consume token 'tok' if it exists. Returns true if the given + /// token was present, false otherwise. pub fn eat(&mut self, tok: &token::Token) -> bool { let is_present = self.token == *tok; if is_present { self.bump() } @@ -538,8 +554,8 @@ impl<'a> Parser<'a> { token::is_keyword(kw, &self.token) } - // if the next token is the given keyword, eat it and return - // true. Otherwise, return false. + /// If the next token is the given keyword, eat it and return + /// true. Otherwise, return false. pub fn eat_keyword(&mut self, kw: keywords::Keyword) -> bool { match self.token { token::IDENT(sid, false) if kw.to_name() == sid.name => { @@ -550,9 +566,9 @@ impl<'a> Parser<'a> { } } - // if the given word is not a keyword, signal an error. - // if the next token is not the given word, signal an error. - // otherwise, eat it. + /// If the given word is not a keyword, signal an error. + /// If the next token is not the given word, signal an error. + /// Otherwise, eat it. pub fn expect_keyword(&mut self, kw: keywords::Keyword) { if !self.eat_keyword(kw) { let id_interned_str = token::get_name(kw.to_name()); @@ -562,7 +578,7 @@ impl<'a> Parser<'a> { } } - // signal an error if the given string is a strict keyword + /// Signal an error if the given string is a strict keyword pub fn check_strict_keywords(&mut self) { if token::is_strict_keyword(&self.token) { let token_str = self.this_token_to_string(); @@ -573,7 +589,7 @@ impl<'a> Parser<'a> { } } - // signal an error if the current token is a reserved keyword + /// Signal an error if the current token is a reserved keyword pub fn check_reserved_keywords(&mut self) { if token::is_reserved_keyword(&self.token) { let token_str = self.this_token_to_string(); @@ -582,8 +598,8 @@ impl<'a> Parser<'a> { } } - // Expect and consume an `&`. If `&&` is seen, replace it with a single - // `&` and continue. If an `&` is not seen, signal an error. + /// Expect and consume an `&`. If `&&` is seen, replace it with a single + /// `&` and continue. If an `&` is not seen, signal an error. fn expect_and(&mut self) { match self.token { token::BINOP(token::AND) => self.bump(), @@ -603,8 +619,8 @@ impl<'a> Parser<'a> { } } - // Expect and consume a `|`. If `||` is seen, replace it with a single - // `|` and continue. If a `|` is not seen, signal an error. + /// Expect and consume a `|`. If `||` is seen, replace it with a single + /// `|` and continue. If a `|` is not seen, signal an error. fn expect_or(&mut self) { match self.token { token::BINOP(token::OR) => self.bump(), @@ -624,26 +640,26 @@ impl<'a> Parser<'a> { } } - // Attempt to consume a `<`. If `<<` is seen, replace it with a single - // `<` and continue. If a `<` is not seen, return false. - // - // This is meant to be used when parsing generics on a path to get the - // starting token. The `force` parameter is used to forcefully break up a - // `<<` token. If `force` is false, then `<<` is only broken when a lifetime - // shows up next. For example, consider the expression: - // - // foo as bar << test - // - // The parser needs to know if `bar <<` is the start of a generic path or if - // it's a left-shift token. If `test` were a lifetime, then it's impossible - // for the token to be a left-shift, but if it's not a lifetime, then it's - // considered a left-shift. - // - // The reason for this is that the only current ambiguity with `<<` is when - // parsing closure types: - // - // foo::<<'a> ||>(); - // impl Foo<<'a> ||>() { ... } + /// Attempt to consume a `<`. If `<<` is seen, replace it with a single + /// `<` and continue. If a `<` is not seen, return false. + /// + /// This is meant to be used when parsing generics on a path to get the + /// starting token. The `force` parameter is used to forcefully break up a + /// `<<` token. If `force` is false, then `<<` is only broken when a lifetime + /// shows up next. For example, consider the expression: + /// + /// foo as bar << test + /// + /// The parser needs to know if `bar <<` is the start of a generic path or if + /// it's a left-shift token. If `test` were a lifetime, then it's impossible + /// for the token to be a left-shift, but if it's not a lifetime, then it's + /// considered a left-shift. + /// + /// The reason for this is that the only current ambiguity with `<<` is when + /// parsing closure types: + /// + /// foo::<<'a> ||>(); + /// impl Foo<<'a> ||>() { ... } fn eat_lt(&mut self, force: bool) -> bool { match self.token { token::LT => { self.bump(); true } @@ -675,7 +691,7 @@ impl<'a> Parser<'a> { } } - // Parse a sequence bracketed by `|` and `|`, stopping before the `|`. + /// Parse a sequence bracketed by `|` and `|`, stopping before the `|`. fn parse_seq_to_before_or<T>( &mut self, sep: &token::Token, @@ -696,9 +712,9 @@ impl<'a> Parser<'a> { vector } - // expect and consume a GT. if a >> is seen, replace it - // with a single > and continue. If a GT is not seen, - // signal an error. + /// Expect and consume a GT. if a >> is seen, replace it + /// with a single > and continue. If a GT is not seen, + /// signal an error. pub fn expect_gt(&mut self) { match self.token { token::GT => self.bump(), @@ -727,8 +743,8 @@ impl<'a> Parser<'a> { } } - // parse a sequence bracketed by '<' and '>', stopping - // before the '>'. + /// Parse a sequence bracketed by '<' and '>', stopping + /// before the '>'. pub fn parse_seq_to_before_gt<T>( &mut self, sep: Option<token::Token>, @@ -762,9 +778,9 @@ impl<'a> Parser<'a> { return v; } - // parse a sequence, including the closing delimiter. The function - // f must consume tokens until reaching the next separator or - // closing bracket. + /// Parse a sequence, including the closing delimiter. The function + /// f must consume tokens until reaching the next separator or + /// closing bracket. pub fn parse_seq_to_end<T>( &mut self, ket: &token::Token, @@ -776,9 +792,9 @@ impl<'a> Parser<'a> { val } - // parse a sequence, not including the closing delimiter. The function - // f must consume tokens until reaching the next separator or - // closing bracket. + /// Parse a sequence, not including the closing delimiter. The function + /// f must consume tokens until reaching the next separator or + /// closing bracket. pub fn parse_seq_to_before_end<T>( &mut self, ket: &token::Token, @@ -801,9 +817,9 @@ impl<'a> Parser<'a> { return v; } - // parse a sequence, including the closing delimiter. The function - // f must consume tokens until reaching the next separator or - // closing bracket. + /// Parse a sequence, including the closing delimiter. The function + /// f must consume tokens until reaching the next separator or + /// closing bracket. pub fn parse_unspanned_seq<T>( &mut self, bra: &token::Token, @@ -817,8 +833,8 @@ impl<'a> Parser<'a> { result } - // parse a sequence parameter of enum variant. For consistency purposes, - // these should not be empty. + /// Parse a sequence parameter of enum variant. For consistency purposes, + /// these should not be empty. pub fn parse_enum_variant_seq<T>( &mut self, bra: &token::Token, @@ -852,7 +868,7 @@ impl<'a> Parser<'a> { spanned(lo, hi, result) } - // advance the parser by one token + /// Advance the parser by one token pub fn bump(&mut self) { self.last_span = self.span; // Stash token for error recovery (sometimes; clone is not necessarily cheap). @@ -862,7 +878,7 @@ impl<'a> Parser<'a> { None }; let next = if self.buffer_start == self.buffer_end { - self.reader.next_token() + real_token(self.reader) } else { // Avoid token copies with `replace`. let buffer_start = self.buffer_start as uint; @@ -880,14 +896,14 @@ impl<'a> Parser<'a> { self.tokens_consumed += 1u; } - // Advance the parser by one token and return the bumped token. + /// Advance the parser by one token and return the bumped token. pub fn bump_and_get(&mut self) -> token::Token { let old_token = replace(&mut self.token, token::UNDERSCORE); self.bump(); old_token } - // EFFECT: replace the current token and span with the given one + /// EFFECT: replace the current token and span with the given one pub fn replace_token(&mut self, next: token::Token, lo: BytePos, @@ -906,7 +922,7 @@ impl<'a> Parser<'a> { -> R { let dist = distance as int; while self.buffer_length() < dist { - self.buffer[self.buffer_end as uint] = self.reader.next_token(); + self.buffer[self.buffer_end as uint] = real_token(self.reader); self.buffer_end = (self.buffer_end + 1) & 3; } f(&self.buffer[((self.buffer_start + dist - 1) & 3) as uint].tok) @@ -940,8 +956,8 @@ impl<'a> Parser<'a> { token::get_ident(id) } - // Is the current token one of the keywords that signals a bare function - // type? + /// Is the current token one of the keywords that signals a bare function + /// type? pub fn token_is_bare_fn_keyword(&mut self) -> bool { if token::is_keyword(keywords::Fn, &self.token) { return true @@ -955,14 +971,14 @@ impl<'a> Parser<'a> { false } - // Is the current token one of the keywords that signals a closure type? + /// Is the current token one of the keywords that signals a closure type? pub fn token_is_closure_keyword(&mut self) -> bool { token::is_keyword(keywords::Unsafe, &self.token) || token::is_keyword(keywords::Once, &self.token) } - // Is the current token one of the keywords that signals an old-style - // closure type (with explicit sigil)? + /// Is the current token one of the keywords that signals an old-style + /// closure type (with explicit sigil)? pub fn token_is_old_style_closure_keyword(&mut self) -> bool { token::is_keyword(keywords::Unsafe, &self.token) || token::is_keyword(keywords::Once, &self.token) || @@ -983,7 +999,7 @@ impl<'a> Parser<'a> { } } - // parse a TyBareFn type: + /// parse a TyBareFn type: pub fn parse_ty_bare_fn(&mut self) -> Ty_ { /* @@ -1014,8 +1030,8 @@ impl<'a> Parser<'a> { }); } - // Parses a procedure type (`proc`). The initial `proc` keyword must - // already have been parsed. + /// Parses a procedure type (`proc`). The initial `proc` keyword must + /// already have been parsed. pub fn parse_proc_type(&mut self) -> Ty_ { /* @@ -1063,7 +1079,7 @@ impl<'a> Parser<'a> { }) } - // parse a TyClosure type + /// Parse a TyClosure type pub fn parse_ty_closure(&mut self) -> Ty_ { /* @@ -1154,7 +1170,7 @@ impl<'a> Parser<'a> { } } - // parse a function type (following the 'fn') + /// Parse a function type (following the 'fn') pub fn parse_ty_fn_decl(&mut self, allow_variadic: bool) -> (P<FnDecl>, Vec<ast::Lifetime>) { /* @@ -1186,7 +1202,7 @@ impl<'a> Parser<'a> { (decl, lifetimes) } - // parse the methods in a trait declaration + /// Parse the methods in a trait declaration pub fn parse_trait_methods(&mut self) -> Vec<TraitMethod> { self.parse_unspanned_seq( &token::LBRACE, @@ -1255,15 +1271,15 @@ impl<'a> Parser<'a> { }) } - // parse a possibly mutable type + /// Parse a possibly mutable type pub fn parse_mt(&mut self) -> MutTy { let mutbl = self.parse_mutability(); let t = self.parse_ty(true); MutTy { ty: t, mutbl: mutbl } } - // parse [mut/const/imm] ID : TY - // now used only by obsolete record syntax parser... + /// Parse [mut/const/imm] ID : TY + /// now used only by obsolete record syntax parser... pub fn parse_ty_field(&mut self) -> TypeField { let lo = self.span.lo; let mutbl = self.parse_mutability(); @@ -1278,7 +1294,7 @@ impl<'a> Parser<'a> { } } - // parse optional return type [ -> TY ] in function decl + /// Parse optional return type [ -> TY ] in function decl pub fn parse_ret_ty(&mut self) -> (RetStyle, P<Ty>) { return if self.eat(&token::RARROW) { let lo = self.span.lo; @@ -1478,8 +1494,8 @@ impl<'a> Parser<'a> { } } - // This version of parse arg doesn't necessarily require - // identifier names. + /// This version of parse arg doesn't necessarily require + /// identifier names. pub fn parse_arg_general(&mut self, require_name: bool) -> Arg { let pat = if require_name || self.is_named_argument() { debug!("parse_arg_general parse_pat (require_name:{:?})", @@ -1504,12 +1520,12 @@ impl<'a> Parser<'a> { } } - // parse a single function argument + /// Parse a single function argument pub fn parse_arg(&mut self) -> Arg { self.parse_arg_general(true) } - // parse an argument in a lambda header e.g. |arg, arg| + /// Parse an argument in a lambda header e.g. |arg, arg| pub fn parse_fn_block_arg(&mut self) -> Arg { let pat = self.parse_pat(); let t = if self.eat(&token::COLON) { @@ -1539,34 +1555,32 @@ impl<'a> Parser<'a> { } } - // matches token_lit = LIT_INT | ... + /// Matches token_lit = LIT_INTEGER | ... pub fn lit_from_token(&mut self, tok: &token::Token) -> Lit_ { match *tok { - token::LIT_BYTE(i) => LitByte(i), - token::LIT_CHAR(i) => LitChar(i), - token::LIT_INT(i, it) => LitInt(i, it), - token::LIT_UINT(u, ut) => LitUint(u, ut), - token::LIT_INT_UNSUFFIXED(i) => LitIntUnsuffixed(i), - token::LIT_FLOAT(s, ft) => { - LitFloat(self.id_to_interned_str(s), ft) - } - token::LIT_FLOAT_UNSUFFIXED(s) => { - LitFloatUnsuffixed(self.id_to_interned_str(s)) - } + token::LIT_BYTE(i) => LitByte(parse::byte_lit(i.as_str()).val0()), + token::LIT_CHAR(i) => LitChar(parse::char_lit(i.as_str()).val0()), + token::LIT_INTEGER(s) => parse::integer_lit(s.as_str(), + &self.sess.span_diagnostic, self.span), + token::LIT_FLOAT(s) => parse::float_lit(s.as_str()), token::LIT_STR(s) => { - LitStr(self.id_to_interned_str(s), ast::CookedStr) + LitStr(token::intern_and_get_ident(parse::str_lit(s.as_str()).as_slice()), + ast::CookedStr) } token::LIT_STR_RAW(s, n) => { - LitStr(self.id_to_interned_str(s), ast::RawStr(n)) + LitStr(token::intern_and_get_ident(parse::raw_str_lit(s.as_str()).as_slice()), + ast::RawStr(n)) } - token::LIT_BINARY_RAW(ref v, _) | - token::LIT_BINARY(ref v) => LitBinary(v.clone()), + token::LIT_BINARY(i) => + LitBinary(parse::binary_lit(i.as_str())), + token::LIT_BINARY_RAW(i, _) => + LitBinary(Rc::new(i.as_str().as_bytes().iter().map(|&x| x).collect())), token::LPAREN => { self.expect(&token::RPAREN); LitNil }, _ => { self.unexpected_last(tok); } } } - // matches lit = true | false | token_lit + /// Matches lit = true | false | token_lit pub fn parse_lit(&mut self) -> Lit { let lo = self.span.lo; let lit = if self.eat_keyword(keywords::True) { @@ -1581,7 +1595,7 @@ impl<'a> Parser<'a> { codemap::Spanned { node: lit, span: mk_sp(lo, self.last_span.hi) } } - // matches '-' lit | lit + /// matches '-' lit | lit pub fn parse_literal_maybe_minus(&mut self) -> Gc<Expr> { let minus_lo = self.span.lo; let minus_present = self.eat(&token::BINOP(token::MINUS)); @@ -1719,7 +1733,7 @@ impl<'a> Parser<'a> { } /// Parses a single lifetime - // matches lifetime = LIFETIME + /// Matches lifetime = LIFETIME pub fn parse_lifetime(&mut self) -> ast::Lifetime { match self.token { token::LIFETIME(i) => { @@ -1779,7 +1793,7 @@ impl<'a> Parser<'a> { token::is_keyword(keywords::Const, tok) } - // parse mutability declaration (mut/const/imm) + /// Parse mutability declaration (mut/const/imm) pub fn parse_mutability(&mut self) -> Mutability { if self.eat_keyword(keywords::Mut) { MutMutable @@ -1788,7 +1802,7 @@ impl<'a> Parser<'a> { } } - // parse ident COLON expr + /// Parse ident COLON expr pub fn parse_field(&mut self) -> Field { let lo = self.span.lo; let i = self.parse_ident(); @@ -1867,9 +1881,9 @@ impl<'a> Parser<'a> { } } - // at the bottom (top?) of the precedence hierarchy, - // parse things like parenthesized exprs, - // macros, return, etc. + /// At the bottom (top?) of the precedence hierarchy, + /// parse things like parenthesized exprs, + /// macros, return, etc. pub fn parse_bottom_expr(&mut self) -> Gc<Expr> { maybe_whole_expr!(self); @@ -1934,7 +1948,12 @@ impl<'a> Parser<'a> { }); return self.mk_expr(lo, body.span.hi, ExprProc(decl, fakeblock)); }, - token::IDENT(id @ ast::Ident{name:token::SELF_KEYWORD_NAME,ctxt:_},false) => { + // FIXME #13626: Should be able to stick in + // token::SELF_KEYWORD_NAME + token::IDENT(id @ ast::Ident{ + name: ast::Name(token::SELF_KEYWORD_NAME_NUM), + ctxt: _ + } ,false) => { self.bump(); let path = ast_util::ident_to_path(mk_sp(lo, hi), id); ex = ExprPath(path); @@ -2107,7 +2126,7 @@ impl<'a> Parser<'a> { return self.mk_expr(lo, hi, ex); } - // parse a block or unsafe block + /// Parse a block or unsafe block pub fn parse_block_expr(&mut self, lo: BytePos, blk_mode: BlockCheckMode) -> Gc<Expr> { self.expect(&token::LBRACE); @@ -2115,7 +2134,7 @@ impl<'a> Parser<'a> { return self.mk_expr(blk.span.lo, blk.span.hi, ExprBlock(blk)); } - // parse a.b or a(13) or a[4] or just a + /// parse a.b or a(13) or a[4] or just a pub fn parse_dot_or_call_expr(&mut self) -> Gc<Expr> { let b = self.parse_bottom_expr(); self.parse_dot_or_call_expr_with(b) @@ -2199,8 +2218,8 @@ impl<'a> Parser<'a> { return e; } - // parse an optional separator followed by a kleene-style - // repetition token (+ or *). + /// Parse an optional separator followed by a kleene-style + /// repetition token (+ or *). pub fn parse_sep_and_zerok(&mut self) -> (Option<token::Token>, bool) { fn parse_zerok(parser: &mut Parser) -> Option<bool> { match parser.token { @@ -2225,7 +2244,7 @@ impl<'a> Parser<'a> { } } - // parse a single token tree from the input. + /// parse a single token tree from the input. pub fn parse_token_tree(&mut self) -> TokenTree { // FIXME #6994: currently, this is too eager. It // parses token trees but also identifies TTSeq's @@ -2341,9 +2360,9 @@ impl<'a> Parser<'a> { } } - // This goofy function is necessary to correctly match parens in Matcher's. - // Otherwise, `$( ( )` would be a valid Matcher, and `$( () )` would be - // invalid. It's similar to common::parse_seq. + /// This goofy function is necessary to correctly match parens in Matcher's. + /// Otherwise, `$( ( )` would be a valid Matcher, and `$( () )` would be + /// invalid. It's similar to common::parse_seq. pub fn parse_matcher_subseq_upto(&mut self, name_idx: &mut uint, ket: &token::Token) @@ -2392,7 +2411,7 @@ impl<'a> Parser<'a> { return spanned(lo, self.span.hi, m); } - // parse a prefix-operator expr + /// Parse a prefix-operator expr pub fn parse_prefix_expr(&mut self) -> Gc<Expr> { let lo = self.span.lo; let hi; @@ -2500,13 +2519,13 @@ impl<'a> Parser<'a> { return self.mk_expr(lo, hi, ex); } - // parse an expression of binops + /// Parse an expression of binops pub fn parse_binops(&mut self) -> Gc<Expr> { let prefix_expr = self.parse_prefix_expr(); self.parse_more_binops(prefix_expr, 0) } - // parse an expression of binops of at least min_prec precedence + /// Parse an expression of binops of at least min_prec precedence pub fn parse_more_binops(&mut self, lhs: Gc<Expr>, min_prec: uint) -> Gc<Expr> { if self.expr_is_complete(lhs) { return lhs; } @@ -2554,9 +2573,9 @@ impl<'a> Parser<'a> { } } - // parse an assignment expression.... - // actually, this seems to be the main entry point for - // parsing an arbitrary expression. + /// Parse an assignment expression.... + /// actually, this seems to be the main entry point for + /// parsing an arbitrary expression. pub fn parse_assign_expr(&mut self) -> Gc<Expr> { let lo = self.span.lo; let lhs = self.parse_binops(); @@ -2590,7 +2609,7 @@ impl<'a> Parser<'a> { } } - // parse an 'if' expression ('if' token already eaten) + /// Parse an 'if' expression ('if' token already eaten) pub fn parse_if_expr(&mut self) -> Gc<Expr> { let lo = self.last_span.lo; let cond = self.parse_expr_res(RESTRICT_NO_STRUCT_LITERAL); @@ -2605,7 +2624,7 @@ impl<'a> Parser<'a> { self.mk_expr(lo, hi, ExprIf(cond, thn, els)) } - // `|args| { ... }` or `{ ...}` like in `do` expressions + /// `|args| { ... }` or `{ ...}` like in `do` expressions pub fn parse_lambda_block_expr(&mut self) -> Gc<Expr> { self.parse_lambda_expr_( |p| { @@ -2634,15 +2653,15 @@ impl<'a> Parser<'a> { }) } - // `|args| expr` + /// `|args| expr` pub fn parse_lambda_expr(&mut self) -> Gc<Expr> { self.parse_lambda_expr_(|p| p.parse_fn_block_decl(), |p| p.parse_expr()) } - // parse something of the form |args| expr - // this is used both in parsing a lambda expr - // and in parsing a block expr as e.g. in for... + /// parse something of the form |args| expr + /// this is used both in parsing a lambda expr + /// and in parsing a block expr as e.g. in for... pub fn parse_lambda_expr_(&mut self, parse_decl: |&mut Parser| -> P<FnDecl>, parse_body: |&mut Parser| -> Gc<Expr>) @@ -2671,7 +2690,7 @@ impl<'a> Parser<'a> { } } - // parse a 'for' .. 'in' expression ('for' token already eaten) + /// Parse a 'for' .. 'in' expression ('for' token already eaten) pub fn parse_for_expr(&mut self, opt_ident: Option<ast::Ident>) -> Gc<Expr> { // Parse: `for <src_pat> in <src_expr> <src_loop_block>` @@ -2737,12 +2756,12 @@ impl<'a> Parser<'a> { return self.mk_expr(lo, hi, ExprMatch(discriminant, arms)); } - // parse an expression + /// Parse an expression pub fn parse_expr(&mut self) -> Gc<Expr> { return self.parse_expr_res(UNRESTRICTED); } - // parse an expression, subject to the given restriction + /// Parse an expression, subject to the given restriction pub fn parse_expr_res(&mut self, r: restriction) -> Gc<Expr> { let old = self.restriction; self.restriction = r; @@ -2751,7 +2770,7 @@ impl<'a> Parser<'a> { return e; } - // parse the RHS of a local variable declaration (e.g. '= 14;') + /// Parse the RHS of a local variable declaration (e.g. '= 14;') fn parse_initializer(&mut self) -> Option<Gc<Expr>> { if self.token == token::EQ { self.bump(); @@ -2761,7 +2780,7 @@ impl<'a> Parser<'a> { } } - // parse patterns, separated by '|' s + /// Parse patterns, separated by '|' s fn parse_pats(&mut self) -> Vec<Gc<Pat>> { let mut pats = Vec::new(); loop { @@ -2824,7 +2843,7 @@ impl<'a> Parser<'a> { (before, slice, after) } - // parse the fields of a struct-like pattern + /// Parse the fields of a struct-like pattern fn parse_pat_fields(&mut self) -> (Vec<ast::FieldPat> , bool) { let mut fields = Vec::new(); let mut etc = false; @@ -2884,7 +2903,7 @@ impl<'a> Parser<'a> { return (fields, etc); } - // parse a pattern. + /// Parse a pattern. pub fn parse_pat(&mut self) -> Gc<Pat> { maybe_whole!(self, NtPat); @@ -3126,9 +3145,9 @@ impl<'a> Parser<'a> { } } - // parse ident or ident @ pat - // used by the copy foo and ref foo patterns to give a good - // error message when parsing mistakes like ref foo(a,b) + /// Parse ident or ident @ pat + /// used by the copy foo and ref foo patterns to give a good + /// error message when parsing mistakes like ref foo(a,b) fn parse_pat_ident(&mut self, binding_mode: ast::BindingMode) -> ast::Pat_ { @@ -3162,7 +3181,7 @@ impl<'a> Parser<'a> { PatIdent(binding_mode, name, sub) } - // parse a local variable declaration + /// Parse a local variable declaration fn parse_local(&mut self) -> Gc<Local> { let lo = self.span.lo; let pat = self.parse_pat(); @@ -3186,14 +3205,14 @@ impl<'a> Parser<'a> { } } - // parse a "let" stmt + /// Parse a "let" stmt fn parse_let(&mut self) -> Gc<Decl> { let lo = self.span.lo; let local = self.parse_local(); box(GC) spanned(lo, self.last_span.hi, DeclLocal(local)) } - // parse a structure field + /// Parse a structure field fn parse_name_and_ty(&mut self, pr: Visibility, attrs: Vec<Attribute> ) -> StructField { let lo = self.span.lo; @@ -3211,8 +3230,8 @@ impl<'a> Parser<'a> { }) } - // parse a statement. may include decl. - // precondition: any attributes are parsed already + /// Parse a statement. may include decl. + /// Precondition: any attributes are parsed already pub fn parse_stmt(&mut self, item_attrs: Vec<Attribute>) -> Gc<Stmt> { maybe_whole!(self, NtStmt); @@ -3315,13 +3334,13 @@ impl<'a> Parser<'a> { } } - // is this expression a successfully-parsed statement? + /// Is this expression a successfully-parsed statement? fn expr_is_complete(&mut self, e: Gc<Expr>) -> bool { return self.restriction == RESTRICT_STMT_EXPR && !classify::expr_requires_semi_to_be_stmt(e); } - // parse a block. No inner attrs are allowed. + /// Parse a block. No inner attrs are allowed. pub fn parse_block(&mut self) -> P<Block> { maybe_whole!(no_clone self, NtBlock); @@ -3331,7 +3350,7 @@ impl<'a> Parser<'a> { return self.parse_block_tail_(lo, DefaultBlock, Vec::new()); } - // parse a block. Inner attrs are allowed. + /// Parse a block. Inner attrs are allowed. fn parse_inner_attrs_and_block(&mut self) -> (Vec<Attribute> , P<Block>) { @@ -3344,15 +3363,15 @@ impl<'a> Parser<'a> { (inner, self.parse_block_tail_(lo, DefaultBlock, next)) } - // Precondition: already parsed the '{' or '#{' - // I guess that also means "already parsed the 'impure'" if - // necessary, and this should take a qualifier. - // some blocks start with "#{"... + /// Precondition: already parsed the '{' or '#{' + /// I guess that also means "already parsed the 'impure'" if + /// necessary, and this should take a qualifier. + /// Some blocks start with "#{"... fn parse_block_tail(&mut self, lo: BytePos, s: BlockCheckMode) -> P<Block> { self.parse_block_tail_(lo, s, Vec::new()) } - // parse the rest of a block expression or function body + /// Parse the rest of a block expression or function body fn parse_block_tail_(&mut self, lo: BytePos, s: BlockCheckMode, first_item_attrs: Vec<Attribute> ) -> P<Block> { let mut stmts = Vec::new(); @@ -3510,18 +3529,18 @@ impl<'a> Parser<'a> { } } - // matches bounds = ( boundseq )? - // where boundseq = ( bound + boundseq ) | bound - // and bound = 'static | ty - // Returns "None" if there's no colon (e.g. "T"); - // Returns "Some(Empty)" if there's a colon but nothing after (e.g. "T:") - // Returns "Some(stuff)" otherwise (e.g. "T:stuff"). - // NB: The None/Some distinction is important for issue #7264. - // - // Note that the `allow_any_lifetime` argument is a hack for now while the - // AST doesn't support arbitrary lifetimes in bounds on type parameters. In - // the future, this flag should be removed, and the return value of this - // function should be Option<~[TyParamBound]> + /// matches optbounds = ( ( : ( boundseq )? )? ) + /// where boundseq = ( bound + boundseq ) | bound + /// and bound = 'static | ty + /// Returns "None" if there's no colon (e.g. "T"); + /// Returns "Some(Empty)" if there's a colon but nothing after (e.g. "T:") + /// Returns "Some(stuff)" otherwise (e.g. "T:stuff"). + /// NB: The None/Some distinction is important for issue #7264. + /// + /// Note that the `allow_any_lifetime` argument is a hack for now while the + /// AST doesn't support arbitrary lifetimes in bounds on type parameters. In + /// the future, this flag should be removed, and the return value of this + /// function should be Option<~[TyParamBound]> fn parse_ty_param_bounds(&mut self, allow_any_lifetime: bool) -> (Option<ast::Lifetime>, OwnedSlice<TyParamBound>) { @@ -3588,7 +3607,7 @@ impl<'a> Parser<'a> { } } - // matches typaram = (unbound`?`)? IDENT optbounds ( EQ ty )? + /// Matches typaram = (unbound`?`)? IDENT optbounds ( EQ ty )? fn parse_ty_param(&mut self) -> TyParam { // This is a bit hacky. Currently we are only interested in a single // unbound, and it may only be `Sized`. To avoid backtracking and other @@ -3632,10 +3651,10 @@ impl<'a> Parser<'a> { } } - // parse a set of optional generic type parameter declarations - // matches generics = ( ) | ( < > ) | ( < typaramseq ( , )? > ) | ( < lifetimes ( , )? > ) - // | ( < lifetimes , typaramseq ( , )? > ) - // where typaramseq = ( typaram ) | ( typaram , typaramseq ) + /// Parse a set of optional generic type parameter declarations + /// matches generics = ( ) | ( < > ) | ( < typaramseq ( , )? > ) | ( < lifetimes ( , )? > ) + /// | ( < lifetimes , typaramseq ( , )? > ) + /// where typaramseq = ( typaram ) | ( typaram , typaramseq ) pub fn parse_generics(&mut self) -> ast::Generics { if self.eat(&token::LT) { let lifetimes = self.parse_lifetimes(); @@ -3727,7 +3746,7 @@ impl<'a> Parser<'a> { (args, variadic) } - // parse the argument list and result type of a function declaration + /// Parse the argument list and result type of a function declaration pub fn parse_fn_decl(&mut self, allow_variadic: bool) -> P<FnDecl> { let (args, variadic) = self.parse_fn_args(true, allow_variadic); @@ -3762,8 +3781,8 @@ impl<'a> Parser<'a> { } } - // parse the argument list and result type of a function - // that may have a self type. + /// Parse the argument list and result type of a function + /// that may have a self type. fn parse_fn_decl_with_self(&mut self, parse_arg_fn: |&mut Parser| -> Arg) -> (ExplicitSelf, P<FnDecl>) { fn maybe_parse_borrowed_explicit_self(this: &mut Parser) @@ -3921,7 +3940,7 @@ impl<'a> Parser<'a> { (spanned(lo, hi, explicit_self), fn_decl) } - // parse the |arg, arg| header on a lambda + /// Parse the |arg, arg| header on a lambda fn parse_fn_block_decl(&mut self) -> P<FnDecl> { let inputs_captures = { if self.eat(&token::OROR) { @@ -3953,7 +3972,7 @@ impl<'a> Parser<'a> { }) } - // Parses the `(arg, arg) -> return_type` header on a procedure. + /// Parses the `(arg, arg) -> return_type` header on a procedure. fn parse_proc_decl(&mut self) -> P<FnDecl> { let inputs = self.parse_unspanned_seq(&token::LPAREN, @@ -3979,7 +3998,7 @@ impl<'a> Parser<'a> { }) } - // parse the name and optional generic types of a function header. + /// Parse the name and optional generic types of a function header. fn parse_fn_header(&mut self) -> (Ident, ast::Generics) { let id = self.parse_ident(); let generics = self.parse_generics(); @@ -3999,7 +4018,7 @@ impl<'a> Parser<'a> { } } - // parse an item-position function declaration. + /// Parse an item-position function declaration. fn parse_item_fn(&mut self, fn_style: FnStyle, abi: abi::Abi) -> ItemInfo { let (ident, generics) = self.parse_fn_header(); let decl = self.parse_fn_decl(false); @@ -4007,7 +4026,7 @@ impl<'a> Parser<'a> { (ident, ItemFn(decl, fn_style, abi, generics, body), Some(inner_attrs)) } - // parse a method in a trait impl, starting with `attrs` attributes. + /// Parse a method in a trait impl, starting with `attrs` attributes. fn parse_method(&mut self, already_parsed_attrs: Option<Vec<Attribute>>) -> Gc<Method> { let next_attrs = self.parse_outer_attributes(); @@ -4043,7 +4062,7 @@ impl<'a> Parser<'a> { } } - // parse trait Foo { ... } + /// Parse trait Foo { ... } fn parse_item_trait(&mut self) -> ItemInfo { let ident = self.parse_ident(); let tps = self.parse_generics(); @@ -4062,9 +4081,9 @@ impl<'a> Parser<'a> { (ident, ItemTrait(tps, sized, traits, meths), None) } - // Parses two variants (with the region/type params always optional): - // impl<T> Foo { ... } - // impl<T> ToString for ~[T] { ... } + /// Parses two variants (with the region/type params always optional): + /// impl<T> Foo { ... } + /// impl<T> ToString for ~[T] { ... } fn parse_item_impl(&mut self) -> ItemInfo { // First, parse type parameters if necessary. let generics = self.parse_generics(); @@ -4117,7 +4136,7 @@ impl<'a> Parser<'a> { (ident, ItemImpl(generics, opt_trait, ty, meths), Some(inner_attrs)) } - // parse a::B<String,int> + /// Parse a::B<String,int> fn parse_trait_ref(&mut self) -> TraitRef { ast::TraitRef { path: self.parse_path(LifetimeAndTypesWithoutColons).path, @@ -4125,7 +4144,7 @@ impl<'a> Parser<'a> { } } - // parse B + C<String,int> + D + /// Parse B + C<String,int> + D fn parse_trait_ref_list(&mut self, ket: &token::Token) -> Vec<TraitRef> { self.parse_seq_to_before_end( ket, @@ -4134,7 +4153,7 @@ impl<'a> Parser<'a> { ) } - // parse struct Foo { ... } + /// Parse struct Foo { ... } fn parse_item_struct(&mut self, is_virtual: bool) -> ItemInfo { let class_name = self.parse_ident(); let generics = self.parse_generics(); @@ -4217,7 +4236,7 @@ impl<'a> Parser<'a> { None) } - // parse a structure field declaration + /// Parse a structure field declaration pub fn parse_single_struct_field(&mut self, vis: Visibility, attrs: Vec<Attribute> ) @@ -4239,7 +4258,7 @@ impl<'a> Parser<'a> { a_var } - // parse an element of a struct definition + /// Parse an element of a struct definition fn parse_struct_decl_field(&mut self) -> StructField { let attrs = self.parse_outer_attributes(); @@ -4251,7 +4270,7 @@ impl<'a> Parser<'a> { return self.parse_single_struct_field(Inherited, attrs); } - // parse visiility: PUB, PRIV, or nothing + /// Parse visiility: PUB, PRIV, or nothing fn parse_visibility(&mut self) -> Visibility { if self.eat_keyword(keywords::Pub) { Public } else { Inherited } @@ -4273,8 +4292,8 @@ impl<'a> Parser<'a> { } } - // given a termination token and a vector of already-parsed - // attributes (of length 0 or 1), parse all of the items in a module + /// Given a termination token and a vector of already-parsed + /// attributes (of length 0 or 1), parse all of the items in a module fn parse_mod_items(&mut self, term: token::Token, first_item_attrs: Vec<Attribute>, @@ -4342,7 +4361,7 @@ impl<'a> Parser<'a> { (id, ItemStatic(ty, m, e), None) } - // parse a `mod <foo> { ... }` or `mod <foo>;` item + /// Parse a `mod <foo> { ... }` or `mod <foo>;` item fn parse_item_mod(&mut self, outer_attrs: &[Attribute]) -> ItemInfo { let id_span = self.span; let id = self.parse_ident(); @@ -4380,7 +4399,7 @@ impl<'a> Parser<'a> { self.mod_path_stack.pop().unwrap(); } - // read a module from a source file. + /// Read a module from a source file. fn eval_src_mod(&mut self, id: ast::Ident, outer_attrs: &[ast::Attribute], @@ -4488,7 +4507,7 @@ impl<'a> Parser<'a> { return (ast::ItemMod(m0), mod_attrs); } - // parse a function declaration from a foreign module + /// Parse a function declaration from a foreign module fn parse_item_foreign_fn(&mut self, vis: ast::Visibility, attrs: Vec<Attribute>) -> Gc<ForeignItem> { let lo = self.span.lo; @@ -4506,7 +4525,7 @@ impl<'a> Parser<'a> { vis: vis } } - // parse a static item from a foreign module + /// Parse a static item from a foreign module fn parse_item_foreign_static(&mut self, vis: ast::Visibility, attrs: Vec<Attribute> ) -> Gc<ForeignItem> { let lo = self.span.lo; @@ -4529,7 +4548,7 @@ impl<'a> Parser<'a> { } } - // parse safe/unsafe and fn + /// Parse safe/unsafe and fn fn parse_fn_style(&mut self) -> FnStyle { if self.eat_keyword(keywords::Fn) { NormalFn } else if self.eat_keyword(keywords::Unsafe) { @@ -4540,8 +4559,8 @@ impl<'a> Parser<'a> { } - // at this point, this is essentially a wrapper for - // parse_foreign_items. + /// At this point, this is essentially a wrapper for + /// parse_foreign_items. fn parse_foreign_mod_items(&mut self, abi: abi::Abi, first_item_attrs: Vec<Attribute> ) @@ -4642,7 +4661,7 @@ impl<'a> Parser<'a> { return IoviItem(item); } - // parse type Foo = Bar; + /// Parse type Foo = Bar; fn parse_item_type(&mut self) -> ItemInfo { let ident = self.parse_ident(); let tps = self.parse_generics(); @@ -4652,8 +4671,8 @@ impl<'a> Parser<'a> { (ident, ItemTy(ty, tps), None) } - // parse a structure-like enum variant definition - // this should probably be renamed or refactored... + /// Parse a structure-like enum variant definition + /// this should probably be renamed or refactored... fn parse_struct_def(&mut self) -> Gc<StructDef> { let mut fields: Vec<StructField> = Vec::new(); while self.token != token::RBRACE { @@ -4669,7 +4688,7 @@ impl<'a> Parser<'a> { }; } - // parse the part of an "enum" decl following the '{' + /// Parse the part of an "enum" decl following the '{' fn parse_enum_def(&mut self, _generics: &ast::Generics) -> EnumDef { let mut variants = Vec::new(); let mut all_nullary = true; @@ -4733,7 +4752,7 @@ impl<'a> Parser<'a> { ast::EnumDef { variants: variants } } - // parse an "enum" declaration + /// Parse an "enum" declaration fn parse_item_enum(&mut self) -> ItemInfo { let id = self.parse_ident(); let generics = self.parse_generics(); @@ -4750,14 +4769,13 @@ impl<'a> Parser<'a> { } } - // Parses a string as an ABI spec on an extern type or module. Consumes - // the `extern` keyword, if one is found. + /// Parses a string as an ABI spec on an extern type or module. Consumes + /// the `extern` keyword, if one is found. fn parse_opt_abi(&mut self) -> Option<abi::Abi> { match self.token { token::LIT_STR(s) | token::LIT_STR_RAW(s, _) => { self.bump(); - let identifier_string = token::get_ident(s); - let the_string = identifier_string.get(); + let the_string = s.as_str(); match abi::lookup(the_string) { Some(abi) => Some(abi), None => { @@ -4777,10 +4795,10 @@ impl<'a> Parser<'a> { } } - // parse one of the items or view items allowed by the - // flags; on failure, return IoviNone. - // NB: this function no longer parses the items inside an - // extern crate. + /// Parse one of the items or view items allowed by the + /// flags; on failure, return IoviNone. + /// NB: this function no longer parses the items inside an + /// extern crate. fn parse_item_or_view_item(&mut self, attrs: Vec<Attribute> , macros_allowed: bool) @@ -4988,7 +5006,7 @@ impl<'a> Parser<'a> { self.parse_macro_use_or_failure(attrs,macros_allowed,lo,visibility) } - // parse a foreign item; on failure, return IoviNone. + /// Parse a foreign item; on failure, return IoviNone. fn parse_foreign_item(&mut self, attrs: Vec<Attribute> , macros_allowed: bool) @@ -5011,7 +5029,7 @@ impl<'a> Parser<'a> { self.parse_macro_use_or_failure(attrs,macros_allowed,lo,visibility) } - // this is the fall-through for parsing items. + /// This is the fall-through for parsing items. fn parse_macro_use_or_failure( &mut self, attrs: Vec<Attribute> , @@ -5095,17 +5113,17 @@ impl<'a> Parser<'a> { } } - // parse, e.g., "use a::b::{z,y}" + /// Parse, e.g., "use a::b::{z,y}" fn parse_use(&mut self) -> ViewItem_ { return ViewItemUse(self.parse_view_path()); } - // matches view_path : MOD? IDENT EQ non_global_path - // | MOD? non_global_path MOD_SEP LBRACE RBRACE - // | MOD? non_global_path MOD_SEP LBRACE ident_seq RBRACE - // | MOD? non_global_path MOD_SEP STAR - // | MOD? non_global_path + /// Matches view_path : MOD? IDENT EQ non_global_path + /// | MOD? non_global_path MOD_SEP LBRACE RBRACE + /// | MOD? non_global_path MOD_SEP LBRACE ident_seq RBRACE + /// | MOD? non_global_path MOD_SEP STAR + /// | MOD? non_global_path fn parse_view_path(&mut self) -> Gc<ViewPath> { let lo = self.span.lo; @@ -5228,10 +5246,10 @@ impl<'a> Parser<'a> { ViewPathSimple(last, path, ast::DUMMY_NODE_ID)); } - // Parses a sequence of items. Stops when it finds program - // text that can't be parsed as an item - // - mod_items uses extern_mod_allowed = true - // - block_tail_ uses extern_mod_allowed = false + /// Parses a sequence of items. Stops when it finds program + /// text that can't be parsed as an item + /// - mod_items uses extern_mod_allowed = true + /// - block_tail_ uses extern_mod_allowed = false fn parse_items_and_view_items(&mut self, first_item_attrs: Vec<Attribute> , mut extern_mod_allowed: bool, @@ -5313,8 +5331,8 @@ impl<'a> Parser<'a> { } } - // Parses a sequence of foreign items. Stops when it finds program - // text that can't be parsed as an item + /// Parses a sequence of foreign items. Stops when it finds program + /// text that can't be parsed as an item fn parse_foreign_items(&mut self, first_item_attrs: Vec<Attribute> , macros_allowed: bool) -> ParsedItemsAndViewItems { @@ -5353,8 +5371,8 @@ impl<'a> Parser<'a> { } } - // Parses a source module as a crate. This is the main - // entry point for the parser. + /// Parses a source module as a crate. This is the main + /// entry point for the parser. pub fn parse_crate_mod(&mut self) -> Crate { let lo = self.span.lo; // parse the crate's inner attrs, maybe (oops) one @@ -5375,9 +5393,9 @@ impl<'a> Parser<'a> { pub fn parse_optional_str(&mut self) -> Option<(InternedString, ast::StrStyle)> { let (s, style) = match self.token { - token::LIT_STR(s) => (self.id_to_interned_str(s), ast::CookedStr), + token::LIT_STR(s) => (self.id_to_interned_str(s.ident()), ast::CookedStr), token::LIT_STR_RAW(s, n) => { - (self.id_to_interned_str(s), ast::RawStr(n)) + (self.id_to_interned_str(s.ident()), ast::RawStr(n)) } _ => return None }; @@ -5392,3 +5410,4 @@ impl<'a> Parser<'a> { } } } + diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index 55db3482a61..5839df67022 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -10,7 +10,6 @@ use ast; use ast::{P, Ident, Name, Mrk}; -use ast_util; use ext::mtwt; use parse::token; use util::interner::{RcStr, StrInterner}; @@ -79,30 +78,37 @@ pub enum Token { QUESTION, /* Literals */ - LIT_BYTE(u8), - LIT_CHAR(char), - LIT_INT(i64, ast::IntTy), - LIT_UINT(u64, ast::UintTy), - LIT_INT_UNSUFFIXED(i64), - LIT_FLOAT(ast::Ident, ast::FloatTy), - LIT_FLOAT_UNSUFFIXED(ast::Ident), - LIT_STR(ast::Ident), - LIT_STR_RAW(ast::Ident, uint), /* raw str delimited by n hash symbols */ - LIT_BINARY(Rc<Vec<u8>>), - LIT_BINARY_RAW(Rc<Vec<u8>>, uint), /* raw binary str delimited by n hash symbols */ + LIT_BYTE(Name), + LIT_CHAR(Name), + LIT_INTEGER(Name), + LIT_FLOAT(Name), + LIT_STR(Name), + LIT_STR_RAW(Name, uint), /* raw str delimited by n hash symbols */ + LIT_BINARY(Name), + LIT_BINARY_RAW(Name, uint), /* raw binary str delimited by n hash symbols */ /* Name components */ - // an identifier contains an "is_mod_name" boolean, - // indicating whether :: follows this token with no - // whitespace in between. - IDENT(ast::Ident, bool), + /// An identifier contains an "is_mod_name" boolean, + /// indicating whether :: follows this token with no + /// whitespace in between. + IDENT(Ident, bool), UNDERSCORE, - LIFETIME(ast::Ident), + LIFETIME(Ident), /* For interpolation */ INTERPOLATED(Nonterminal), + DOC_COMMENT(Name), + + // Junk. These carry no data because we don't really care about the data + // they *would* carry, and don't really want to allocate a new ident for + // them. Instead, users could extract that from the associated span. + + /// Whitespace + WS, + /// Comment + COMMENT, + SHEBANG(Name), - DOC_COMMENT(ast::Ident), EOF, } @@ -115,11 +121,12 @@ pub enum Nonterminal { NtPat( Gc<ast::Pat>), NtExpr(Gc<ast::Expr>), NtTy( P<ast::Ty>), - // see IDENT, above, for meaning of bool in NtIdent: - NtIdent(Box<ast::Ident>, bool), - NtMeta(Gc<ast::MetaItem>), // stuff inside brackets for attributes + /// See IDENT, above, for meaning of bool in NtIdent: + NtIdent(Box<Ident>, bool), + /// Stuff inside brackets for attributes + NtMeta(Gc<ast::MetaItem>), NtPath(Box<ast::Path>), - NtTT( Gc<ast::TokenTree>), // needs @ed to break a circularity + NtTT( Gc<ast::TokenTree>), // needs Gc'd to break a circularity NtMatchers(Vec<ast::Matcher> ) } @@ -200,54 +207,28 @@ pub fn to_string(t: &Token) -> String { /* Literals */ LIT_BYTE(b) => { - let mut res = String::from_str("b'"); - (b as char).escape_default(|c| { - res.push_char(c); - }); - res.push_char('\''); - res + format!("b'{}'", b.as_str()) } LIT_CHAR(c) => { - let mut res = String::from_str("'"); - c.escape_default(|c| { - res.push_char(c); - }); - res.push_char('\''); - res - } - LIT_INT(i, t) => ast_util::int_ty_to_string(t, Some(i)), - LIT_UINT(u, t) => ast_util::uint_ty_to_string(t, Some(u)), - LIT_INT_UNSUFFIXED(i) => { (i as u64).to_string() } - LIT_FLOAT(s, t) => { - let mut body = String::from_str(get_ident(s).get()); - if body.as_slice().ends_with(".") { - body.push_char('0'); // `10.f` is not a float literal - } - body.push_str(ast_util::float_ty_to_string(t).as_slice()); - body + format!("'{}'", c.as_str()) } - LIT_FLOAT_UNSUFFIXED(s) => { - let mut body = String::from_str(get_ident(s).get()); - if body.as_slice().ends_with(".") { - body.push_char('0'); // `10.f` is not a float literal - } - body + LIT_INTEGER(c) | LIT_FLOAT(c) => { + c.as_str().to_string() } + LIT_STR(s) => { - format!("\"{}\"", get_ident(s).get().escape_default()) + format!("\"{}\"", s.as_str()) } LIT_STR_RAW(s, n) => { format!("r{delim}\"{string}\"{delim}", - delim="#".repeat(n), string=get_ident(s)) + delim="#".repeat(n), string=s.as_str()) } - LIT_BINARY(ref v) => { - format!( - "b\"{}\"", - v.iter().map(|&b| b as char).collect::<String>().escape_default()) + LIT_BINARY(v) => { + format!("b\"{}\"", v.as_str()) } - LIT_BINARY_RAW(ref s, n) => { + LIT_BINARY_RAW(s, n) => { format!("br{delim}\"{string}\"{delim}", - delim="#".repeat(n), string=s.as_slice().to_ascii().as_str_ascii()) + delim="#".repeat(n), string=s.as_str()) } /* Name components */ @@ -258,8 +239,12 @@ pub fn to_string(t: &Token) -> String { UNDERSCORE => "_".to_string(), /* Other */ - DOC_COMMENT(s) => get_ident(s).get().to_string(), + DOC_COMMENT(s) => s.as_str().to_string(), EOF => "<eof>".to_string(), + WS => " ".to_string(), + COMMENT => "/* */".to_string(), + SHEBANG(s) => format!("/* shebang: {}*/", s.as_str()), + INTERPOLATED(ref nt) => { match nt { &NtExpr(ref e) => ::print::pprust::expr_to_string(&**e), @@ -296,11 +281,8 @@ pub fn can_begin_expr(t: &Token) -> bool { TILDE => true, LIT_BYTE(_) => true, LIT_CHAR(_) => true, - LIT_INT(_, _) => true, - LIT_UINT(_, _) => true, - LIT_INT_UNSUFFIXED(_) => true, - LIT_FLOAT(_, _) => true, - LIT_FLOAT_UNSUFFIXED(_) => true, + LIT_INTEGER(_) => true, + LIT_FLOAT(_) => true, LIT_STR(_) => true, LIT_STR_RAW(_, _) => true, LIT_BINARY(_) => true, @@ -337,11 +319,8 @@ pub fn is_lit(t: &Token) -> bool { match *t { LIT_BYTE(_) => true, LIT_CHAR(_) => true, - LIT_INT(_, _) => true, - LIT_UINT(_, _) => true, - LIT_INT_UNSUFFIXED(_) => true, - LIT_FLOAT(_, _) => true, - LIT_FLOAT_UNSUFFIXED(_) => true, + LIT_INTEGER(_) => true, + LIT_FLOAT(_) => true, LIT_STR(_) => true, LIT_STR_RAW(_, _) => true, LIT_BINARY(_) => true, @@ -395,19 +374,19 @@ macro_rules! declare_special_idents_and_keywords {( $( ($rk_name:expr, $rk_variant:ident, $rk_str:expr); )* } ) => { - static STRICT_KEYWORD_START: Name = first!($( $sk_name, )*); - static STRICT_KEYWORD_FINAL: Name = last!($( $sk_name, )*); - static RESERVED_KEYWORD_START: Name = first!($( $rk_name, )*); - static RESERVED_KEYWORD_FINAL: Name = last!($( $rk_name, )*); + static STRICT_KEYWORD_START: Name = first!($( Name($sk_name), )*); + static STRICT_KEYWORD_FINAL: Name = last!($( Name($sk_name), )*); + static RESERVED_KEYWORD_START: Name = first!($( Name($rk_name), )*); + static RESERVED_KEYWORD_FINAL: Name = last!($( Name($rk_name), )*); pub mod special_idents { - use ast::Ident; - $( pub static $si_static: Ident = Ident { name: $si_name, ctxt: 0 }; )* + use ast::{Ident, Name}; + $( pub static $si_static: Ident = Ident { name: Name($si_name), ctxt: 0 }; )* } pub mod special_names { use ast::Name; - $( pub static $si_static: Name = $si_name; )* + $( pub static $si_static: Name = Name($si_name); )* } /** @@ -428,8 +407,8 @@ macro_rules! declare_special_idents_and_keywords {( impl Keyword { pub fn to_name(&self) -> Name { match *self { - $( $sk_variant => $sk_name, )* - $( $rk_variant => $rk_name, )* + $( $sk_variant => Name($sk_name), )* + $( $rk_variant => Name($rk_name), )* } } } @@ -448,8 +427,11 @@ macro_rules! declare_special_idents_and_keywords {( }} // If the special idents get renumbered, remember to modify these two as appropriate -pub static SELF_KEYWORD_NAME: Name = 1; -static STATIC_KEYWORD_NAME: Name = 2; +pub static SELF_KEYWORD_NAME: Name = Name(SELF_KEYWORD_NAME_NUM); +static STATIC_KEYWORD_NAME: Name = Name(STATIC_KEYWORD_NAME_NUM); + +pub static SELF_KEYWORD_NAME_NUM: u32 = 1; +static STATIC_KEYWORD_NAME_NUM: u32 = 2; // NB: leaving holes in the ident table is bad! a different ident will get // interned with the id from the hole, but it will be between the min and max @@ -459,8 +441,8 @@ declare_special_idents_and_keywords! { pub mod special_idents { // These ones are statics (0, invalid, ""); - (super::SELF_KEYWORD_NAME, self_, "self"); - (super::STATIC_KEYWORD_NAME, statik, "static"); + (super::SELF_KEYWORD_NAME_NUM, self_, "self"); + (super::STATIC_KEYWORD_NAME_NUM, statik, "static"); (3, static_lifetime, "'static"); // for matcher NTs @@ -500,8 +482,8 @@ declare_special_idents_and_keywords! { (29, Ref, "ref"); (30, Return, "return"); // Static and Self are also special idents (prefill de-dupes) - (super::STATIC_KEYWORD_NAME, Static, "static"); - (super::SELF_KEYWORD_NAME, Self, "self"); + (super::STATIC_KEYWORD_NAME_NUM, Static, "static"); + (super::SELF_KEYWORD_NAME_NUM, Self, "self"); (31, Struct, "struct"); (32, Super, "super"); (33, True, "true"); @@ -683,20 +665,20 @@ pub fn gensym(s: &str) -> Name { /// Maps a string to an identifier with an empty syntax context. #[inline] -pub fn str_to_ident(s: &str) -> ast::Ident { - ast::Ident::new(intern(s)) +pub fn str_to_ident(s: &str) -> Ident { + Ident::new(intern(s)) } /// Maps a string to a gensym'ed identifier. #[inline] -pub fn gensym_ident(s: &str) -> ast::Ident { - ast::Ident::new(gensym(s)) +pub fn gensym_ident(s: &str) -> Ident { + Ident::new(gensym(s)) } // create a fresh name that maps to the same string as the old one. // note that this guarantees that str_ptr_eq(ident_to_string(src),interner_get(fresh_name(src))); // that is, that the new name and the old one are connected to ptr_eq strings. -pub fn fresh_name(src: &ast::Ident) -> Name { +pub fn fresh_name(src: &Ident) -> Name { let interner = get_ident_interner(); interner.gensym_copy(src.name) // following: debug version. Could work in final except that it's incompatible with @@ -708,7 +690,7 @@ pub fn fresh_name(src: &ast::Ident) -> Name { // create a fresh mark. pub fn fresh_mark() -> Mrk { - gensym("mark") + gensym("mark").uint() as u32 } // See the macro above about the types of keywords @@ -722,10 +704,13 @@ pub fn is_keyword(kw: keywords::Keyword, tok: &Token) -> bool { pub fn is_any_keyword(tok: &Token) -> bool { match *tok { - token::IDENT(sid, false) => match sid.name { - SELF_KEYWORD_NAME | STATIC_KEYWORD_NAME | - STRICT_KEYWORD_START .. RESERVED_KEYWORD_FINAL => true, - _ => false, + token::IDENT(sid, false) => { + let n = sid.name; + + n == SELF_KEYWORD_NAME + || n == STATIC_KEYWORD_NAME + || STRICT_KEYWORD_START <= n + && n <= RESERVED_KEYWORD_FINAL }, _ => false } @@ -733,10 +718,13 @@ pub fn is_any_keyword(tok: &Token) -> bool { pub fn is_strict_keyword(tok: &Token) -> bool { match *tok { - token::IDENT(sid, false) => match sid.name { - SELF_KEYWORD_NAME | STATIC_KEYWORD_NAME | - STRICT_KEYWORD_START .. STRICT_KEYWORD_FINAL => true, - _ => false, + token::IDENT(sid, false) => { + let n = sid.name; + + n == SELF_KEYWORD_NAME + || n == STATIC_KEYWORD_NAME + || STRICT_KEYWORD_START <= n + && n <= STRICT_KEYWORD_FINAL }, _ => false, } @@ -744,9 +732,11 @@ pub fn is_strict_keyword(tok: &Token) -> bool { pub fn is_reserved_keyword(tok: &Token) -> bool { match *tok { - token::IDENT(sid, false) => match sid.name { - RESERVED_KEYWORD_START .. RESERVED_KEYWORD_FINAL => true, - _ => false, + token::IDENT(sid, false) => { + let n = sid.name; + + RESERVED_KEYWORD_START <= n + && n <= RESERVED_KEYWORD_FINAL }, _ => false, } @@ -768,7 +758,7 @@ mod test { use ext::mtwt; fn mark_ident(id : ast::Ident, m : ast::Mrk) -> ast::Ident { - ast::Ident{name:id.name,ctxt:mtwt::apply_mark(m,id.ctxt)} + ast::Ident { name: id.name, ctxt:mtwt::apply_mark(m, id.ctxt) } } #[test] fn mtwt_token_eq_test() { |