diff options
Diffstat (limited to 'src/libsyntax/parse')
| -rw-r--r-- | src/libsyntax/parse/lexer/mod.rs | 179 | ||||
| -rw-r--r-- | src/libsyntax/parse/mod.rs | 145 | ||||
| -rw-r--r-- | src/libsyntax/parse/parser.rs | 132 | ||||
| -rw-r--r-- | src/libsyntax/parse/token.rs | 55 |
4 files changed, 288 insertions, 223 deletions
diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs index 01a66243a96..fbca4868255 100644 --- a/src/libsyntax/parse/lexer/mod.rs +++ b/src/libsyntax/parse/lexer/mod.rs @@ -369,6 +369,25 @@ impl<'a> StringReader<'a> { self.nextnextch() == Some(c) } + /// Eats <XID_start><XID_continue>*, if possible. + fn scan_optional_raw_name(&mut self) -> Option<ast::Name> { + if !ident_start(self.curr) { + return None + } + let start = self.last_pos; + while ident_continue(self.curr) { + self.bump(); + } + + self.with_str_from(start, |string| { + if string == "_" { + None + } else { + Some(token::intern(string)) + } + }) + } + /// PRECONDITION: self.curr is not whitespace /// Eats any kind of comment. fn scan_comment(&mut self) -> Option<TokenAndSpan> { @@ -638,7 +657,7 @@ impl<'a> StringReader<'a> { } /// Lex a LIT_INTEGER or a LIT_FLOAT - fn scan_number(&mut self, c: char) -> token::Token { + fn scan_number(&mut self, c: char) -> token::Lit { let mut num_digits; let mut base = 10; let start_bpos = self.last_pos; @@ -653,19 +672,9 @@ impl<'a> StringReader<'a> { '0'...'9' | '_' | '.' => { num_digits = self.scan_digits(10) + 1; } - 'u' | 'i' => { - self.scan_int_suffix(); - return token::LitInteger(self.name_from(start_bpos)); - }, - 'f' => { - let last_pos = self.last_pos; - self.scan_float_suffix(); - self.check_float_base(start_bpos, last_pos, base); - return token::LitFloat(self.name_from(start_bpos)); - } _ => { // just a 0 - return token::LitInteger(self.name_from(start_bpos)); + return token::Integer(self.name_from(start_bpos)); } } } else if c.is_digit_radix(10) { @@ -676,9 +685,7 @@ impl<'a> StringReader<'a> { if num_digits == 0 { self.err_span_(start_bpos, self.last_pos, "no valid digits found for number"); - // eat any suffix - self.scan_int_suffix(); - return token::LitInteger(token::intern("0")); + return token::Integer(token::intern("0")); } // might be a float, but don't be greedy if this is actually an @@ -692,29 +699,20 @@ impl<'a> StringReader<'a> { if self.curr.unwrap_or('\0').is_digit_radix(10) { self.scan_digits(10); self.scan_float_exponent(); - self.scan_float_suffix(); } let last_pos = self.last_pos; self.check_float_base(start_bpos, last_pos, base); - return token::LitFloat(self.name_from(start_bpos)); - } else if self.curr_is('f') { - // or it might be an integer literal suffixed as a float - self.scan_float_suffix(); - let last_pos = self.last_pos; - self.check_float_base(start_bpos, last_pos, base); - return token::LitFloat(self.name_from(start_bpos)); + return token::Float(self.name_from(start_bpos)); } else { // it might be a float if it has an exponent if self.curr_is('e') || self.curr_is('E') { self.scan_float_exponent(); - self.scan_float_suffix(); let last_pos = self.last_pos; self.check_float_base(start_bpos, last_pos, base); - return token::LitFloat(self.name_from(start_bpos)); + return token::Float(self.name_from(start_bpos)); } // but we certainly have an integer! - self.scan_int_suffix(); - return token::LitInteger(self.name_from(start_bpos)); + return token::Integer(self.name_from(start_bpos)); } } @@ -850,55 +848,6 @@ impl<'a> StringReader<'a> { true } - /// Scan over an int literal suffix. - fn scan_int_suffix(&mut self) { - match self.curr { - Some('i') | Some('u') => { - self.bump(); - - if self.curr_is('8') { - self.bump(); - } else if self.curr_is('1') { - if !self.nextch_is('6') { - self.err_span_(self.last_pos, self.pos, - "illegal int suffix"); - } else { - self.bump(); self.bump(); - } - } else if self.curr_is('3') { - if !self.nextch_is('2') { - self.err_span_(self.last_pos, self.pos, - "illegal int suffix"); - } else { - self.bump(); self.bump(); - } - } else if self.curr_is('6') { - if !self.nextch_is('4') { - self.err_span_(self.last_pos, self.pos, - "illegal int suffix"); - } else { - self.bump(); self.bump(); - } - } - }, - _ => { } - } - } - - /// Scan over a float literal suffix - fn scan_float_suffix(&mut self) { - if self.curr_is('f') { - if (self.nextch_is('3') && self.nextnextch_is('2')) - || (self.nextch_is('6') && self.nextnextch_is('4')) { - self.bump(); - self.bump(); - self.bump(); - } else { - self.err_span_(self.last_pos, self.pos, "illegal float suffix"); - } - } - } - /// Scan over a float exponent. fn scan_float_exponent(&mut self) { if self.curr_is('e') || self.curr_is('E') { @@ -967,7 +916,10 @@ impl<'a> StringReader<'a> { } if is_dec_digit(c) { - return self.scan_number(c.unwrap()); + let num = self.scan_number(c.unwrap()); + let suffix = self.scan_optional_raw_name(); + debug!("next_token_inner: scanned number {}, {}", num, suffix); + return token::Literal(num, suffix) } if self.read_embedded_ident { @@ -1126,17 +1078,19 @@ impl<'a> StringReader<'a> { } let id = if valid { self.name_from(start) } else { token::intern("0") }; self.bump(); // advance curr past token - return token::LitChar(id); + let suffix = self.scan_optional_raw_name(); + return token::Literal(token::Char(id), suffix); } 'b' => { self.bump(); - return match self.curr { + let lit = match self.curr { Some('\'') => self.scan_byte(), Some('"') => self.scan_byte_string(), Some('r') => self.scan_raw_byte_string(), _ => unreachable!() // Should have been a token::Ident above. }; - + let suffix = self.scan_optional_raw_name(); + return token::Literal(lit, suffix); } '"' => { let start_bpos = self.last_pos; @@ -1157,7 +1111,8 @@ impl<'a> StringReader<'a> { let id = if valid { self.name_from(start_bpos + BytePos(1)) } else { token::intern("??") }; self.bump(); - return token::LitStr(id); + let suffix = self.scan_optional_raw_name(); + return token::Literal(token::Str_(id), suffix); } 'r' => { let start_bpos = self.last_pos; @@ -1224,7 +1179,8 @@ impl<'a> StringReader<'a> { } else { token::intern("??") }; - return token::LitStrRaw(id, hash_count); + let suffix = self.scan_optional_raw_name(); + return token::Literal(token::StrRaw(id, hash_count), suffix); } '-' => { if self.nextch_is('>') { @@ -1293,7 +1249,7 @@ impl<'a> StringReader<'a> { || (self.curr_is('#') && self.nextch_is('!') && !self.nextnextch_is('[')) } - fn scan_byte(&mut self) -> token::Token { + fn scan_byte(&mut self) -> token::Lit { self.bump(); let start = self.last_pos; @@ -1314,10 +1270,10 @@ impl<'a> StringReader<'a> { let id = if valid { self.name_from(start) } else { token::intern("??") }; self.bump(); // advance curr past token - return token::LitByte(id); + return token::Byte(id); } - fn scan_byte_string(&mut self) -> token::Token { + fn scan_byte_string(&mut self) -> token::Lit { self.bump(); let start = self.last_pos; let mut valid = true; @@ -1336,10 +1292,10 @@ impl<'a> StringReader<'a> { } let id = if valid { self.name_from(start) } else { token::intern("??") }; self.bump(); - return token::LitBinary(id); + return token::Binary(id); } - fn scan_raw_byte_string(&mut self) -> token::Token { + fn scan_raw_byte_string(&mut self) -> token::Lit { let start_bpos = self.last_pos; self.bump(); let mut hash_count = 0u; @@ -1387,8 +1343,9 @@ impl<'a> StringReader<'a> { self.bump(); } self.bump(); - return token::LitBinaryRaw(self.name_from_to(content_start_bpos, content_end_bpos), - hash_count); + return token::BinaryRaw(self.name_from_to(content_start_bpos, + content_end_bpos), + hash_count); } } @@ -1535,17 +1492,17 @@ mod test { #[test] fn character_a() { assert_eq!(setup(&mk_sh(), "'a'".to_string()).next_token().tok, - token::LitChar(token::intern("a"))); + token::Literal(token::Char(token::intern("a")), None)); } #[test] fn character_space() { assert_eq!(setup(&mk_sh(), "' '".to_string()).next_token().tok, - token::LitChar(token::intern(" "))); + token::Literal(token::Char(token::intern(" ")), None)); } #[test] fn character_escaped() { assert_eq!(setup(&mk_sh(), "'\\n'".to_string()).next_token().tok, - token::LitChar(token::intern("\\n"))); + token::Literal(token::Char(token::intern("\\n")), None)); } #[test] fn lifetime_name() { @@ -1557,7 +1514,41 @@ mod test { assert_eq!(setup(&mk_sh(), "r###\"\"#a\\b\x00c\"\"###".to_string()).next_token() .tok, - token::LitStrRaw(token::intern("\"#a\\b\x00c\""), 3)); + token::Literal(token::StrRaw(token::intern("\"#a\\b\x00c\""), 3), None)); + } + + #[test] fn literal_suffixes() { + macro_rules! test { + ($input: expr, $tok_type: ident, $tok_contents: expr) => {{ + assert_eq!(setup(&mk_sh(), format!("{}suffix", $input)).next_token().tok, + token::Literal(token::$tok_type(token::intern($tok_contents)), + Some(token::intern("suffix")))); + // with a whitespace separator: + assert_eq!(setup(&mk_sh(), format!("{} suffix", $input)).next_token().tok, + token::Literal(token::$tok_type(token::intern($tok_contents)), + None)); + }} + } + + test!("'a'", Char, "a"); + test!("b'a'", Byte, "a"); + test!("\"a\"", Str_, "a"); + test!("b\"a\"", Binary, "a"); + test!("1234", Integer, "1234"); + test!("0b101", Integer, "0b101"); + test!("0xABC", Integer, "0xABC"); + test!("1.0", Float, "1.0"); + test!("1.0e10", Float, "1.0e10"); + + assert_eq!(setup(&mk_sh(), "2u".to_string()).next_token().tok, + token::Literal(token::Integer(token::intern("2")), + Some(token::intern("u")))); + assert_eq!(setup(&mk_sh(), "r###\"raw\"###suffix".to_string()).next_token().tok, + token::Literal(token::StrRaw(token::intern("raw"), 3), + Some(token::intern("suffix")))); + assert_eq!(setup(&mk_sh(), "br###\"raw\"###suffix".to_string()).next_token().tok, + token::Literal(token::BinaryRaw(token::intern("raw"), 3), + Some(token::intern("suffix")))); } #[test] fn line_doc_comments() { @@ -1573,7 +1564,7 @@ mod test { token::Comment => { }, _ => panic!("expected a comment!") } - assert_eq!(lexer.next_token().tok, token::LitChar(token::intern("a"))); + assert_eq!(lexer.next_token().tok, token::Literal(token::Char(token::intern("a")), None)); } } diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index 3ce49b9d7a3..96659031e6a 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -511,28 +511,41 @@ pub fn raw_str_lit(lit: &str) -> String { res } -pub fn float_lit(s: &str) -> ast::Lit_ { - debug!("float_lit: {}", s); - // FIXME #2252: bounds checking float literals is defered until trans - let s2 = s.chars().filter(|&c| c != '_').collect::<String>(); - let s = s2.as_slice(); - - let mut ty = None; - - if s.ends_with("f32") { - ty = Some(ast::TyF32); - } else if s.ends_with("f64") { - ty = Some(ast::TyF64); - } +// check if `s` looks like i32 or u1234 etc. +fn looks_like_width_suffix(first_chars: &[char], s: &str) -> bool { + s.len() > 1 && + first_chars.contains(&s.char_at(0)) && + s.slice_from(1).chars().all(|c| '0' <= c && c <= '9') +} +fn filtered_float_lit(data: token::InternedString, suffix: Option<&str>, + sd: &SpanHandler, sp: Span) -> ast::Lit_ { + debug!("filtered_float_lit: {}, {}", data, suffix); + match suffix { + Some("f32") => ast::LitFloat(data, ast::TyF32), + Some("f64") => ast::LitFloat(data, ast::TyF64), + Some(suf) => { + if suf.len() >= 2 && looks_like_width_suffix(&['f'], suf) { + // if it looks like a width, lets try to be helpful. + sd.span_err(sp, &*format!("illegal width `{}` for float literal, \ + valid widths are 32 and 64", suf.slice_from(1))); + } else { + sd.span_err(sp, &*format!("illegal suffix `{}` for float literal, \ + valid suffixes are `f32` and `f64`", suf)); + } - match ty { - Some(t) => { - ast::LitFloat(token::intern_and_get_ident(s.slice_to(s.len() - t.suffix_len())), t) - }, - None => ast::LitFloatUnsuffixed(token::intern_and_get_ident(s)) + ast::LitFloatUnsuffixed(data) + } + None => ast::LitFloatUnsuffixed(data) } } +pub fn float_lit(s: &str, suffix: Option<&str>, sd: &SpanHandler, sp: Span) -> ast::Lit_ { + debug!("float_lit: {}, {}", s, suffix); + // FIXME #2252: bounds checking float literals is defered until trans + let s = s.chars().filter(|&c| c != '_').collect::<String>(); + let data = token::intern_and_get_ident(&*s); + filtered_float_lit(data, suffix, sd, sp) +} /// Parse a string representing a byte literal into its final form. Similar to `char_lit` pub fn byte_lit(lit: &str) -> (u8, uint) { @@ -626,24 +639,19 @@ pub fn binary_lit(lit: &str) -> Rc<Vec<u8>> { Rc::new(res) } -pub fn integer_lit(s: &str, sd: &SpanHandler, sp: Span) -> ast::Lit_ { +pub fn integer_lit(s: &str, suffix: Option<&str>, sd: &SpanHandler, sp: Span) -> ast::Lit_ { // s can only be ascii, byte indexing is fine let s2 = s.chars().filter(|&c| c != '_').collect::<String>(); let mut s = s2.as_slice(); - debug!("parse_integer_lit: {}", s); - - if s.len() == 1 { - let n = (s.char_at(0)).to_digit(10).unwrap(); - return ast::LitInt(n as u64, ast::UnsuffixedIntLit(ast::Sign::new(n))); - } + debug!("integer_lit: {}, {}", s, suffix); let mut base = 10; let orig = s; let mut ty = ast::UnsuffixedIntLit(ast::Plus); - if s.char_at(0) == '0' { + if s.char_at(0) == '0' && s.len() > 1 { match s.char_at(1) { 'x' => base = 16, 'o' => base = 8, @@ -652,57 +660,56 @@ pub fn integer_lit(s: &str, sd: &SpanHandler, sp: Span) -> ast::Lit_ { } } + // 1f64 and 2f32 etc. are valid float literals. + match suffix { + Some(suf) if looks_like_width_suffix(&['f'], suf) => { + match base { + 16u => sd.span_err(sp, "hexadecimal float literal is not supported"), + 8u => sd.span_err(sp, "octal float literal is not supported"), + 2u => sd.span_err(sp, "binary float literal is not supported"), + _ => () + } + let ident = token::intern_and_get_ident(&*s); + return filtered_float_lit(ident, suffix, sd, sp) + } + _ => {} + } + if base != 10 { s = s.slice_from(2); } - let last = s.len() - 1; - match s.char_at(last) { - 'i' => ty = ast::SignedIntLit(ast::TyI, ast::Plus), - 'u' => ty = ast::UnsignedIntLit(ast::TyU), - '8' => { - if s.len() > 2 { - match s.char_at(last - 1) { - 'i' => ty = ast::SignedIntLit(ast::TyI8, ast::Plus), - 'u' => ty = ast::UnsignedIntLit(ast::TyU8), - _ => { } - } - } - }, - '6' => { - if s.len() > 3 && s.char_at(last - 1) == '1' { - match s.char_at(last - 2) { - 'i' => ty = ast::SignedIntLit(ast::TyI16, ast::Plus), - 'u' => ty = ast::UnsignedIntLit(ast::TyU16), - _ => { } - } - } - }, - '2' => { - if s.len() > 3 && s.char_at(last - 1) == '3' { - match s.char_at(last - 2) { - 'i' => ty = ast::SignedIntLit(ast::TyI32, ast::Plus), - 'u' => ty = ast::UnsignedIntLit(ast::TyU32), - _ => { } - } - } - }, - '4' => { - if s.len() > 3 && s.char_at(last - 1) == '6' { - match s.char_at(last - 2) { - 'i' => ty = ast::SignedIntLit(ast::TyI64, ast::Plus), - 'u' => ty = ast::UnsignedIntLit(ast::TyU64), - _ => { } + if let Some(suf) = suffix { + if suf.is_empty() { sd.span_bug(sp, "found empty literal suffix in Some")} + ty = match suf { + "i" => ast::SignedIntLit(ast::TyI, ast::Plus), + "i8" => ast::SignedIntLit(ast::TyI8, ast::Plus), + "i16" => ast::SignedIntLit(ast::TyI16, ast::Plus), + "i32" => ast::SignedIntLit(ast::TyI32, ast::Plus), + "i64" => ast::SignedIntLit(ast::TyI64, ast::Plus), + "u" => ast::UnsignedIntLit(ast::TyU), + "u8" => ast::UnsignedIntLit(ast::TyU8), + "u16" => ast::UnsignedIntLit(ast::TyU16), + "u32" => ast::UnsignedIntLit(ast::TyU32), + "u64" => ast::UnsignedIntLit(ast::TyU64), + _ => { + // i<digits> and u<digits> look like widths, so lets + // give an error message along those lines + if looks_like_width_suffix(&['i', 'u'], suf) { + sd.span_err(sp, &*format!("illegal width `{}` for integer literal; \ + valid widths are 8, 16, 32 and 64", + suf.slice_from(1))); + } else { + sd.span_err(sp, &*format!("illegal suffix `{}` for numeric literal", suf)); } + + ty } - }, - _ => { } + } } - debug!("The suffix is {}, base {}, the new string is {}, the original \ - string was {}", ty, base, s, orig); - - s = s.slice_to(s.len() - ty.suffix_len()); + debug!("integer_lit: the type is {}, base {}, the new string is {}, the original \ + string was {}, the original suffix was {}", ty, base, s, orig, suffix); let res: u64 = match ::std::num::from_str_radix(s, base) { Some(r) => r, diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index 50b1a2204b0..a6fe3902395 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -646,6 +646,20 @@ impl<'a> Parser<'a> { } } + pub fn expect_no_suffix(&mut self, sp: Span, kind: &str, suffix: Option<ast::Name>) { + match suffix { + None => {/* everything ok */} + Some(suf) => { + let text = suf.as_str(); + if text.is_empty() { + self.span_bug(sp, "found empty literal suffix in Some") + } + self.span_err(sp, &*format!("{} with a suffix is illegal", kind)); + } + } + } + + /// Attempt to consume a `<`. If `<<` is seen, replace it with a single /// `<` and continue. If a `<` is not seen, return false. /// @@ -968,6 +982,9 @@ impl<'a> Parser<'a> { pub fn span_err(&mut self, sp: Span, m: &str) { self.sess.span_diagnostic.span_err(sp, m) } + pub fn span_bug(&mut self, sp: Span, m: &str) -> ! { + self.sess.span_diagnostic.span_bug(sp, m) + } pub fn abort_if_errors(&mut self) { self.sess.span_diagnostic.handler().abort_if_errors(); } @@ -1502,17 +1519,17 @@ impl<'a> Parser<'a> { } else if self.eat_keyword(keywords::Proc) { self.parse_proc_type(Vec::new()) } else if self.token == token::Lt { - // QUALIFIED PATH + // QUALIFIED PATH `<TYPE as TRAIT_REF>::item` self.bump(); - let for_type = self.parse_ty(true); + let self_type = self.parse_ty(true); self.expect_keyword(keywords::As); - let trait_name = self.parse_path(LifetimeAndTypesWithoutColons); + let trait_ref = self.parse_trait_ref(); self.expect(&token::Gt); self.expect(&token::ModSep); let item_name = self.parse_ident(); TyQPath(P(QPath { - for_type: for_type, - trait_name: trait_name.path, + self_type: self_type, + trait_ref: P(trait_ref), item_name: item_name, })) } else if self.token == token::ModSep || @@ -1640,24 +1657,53 @@ impl<'a> Parser<'a> { /// Matches token_lit = LIT_INTEGER | ... pub fn lit_from_token(&mut self, tok: &token::Token) -> Lit_ { match *tok { - token::LitByte(i) => LitByte(parse::byte_lit(i.as_str()).val0()), - token::LitChar(i) => LitChar(parse::char_lit(i.as_str()).val0()), - token::LitInteger(s) => parse::integer_lit(s.as_str(), - &self.sess.span_diagnostic, - self.last_span), - token::LitFloat(s) => parse::float_lit(s.as_str()), - token::LitStr(s) => { - LitStr(token::intern_and_get_ident(parse::str_lit(s.as_str()).as_slice()), - ast::CookedStr) - } - token::LitStrRaw(s, n) => { - LitStr(token::intern_and_get_ident(parse::raw_str_lit(s.as_str()).as_slice()), - ast::RawStr(n)) + token::Literal(lit, suf) => { + let (suffix_illegal, out) = match lit { + token::Byte(i) => (true, LitByte(parse::byte_lit(i.as_str()).val0())), + token::Char(i) => (true, LitChar(parse::char_lit(i.as_str()).val0())), + + // there are some valid suffixes for integer and + // float literals, so all the handling is done + // internally. + token::Integer(s) => { + (false, parse::integer_lit(s.as_str(), + suf.as_ref().map(|s| s.as_str()), + &self.sess.span_diagnostic, + self.last_span)) + } + token::Float(s) => { + (false, parse::float_lit(s.as_str(), + suf.as_ref().map(|s| s.as_str()), + &self.sess.span_diagnostic, + self.last_span)) + } + + token::Str_(s) => { + (true, + LitStr(token::intern_and_get_ident(parse::str_lit(s.as_str()).as_slice()), + ast::CookedStr)) + } + token::StrRaw(s, n) => { + (true, + LitStr( + token::intern_and_get_ident( + parse::raw_str_lit(s.as_str()).as_slice()), + ast::RawStr(n))) + } + token::Binary(i) => + (true, LitBinary(parse::binary_lit(i.as_str()))), + token::BinaryRaw(i, _) => + (true, + LitBinary(Rc::new(i.as_str().as_bytes().iter().map(|&x| x).collect()))), + }; + + if suffix_illegal { + let sp = self.last_span; + self.expect_no_suffix(sp, &*format!("{} literal", lit.short_name()), suf) + } + + out } - token::LitBinary(i) => - LitBinary(parse::binary_lit(i.as_str())), - token::LitBinaryRaw(i, _) => - LitBinary(Rc::new(i.as_str().as_bytes().iter().map(|&x| x).collect())), _ => { self.unexpected_last(tok); } } } @@ -2424,7 +2470,10 @@ impl<'a> Parser<'a> { } } } - token::LitInteger(n) => { + token::Literal(token::Integer(n), suf) => { + let sp = self.span; + self.expect_no_suffix(sp, "tuple index", suf); + let index = n.as_str(); let dot = self.last_span.hi; hi = self.span.hi; @@ -2449,7 +2498,7 @@ impl<'a> Parser<'a> { } } } - token::LitFloat(n) => { + token::Literal(token::Float(n), _suf) => { self.bump(); let last_span = self.last_span; let fstr = n.as_str(); @@ -5085,12 +5134,17 @@ impl<'a> Parser<'a> { self.expect(&token::Semi); (path, the_ident) }, - token::LitStr(..) | token::LitStrRaw(..) => { - let path = self.parse_str(); + token::Literal(token::Str_(..), suf) | token::Literal(token::StrRaw(..), suf) => { + let sp = self.span; + self.expect_no_suffix(sp, "extern crate name", suf); + // forgo the internal suffix check of `parse_str` to + // avoid repeats (this unwrap will always succeed due + // to the restriction of the `match`) + let (s, style, _) = self.parse_optional_str().unwrap(); self.expect_keyword(keywords::As); let the_ident = self.parse_ident(); self.expect(&token::Semi); - (Some(path), the_ident) + (Some((s, style)), the_ident) }, _ => { let span = self.span; @@ -5267,7 +5321,9 @@ impl<'a> Parser<'a> { /// the `extern` keyword, if one is found. fn parse_opt_abi(&mut self) -> Option<abi::Abi> { match self.token { - token::LitStr(s) | token::LitStrRaw(s, _) => { + token::Literal(token::Str_(s), suf) | token::Literal(token::StrRaw(s, _), suf) => { + let sp = self.span; + self.expect_no_suffix(sp, "ABI spec", suf); self.bump(); let the_string = s.as_str(); match abi::lookup(the_string) { @@ -5910,21 +5966,27 @@ impl<'a> Parser<'a> { } pub fn parse_optional_str(&mut self) - -> Option<(InternedString, ast::StrStyle)> { - let (s, style) = match self.token { - token::LitStr(s) => (self.id_to_interned_str(s.ident()), ast::CookedStr), - token::LitStrRaw(s, n) => { - (self.id_to_interned_str(s.ident()), ast::RawStr(n)) + -> Option<(InternedString, ast::StrStyle, Option<ast::Name>)> { + let ret = match self.token { + token::Literal(token::Str_(s), suf) => { + (self.id_to_interned_str(s.ident()), ast::CookedStr, suf) + } + token::Literal(token::StrRaw(s, n), suf) => { + (self.id_to_interned_str(s.ident()), ast::RawStr(n), suf) } _ => return None }; self.bump(); - Some((s, style)) + Some(ret) } pub fn parse_str(&mut self) -> (InternedString, StrStyle) { match self.parse_optional_str() { - Some(s) => { s } + Some((s, style, suf)) => { + let sp = self.last_span; + self.expect_no_suffix(sp, "str literal", suf); + (s, style) + } _ => self.fatal("expected string literal") } } diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index 298328d73ef..4272b57a4dc 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -12,6 +12,7 @@ pub use self::BinOpToken::*; pub use self::Nonterminal::*; pub use self::DelimToken::*; pub use self::IdentStyle::*; +pub use self::Lit::*; pub use self::Token::*; use ast; @@ -59,6 +60,31 @@ pub enum IdentStyle { Plain, } +#[deriving(Clone, Encodable, Decodable, PartialEq, Eq, Hash, Show)] +pub enum Lit { + Byte(ast::Name), + Char(ast::Name), + Integer(ast::Name), + Float(ast::Name), + Str_(ast::Name), + StrRaw(ast::Name, uint), /* raw str delimited by n hash symbols */ + Binary(ast::Name), + BinaryRaw(ast::Name, uint), /* raw binary str delimited by n hash symbols */ +} + +impl Lit { + pub fn short_name(&self) -> &'static str { + match *self { + Byte(_) => "byte", + Char(_) => "char", + Integer(_) => "integer", + Float(_) => "float", + Str_(_) | StrRaw(..) => "str", + Binary(_) | BinaryRaw(..) => "binary str" + } + } +} + #[allow(non_camel_case_types)] #[deriving(Clone, Encodable, Decodable, PartialEq, Eq, Hash, Show)] pub enum Token { @@ -98,14 +124,7 @@ pub enum Token { CloseDelim(DelimToken), /* Literals */ - LitByte(ast::Name), - LitChar(ast::Name), - LitInteger(ast::Name), - LitFloat(ast::Name), - LitStr(ast::Name), - LitStrRaw(ast::Name, uint), /* raw str delimited by n hash symbols */ - LitBinary(ast::Name), - LitBinaryRaw(ast::Name, uint), /* raw binary str delimited by n hash symbols */ + Literal(Lit, Option<ast::Name>), /* Name components */ Ident(ast::Ident, IdentStyle), @@ -145,14 +164,7 @@ impl Token { Ident(_, _) => true, Underscore => true, Tilde => true, - LitByte(_) => true, - LitChar(_) => true, - LitInteger(_) => true, - LitFloat(_) => true, - LitStr(_) => true, - LitStrRaw(_, _) => true, - LitBinary(_) => true, - LitBinaryRaw(_, _) => true, + Literal(_, _) => true, Pound => true, At => true, Not => true, @@ -173,15 +185,8 @@ impl Token { /// Returns `true` if the token is any literal pub fn is_lit(&self) -> bool { match *self { - LitByte(_) => true, - LitChar(_) => true, - LitInteger(_) => true, - LitFloat(_) => true, - LitStr(_) => true, - LitStrRaw(_, _) => true, - LitBinary(_) => true, - LitBinaryRaw(_, _) => true, - _ => false, + Literal(_, _) => true, + _ => false, } } |