From 25c4676dfa805e027564116b9c37fee7aeaf1cc4 Mon Sep 17 00:00:00 2001 From: John Clements Date: Mon, 4 Feb 2013 13:15:17 -0800 Subject: Commenting, test cases, cleanup --- src/libsyntax/parse/common.rs | 26 +++++++++++++++++++++--- src/libsyntax/parse/mod.rs | 47 ++++++++++++++++++++++++++++++++++++++++++- src/libsyntax/parse/parser.rs | 43 +++++++++++++++++++++++++-------------- src/libsyntax/parse/token.rs | 1 + 4 files changed, 98 insertions(+), 19 deletions(-) (limited to 'src/libsyntax/parse') diff --git a/src/libsyntax/parse/common.rs b/src/libsyntax/parse/common.rs index e0d53fadfa0..7e74163b6bf 100644 --- a/src/libsyntax/parse/common.rs +++ b/src/libsyntax/parse/common.rs @@ -20,6 +20,8 @@ use core::option::{None, Option, Some}; use core::option; use std::oldmap::HashMap; +// seq_sep : a sequence separator (token) +// and whether a trailing separator is allowed. pub type seq_sep = { sep: Option, trailing_sep_allowed: bool @@ -51,6 +53,8 @@ pub impl Parser { + token_to_str(self.reader, self.token) + ~"`"); } + // expect and consume the token t. Signal an error if + // the next token is not t. fn expect(t: token::Token) { if self.token == t { self.bump(); @@ -88,6 +92,8 @@ pub impl Parser { return self.parse_ident(); } + // consume token 'tok' if it exists. Returns true if the given + // token was present, false otherwise. fn eat(tok: token::Token) -> bool { return if self.token == tok { self.bump(); true } else { false }; } @@ -185,6 +191,8 @@ pub impl Parser { } } + // expect and consume a GT. if a >> is seen, replace it + // with a single > and continue. fn expect_gt() { if self.token == token::GT { self.bump(); @@ -202,16 +210,19 @@ pub impl Parser { } } + // parse a sequence bracketed by '<' and '>', stopping + // before the '>'. fn parse_seq_to_before_gt(sep: Option, f: fn(Parser) -> T) -> ~[T] { let mut first = true; let mut v = ~[]; while self.token != token::GT + // wait... isn't this going to eat a whole '>>' ? && self.token != token::BINOP(token::SHR) { match sep { Some(ref t) => { if first { first = false; } - else { self.expect((*t)); } + else { self.expect(*t); } } _ => () } @@ -229,6 +240,7 @@ pub impl Parser { return v; } + // parse a sequence bracketed by '<' and '>' fn parse_seq_lt_gt(sep: Option, f: fn(Parser) -> T) -> spanned<~[T]> { let lo = self.span.lo; @@ -239,6 +251,9 @@ pub impl Parser { return spanned(lo, hi, result); } + // parse a sequence, including the closing delimiter. The function + // f must consume tokens until reaching the next separator or + // closing bracket. fn parse_seq_to_end(ket: token::Token, sep: seq_sep, f: fn(Parser) -> T) -> ~[T] { let val = self.parse_seq_to_before_end(ket, sep, f); @@ -246,7 +261,9 @@ pub impl Parser { return val; } - + // parse a sequence, not including the closing delimiter. The function + // f must consume tokens until reaching the next separator or + // closing bracket. fn parse_seq_to_before_end(ket: token::Token, sep: seq_sep, f: fn(Parser) -> T) -> ~[T] { let mut first: bool = true; @@ -255,7 +272,7 @@ pub impl Parser { match sep.sep { Some(ref t) => { if first { first = false; } - else { self.expect((*t)); } + else { self.expect(*t); } } _ => () } @@ -265,6 +282,9 @@ pub impl Parser { return v; } + // parse a sequence, including the closing delimiter. The function + // f must consume tokens until reaching the next separator or + // closing bracket. fn parse_unspanned_seq(bra: token::Token, ket: token::Token, sep: seq_sep, diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index b8e671b3265..b863e2bd0e4 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -183,7 +183,6 @@ pub fn new_parser_from_file(sess: parse_sess, let srdr = lexer::new_string_reader(sess.span_diagnostic, filemap, sess.interner); - Ok(Parser(sess, cfg, srdr as reader)) } @@ -222,3 +221,49 @@ pub fn new_parser_from_tts(sess: parse_sess, cfg: ast::crate_cfg, return Parser(sess, cfg, trdr as reader) } + +#[cfg(test)] +mod test { + use super::*; + use std::serialize::Encodable; + use std; + use core::dvec; + use core::str; + use util::testing::*; + + #[test] fn to_json_str (val: Encodable) -> ~str { + let bw = @io::BytesWriter {bytes: dvec::DVec(), pos: 0}; + val.encode(~std::json::Encoder(bw as io::Writer)); + str::from_bytes(bw.bytes.data) + } + + #[test] fn alltts () { + let tts = parse_tts_from_source_str( + ~"bogofile", + @~"fn foo (x : int) { x; }", + ~[], + new_parse_sess(None)); + check_equal(to_json_str(tts as Encodable::), + //[["tt_tok",["IDENT","fn"]]] + ~"abc" + ); + let ast1 = new_parser_from_tts(new_parse_sess(None),~[],tts) + .parse_item(~[]); + let ast2 = parse_item_from_source_str( + ~"bogofile", + @~"fn foo (x : int) { x; }", + ~[],~[], + new_parse_sess(None)); + check_equal(ast1,ast2); + } +} + +// +// Local Variables: +// mode: rust +// fill-column: 78; +// indent-tabs-mode: nil +// c-basic-offset: 4 +// buffer-file-coding-system: utf-8-unix +// End: +// diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index 6382413b081..3c1f306ff07 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -182,7 +182,8 @@ pure fn maybe_append(+lhs: ~[attribute], rhs: Option<~[attribute]>) /* ident is handled by common.rs */ -pub fn Parser(sess: parse_sess, +pub fn Parser(sess: parse_sess + , cfg: ast::crate_cfg, +rdr: reader) -> Parser { @@ -1238,6 +1239,8 @@ pub impl Parser { return e; } + // parse an optional separator followed by a kleene-style + // repetition token (+ or *). fn parse_sep_and_zerok() -> (Option, bool) { if self.token == token::BINOP(token::STAR) || self.token == token::BINOP(token::PLUS) { @@ -1258,20 +1261,18 @@ pub impl Parser { } } + // parse a single token tree from the input. fn parse_token_tree() -> token_tree { maybe_whole!(deref self, nt_tt); - fn parse_tt_tok(p: Parser, delim_ok: bool) -> token_tree { + fn parse_non_delim_tt_tok(p: Parser) -> token_tree { maybe_whole!(deref p, nt_tt); match p.token { token::RPAREN | token::RBRACE | token::RBRACKET - if !delim_ok => { + => { p.fatal(~"incorrect close delimiter: `" + token_to_str(p.reader, p.token) + ~"`"); } - token::EOF => { - p.fatal(~"file ended in the middle of a macro invocation"); - } /* we ought to allow different depths of unquotation */ token::DOLLAR if p.quote_depth > 0u => { p.bump(); @@ -1282,32 +1283,43 @@ pub impl Parser { seq_sep_none(), |p| p.parse_token_tree()); let (s, z) = p.parse_sep_and_zerok(); - return tt_seq(mk_sp(sp.lo ,p.span.hi), seq.node, s, z); + tt_seq(mk_sp(sp.lo ,p.span.hi), seq.node, s, z) } else { - return tt_nonterminal(sp, p.parse_ident()); + tt_nonterminal(sp, p.parse_ident()) } } - _ => { /* ok */ } + _ => { + parse_any_tt_tok(p) + } } + } + + // turn the next token into a tt_tok: + fn parse_any_tt_tok(p: Parser) -> token_tree{ let res = tt_tok(p.span, p.token); p.bump(); - return res; + res } - return match self.token { + match self.token { + token::EOF => { + self.fatal(~"file ended in the middle of a macro invocation"); + } token::LPAREN | token::LBRACE | token::LBRACKET => { // tjc: ?????? let ket = token::flip_delimiter(copy self.token); tt_delim(vec::append( - ~[parse_tt_tok(self, true)], + // the open delimiter: + ~[parse_any_tt_tok(self)], vec::append( self.parse_seq_to_before_end( ket, seq_sep_none(), |p| p.parse_token_tree()), - ~[parse_tt_tok(self, true)]))) + // the close delimiter: + ~[parse_any_tt_tok(self)]))) } - _ => parse_tt_tok(self, false) - }; + _ => parse_non_delim_tt_tok(self) + } } fn parse_all_token_trees() -> ~[token_tree] { @@ -3999,6 +4011,7 @@ pub impl Parser { } } + // // Local Variables: // mode: rust diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index b8d756d893a..1f8b04630e2 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -86,6 +86,7 @@ pub enum Token { LIT_STR(ast::ident), /* Name components */ + // an identifier contains an "is_mod_name" boolean. IDENT(ast::ident, bool), UNDERSCORE, LIFETIME(ast::ident), -- cgit 1.4.1-3-g733a5 From ded95d2c2891d7ffec1277cb4969ed956ff0bd25 Mon Sep 17 00:00:00 2001 From: John Clements Date: Fri, 8 Feb 2013 18:50:12 -0800 Subject: deriving_eq for tokens and binops Note that the replaced definition of equality on tokens contains a *huge* shortcut on INTERPOLATED tokens (those that contain ASTs), whereby any two INTERPOLATED tokens are considered equal. This seems like a really broken notion of equality, but it appears that the existing test cases and the compiler don't depend on it. Niko noticed this, BTW. Replace long definition of Eq on tokens and binops w --- src/libsyntax/parse/token.rs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) (limited to 'src/libsyntax/parse') diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index 1f8b04630e2..391e8b04336 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -25,6 +25,7 @@ use std::oldmap::HashMap; #[auto_encode] #[auto_decode] +#[deriving_eq] pub enum binop { PLUS, MINUS, @@ -518,12 +519,6 @@ pub fn reserved_keyword_table() -> HashMap<~str, ()> { words } -impl binop : cmp::Eq { - pure fn eq(&self, other: &binop) -> bool { - ((*self) as uint) == ((*other) as uint) - } - pure fn ne(&self, other: &binop) -> bool { !(*self).eq(other) } -} impl Token : cmp::Eq { pure fn eq(&self, other: &Token) -> bool { -- cgit 1.4.1-3-g733a5 From f9d789fa083220cc9d84cbea94868606600c64a9 Mon Sep 17 00:00:00 2001 From: John Clements Date: Wed, 13 Feb 2013 15:38:42 -0800 Subject: cleanup, fix test case --- src/libcore/dvec.rs | 2 +- src/libsyntax/ext/base.rs | 2 +- src/libsyntax/parse/common.rs | 1 - src/libsyntax/parse/mod.rs | 13 +++++++++++-- 4 files changed, 13 insertions(+), 5 deletions(-) (limited to 'src/libsyntax/parse') diff --git a/src/libcore/dvec.rs b/src/libcore/dvec.rs index c7a0300a978..fe36ed15960 100644 --- a/src/libcore/dvec.rs +++ b/src/libcore/dvec.rs @@ -229,7 +229,7 @@ impl DVec { impl DVec { /** - * Append all elements of a vector to the end of the list. + * Append all elements of a vector to the end of the list * * Equivalent to `append_iter()` but potentially more efficient. */ diff --git a/src/libsyntax/ext/base.rs b/src/libsyntax/ext/base.rs index a85a7990ace..c924acd577d 100644 --- a/src/libsyntax/ext/base.rs +++ b/src/libsyntax/ext/base.rs @@ -59,7 +59,7 @@ pub enum MacResult { MRExpr(@ast::expr), MRItem(@ast::item), MRAny(fn@()-> @ast::expr, fn@()-> Option<@ast::item>, fn@()->@ast::stmt), - MRDef(MacroDef), + MRDef(MacroDef) } pub enum SyntaxExtension { diff --git a/src/libsyntax/parse/common.rs b/src/libsyntax/parse/common.rs index 7e74163b6bf..e7b5005d8db 100644 --- a/src/libsyntax/parse/common.rs +++ b/src/libsyntax/parse/common.rs @@ -217,7 +217,6 @@ pub impl Parser { let mut first = true; let mut v = ~[]; while self.token != token::GT - // wait... isn't this going to eat a whole '>>' ? && self.token != token::BINOP(token::SHR) { match sep { Some(ref t) => { diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index b863e2bd0e4..12038898a9d 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -244,8 +244,17 @@ mod test { ~[], new_parse_sess(None)); check_equal(to_json_str(tts as Encodable::), - //[["tt_tok",["IDENT","fn"]]] - ~"abc" + ~"[[\"tt_tok\",[,[\"IDENT\",[\"fn\",false]]]],\ + [\"tt_tok\",[,[\"IDENT\",[\"foo\",false]]]],\ + [\"tt_delim\",[[[\"tt_tok\",[,[\"LPAREN\",[]]]],\ + [\"tt_tok\",[,[\"IDENT\",[\"x\",false]]]],\ + [\"tt_tok\",[,[\"COLON\",[]]]],\ + [\"tt_tok\",[,[\"IDENT\",[\"int\",false]]]],\ + [\"tt_tok\",[,[\"RPAREN\",[]]]]]]],\ + [\"tt_delim\",[[[\"tt_tok\",[,[\"LBRACE\",[]]]],\ + [\"tt_tok\",[,[\"IDENT\",[\"x\",false]]]],\ + [\"tt_tok\",[,[\"SEMI\",[]]]],\ + [\"tt_tok\",[,[\"RBRACE\",[]]]]]]]]" ); let ast1 = new_parser_from_tts(new_parse_sess(None),~[],tts) .parse_item(~[]); -- cgit 1.4.1-3-g733a5