diff options
Diffstat (limited to 'src/libsyntax/parse')
| -rw-r--r-- | src/libsyntax/parse/attr.rs | 5 | ||||
| -rw-r--r-- | src/libsyntax/parse/lexer/mod.rs | 27 | ||||
| -rw-r--r-- | src/libsyntax/parse/parser.rs | 315 | ||||
| -rw-r--r-- | src/libsyntax/parse/token.rs | 35 |
4 files changed, 161 insertions, 221 deletions
diff --git a/src/libsyntax/parse/attr.rs b/src/libsyntax/parse/attr.rs index 3cb34fa3c91..983c882eafc 100644 --- a/src/libsyntax/parse/attr.rs +++ b/src/libsyntax/parse/attr.rs @@ -215,7 +215,10 @@ impl<'a> Parser<'a> { /// meta_item_inner : (meta_item | UNSUFFIXED_LIT) (',' meta_item_inner)? ; pub fn parse_meta_item(&mut self) -> PResult<'a, P<ast::MetaItem>> { let nt_meta = match self.token { - token::Interpolated(token::NtMeta(ref e)) => Some(e.clone()), + token::Interpolated(ref nt) => match **nt { + token::NtMeta(ref e) => Some(e.clone()), + _ => None, + }, _ => None, }; diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs index 5e20f6e4192..cf48c445c80 100644 --- a/src/libsyntax/parse/lexer/mod.rs +++ b/src/libsyntax/parse/lexer/mod.rs @@ -22,7 +22,7 @@ use std::char; use std::mem::replace; use std::rc::Rc; -pub use ext::tt::transcribe::{TtReader, new_tt_reader, new_tt_reader_with_doc_flag}; +pub use ext::tt::transcribe::{TtReader, new_tt_reader}; pub mod comments; mod unicode_chars; @@ -171,31 +171,10 @@ impl<'a> Reader for TtReader<'a> { self.fatal_errs.clear(); } fn peek(&self) -> TokenAndSpan { - self.next_tok.clone().unwrap_or(TokenAndSpan { + TokenAndSpan { tok: self.cur_tok.clone(), sp: self.cur_span, - }) - } -} - -impl<'a, 'b> Reader for &'b mut TtReader<'a> { - fn is_eof(&self) -> bool { - (**self).is_eof() - } - fn try_next_token(&mut self) -> Result<TokenAndSpan, ()> { - (**self).try_next_token() - } - fn fatal(&self, m: &str) -> FatalError { - (**self).fatal(m) - } - fn err(&self, m: &str) { - (**self).err(m) - } - fn emit_fatal_errors(&mut self) { - (**self).emit_fatal_errors() - } - fn peek(&self) -> TokenAndSpan { - (**self).peek() + } } } diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index b80aa667be6..b670a738473 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -107,125 +107,41 @@ pub enum SemiColonMode { /// be. The important thing is to make sure that lookahead doesn't balk at /// `token::Interpolated` tokens. macro_rules! maybe_whole_expr { - ($p:expr) => ( - { - let found = match $p.token { - token::Interpolated(token::NtExpr(ref e)) => { - Some((*e).clone()) + ($p:expr) => { + if let token::Interpolated(nt) = $p.token.clone() { + match *nt { + token::NtExpr(ref e) => { + $p.bump(); + return Ok((*e).clone()); } - token::Interpolated(token::NtPath(_)) => { - // FIXME: The following avoids an issue with lexical borrowck scopes, - // but the clone is unfortunate. - let pt = match $p.token { - token::Interpolated(token::NtPath(ref pt)) => (**pt).clone(), - _ => unreachable!() - }; + token::NtPath(ref path) => { + $p.bump(); let span = $p.span; - Some($p.mk_expr(span.lo, span.hi, ExprKind::Path(None, pt), ThinVec::new())) + let kind = ExprKind::Path(None, (*path).clone()); + return Ok($p.mk_expr(span.lo, span.hi, kind, ThinVec::new())); } - token::Interpolated(token::NtBlock(_)) => { - // FIXME: The following avoids an issue with lexical borrowck scopes, - // but the clone is unfortunate. - let b = match $p.token { - token::Interpolated(token::NtBlock(ref b)) => (*b).clone(), - _ => unreachable!() - }; + token::NtBlock(ref block) => { + $p.bump(); let span = $p.span; - Some($p.mk_expr(span.lo, span.hi, ExprKind::Block(b), ThinVec::new())) + let kind = ExprKind::Block((*block).clone()); + return Ok($p.mk_expr(span.lo, span.hi, kind, ThinVec::new())); } - _ => None + _ => {}, }; - match found { - Some(e) => { - $p.bump(); - return Ok(e); - } - None => () - } } - ) + } } /// As maybe_whole_expr, but for things other than expressions macro_rules! maybe_whole { - ($p:expr, $constructor:ident) => ( - { - let found = match ($p).token { - token::Interpolated(token::$constructor(_)) => { - Some(($p).bump_and_get()) - } - _ => None - }; - if let Some(token::Interpolated(token::$constructor(x))) = found { - return Ok(x.clone()); + ($p:expr, $constructor:ident, |$x:ident| $e:expr) => { + if let token::Interpolated(nt) = $p.token.clone() { + if let token::$constructor($x) = (*nt).clone() { + $p.bump(); + return Ok($e); } } - ); - (no_clone $p:expr, $constructor:ident) => ( - { - let found = match ($p).token { - token::Interpolated(token::$constructor(_)) => { - Some(($p).bump_and_get()) - } - _ => None - }; - if let Some(token::Interpolated(token::$constructor(x))) = found { - return Ok(x); - } - } - ); - (no_clone_from_p $p:expr, $constructor:ident) => ( - { - let found = match ($p).token { - token::Interpolated(token::$constructor(_)) => { - Some(($p).bump_and_get()) - } - _ => None - }; - if let Some(token::Interpolated(token::$constructor(x))) = found { - return Ok(x.unwrap()); - } - } - ); - (deref $p:expr, $constructor:ident) => ( - { - let found = match ($p).token { - token::Interpolated(token::$constructor(_)) => { - Some(($p).bump_and_get()) - } - _ => None - }; - if let Some(token::Interpolated(token::$constructor(x))) = found { - return Ok((*x).clone()); - } - } - ); - (Some deref $p:expr, $constructor:ident) => ( - { - let found = match ($p).token { - token::Interpolated(token::$constructor(_)) => { - Some(($p).bump_and_get()) - } - _ => None - }; - if let Some(token::Interpolated(token::$constructor(x))) = found { - return Ok(Some((*x).clone())); - } - } - ); - (pair_empty $p:expr, $constructor:ident) => ( - { - let found = match ($p).token { - token::Interpolated(token::$constructor(_)) => { - Some(($p).bump_and_get()) - } - _ => None - }; - if let Some(token::Interpolated(token::$constructor(x))) = found { - return Ok((Vec::new(), x)); - } - } - ) + }; } fn maybe_append(mut lhs: Vec<Attribute>, rhs: Option<Vec<Attribute>>) @@ -294,6 +210,9 @@ pub struct Parser<'a> { /// into modules, and sub-parsers have new values for this name. pub root_module_name: Option<String>, pub expected_tokens: Vec<TokenType>, + pub tts: Vec<(TokenTree, usize)>, + pub desugar_doc_comments: bool, + pub allow_interpolated_tts: bool, } #[derive(PartialEq, Eq, Clone)] @@ -357,21 +276,18 @@ impl From<P<Expr>> for LhsExpr { } impl<'a> Parser<'a> { - pub fn new(sess: &'a ParseSess, mut rdr: Box<Reader+'a>) -> Self { - let tok0 = rdr.real_token(); - let span = tok0.sp; - let mut directory = match span { - syntax_pos::DUMMY_SP => PathBuf::new(), - _ => PathBuf::from(sess.codemap().span_to_filename(span)), - }; - directory.pop(); + pub fn new(sess: &'a ParseSess, rdr: Box<Reader+'a>) -> Self { + Parser::new_with_doc_flag(sess, rdr, false) + } - Parser { + pub fn new_with_doc_flag(sess: &'a ParseSess, rdr: Box<Reader+'a>, desugar_doc_comments: bool) + -> Self { + let mut parser = Parser { reader: rdr, sess: sess, - token: tok0.tok, - span: span, - prev_span: span, + token: token::Underscore, + span: syntax_pos::DUMMY_SP, + prev_span: syntax_pos::DUMMY_SP, prev_token_kind: PrevTokenKind::Other, lookahead_buffer: Default::default(), tokens_consumed: 0, @@ -379,11 +295,63 @@ impl<'a> Parser<'a> { quote_depth: 0, parsing_token_tree: false, obsolete_set: HashSet::new(), - directory: directory, + directory: PathBuf::new(), open_braces: Vec::new(), owns_directory: true, root_module_name: None, expected_tokens: Vec::new(), + tts: Vec::new(), + desugar_doc_comments: desugar_doc_comments, + allow_interpolated_tts: true, + }; + + let tok = parser.next_tok(); + parser.token = tok.tok; + parser.span = tok.sp; + if parser.span != syntax_pos::DUMMY_SP { + parser.directory = PathBuf::from(sess.codemap().span_to_filename(parser.span)); + parser.directory.pop(); + } + parser + } + + fn next_tok(&mut self) -> TokenAndSpan { + 'outer: loop { + let mut tok = if let Some((tts, i)) = self.tts.pop() { + let tt = tts.get_tt(i); + if i + 1 < tts.len() { + self.tts.push((tts, i + 1)); + } + if let TokenTree::Token(sp, tok) = tt { + TokenAndSpan { tok: tok, sp: sp } + } else { + self.tts.push((tt, 0)); + continue + } + } else { + self.reader.real_token() + }; + + loop { + let nt = match tok.tok { + token::Interpolated(ref nt) => nt.clone(), + token::DocComment(name) if self.desugar_doc_comments => { + self.tts.push((TokenTree::Token(tok.sp, token::DocComment(name)), 0)); + continue 'outer + } + _ => return tok, + }; + match *nt { + token::NtTT(TokenTree::Token(sp, ref t)) => { + tok = TokenAndSpan { tok: t.clone(), sp: sp }; + } + token::NtTT(ref tt) => { + self.tts.push((tt.clone(), 0)); + continue 'outer + } + _ => return tok, + } + } } } @@ -516,9 +484,6 @@ impl<'a> Parser<'a> { self.bump(); Ok(i) } - token::Interpolated(token::NtIdent(..)) => { - self.bug("ident interpolation not converted to real token"); - } _ => { Err(if self.prev_token_kind == PrevTokenKind::DocComment { self.span_fatal_help(self.prev_span, @@ -935,7 +900,7 @@ impl<'a> Parser<'a> { }; let next = if self.lookahead_buffer.start == self.lookahead_buffer.end { - self.reader.real_token() + self.next_tok() } else { // Avoid token copies with `replace`. let old_start = self.lookahead_buffer.start; @@ -980,7 +945,7 @@ impl<'a> Parser<'a> { f(&self.token) } else if dist < LOOKAHEAD_BUFFER_CAPACITY { while self.lookahead_buffer.len() < dist { - self.lookahead_buffer.buffer[self.lookahead_buffer.end] = self.reader.real_token(); + self.lookahead_buffer.buffer[self.lookahead_buffer.end] = self.next_tok(); self.lookahead_buffer.end = (self.lookahead_buffer.end + 1) % LOOKAHEAD_BUFFER_CAPACITY; } @@ -1162,7 +1127,7 @@ impl<'a> Parser<'a> { /// Parse the items in a trait declaration pub fn parse_trait_item(&mut self) -> PResult<'a, TraitItem> { - maybe_whole!(no_clone_from_p self, NtTraitItem); + maybe_whole!(self, NtTraitItem, |x| x); let mut attrs = self.parse_outer_attributes()?; let lo = self.span.lo; @@ -1331,7 +1296,7 @@ impl<'a> Parser<'a> { /// Parse a type. pub fn parse_ty(&mut self) -> PResult<'a, P<Ty>> { - maybe_whole!(no_clone self, NtTy); + maybe_whole!(self, NtTy, |x| x); let lo = self.span.lo; @@ -1476,7 +1441,7 @@ impl<'a> Parser<'a> { /// This version of parse arg doesn't necessarily require /// identifier names. pub fn parse_arg_general(&mut self, require_name: bool) -> PResult<'a, Arg> { - maybe_whole!(no_clone self, NtArg); + maybe_whole!(self, NtArg, |x| x); let pat = if require_name || self.is_named_argument() { debug!("parse_arg_general parse_pat (require_name:{})", @@ -1542,12 +1507,13 @@ impl<'a> Parser<'a> { /// Matches token_lit = LIT_INTEGER | ... pub fn parse_lit_token(&mut self) -> PResult<'a, LitKind> { let out = match self.token { - token::Interpolated(token::NtExpr(ref v)) => { - match v.node { + token::Interpolated(ref nt) => match **nt { + token::NtExpr(ref v) => match v.node { ExprKind::Lit(ref lit) => { lit.node.clone() } _ => { return self.unexpected_last(&self.token); } - } - } + }, + _ => { return self.unexpected_last(&self.token); } + }, token::Literal(lit, suf) => { let (suffix_illegal, out) = match lit { token::Byte(i) => (true, LitKind::Byte(parse::byte_lit(&i.as_str()).0)), @@ -1703,14 +1669,7 @@ impl<'a> Parser<'a> { /// bounds are permitted and whether `::` must precede type parameter /// groups. pub fn parse_path(&mut self, mode: PathStyle) -> PResult<'a, ast::Path> { - // Check for a whole path... - let found = match self.token { - token::Interpolated(token::NtPath(_)) => Some(self.bump_and_get()), - _ => None, - }; - if let Some(token::Interpolated(token::NtPath(path))) = found { - return Ok(*path); - } + maybe_whole!(self, NtPath, |x| x); let lo = self.span.lo; let is_global = self.eat(&token::ModSep); @@ -2746,8 +2705,6 @@ impl<'a> Parser<'a> { // and token::SubstNt's; it's too early to know yet // whether something will be a nonterminal or a seq // yet. - maybe_whole!(deref self, NtTT); - match self.token { token::Eof => { let mut err: DiagnosticBuilder<'a> = @@ -2760,6 +2717,17 @@ impl<'a> Parser<'a> { Err(err) }, token::OpenDelim(delim) => { + if self.tts.last().map(|&(_, i)| i == 1).unwrap_or(false) { + let tt = self.tts.pop().unwrap().0; + self.bump(); + return Ok(if self.allow_interpolated_tts { + // avoid needlessly reparsing token trees in recursive macro expansions + TokenTree::Token(tt.span(), token::Interpolated(Rc::new(token::NtTT(tt)))) + } else { + tt + }); + } + let parsing_token_tree = ::std::mem::replace(&mut self.parsing_token_tree, true); // The span for beginning of the delimited section let pre_span = self.span; @@ -2833,29 +2801,20 @@ impl<'a> Parser<'a> { close_span: close_span, }))) }, + token::CloseDelim(_) => { + // An unexpected closing delimiter (i.e., there is no + // matching opening delimiter). + let token_str = self.this_token_to_string(); + let err = self.diagnostic().struct_span_err(self.span, + &format!("unexpected close delimiter: `{}`", token_str)); + Err(err) + }, + /* we ought to allow different depths of unquotation */ + token::Dollar | token::SubstNt(..) if self.quote_depth > 0 => { + self.parse_unquoted() + } _ => { - // invariants: the current token is not a left-delimiter, - // not an EOF, and not the desired right-delimiter (if - // it were, parse_seq_to_before_end would have prevented - // reaching this point). - maybe_whole!(deref self, NtTT); - match self.token { - token::CloseDelim(_) => { - // An unexpected closing delimiter (i.e., there is no - // matching opening delimiter). - let token_str = self.this_token_to_string(); - let err = self.diagnostic().struct_span_err(self.span, - &format!("unexpected close delimiter: `{}`", token_str)); - Err(err) - }, - /* we ought to allow different depths of unquotation */ - token::Dollar | token::SubstNt(..) if self.quote_depth > 0 => { - self.parse_unquoted() - } - _ => { - Ok(TokenTree::Token(self.span, self.bump_and_get())) - } - } + Ok(TokenTree::Token(self.span, self.bump_and_get())) } } } @@ -3336,7 +3295,7 @@ impl<'a> Parser<'a> { } pub fn parse_arm(&mut self) -> PResult<'a, Arm> { - maybe_whole!(no_clone self, NtArm); + maybe_whole!(self, NtArm, |x| x); let attrs = self.parse_outer_attributes()?; let pats = self.parse_pats()?; @@ -3592,7 +3551,7 @@ impl<'a> Parser<'a> { /// Parse a pattern. pub fn parse_pat(&mut self) -> PResult<'a, P<Pat>> { - maybe_whole!(self, NtPat); + maybe_whole!(self, NtPat, |x| x); let lo = self.span.lo; let pat; @@ -3897,7 +3856,7 @@ impl<'a> Parser<'a> { fn parse_stmt_without_recovery(&mut self, macro_legacy_warnings: bool) -> PResult<'a, Option<Stmt>> { - maybe_whole!(Some deref self, NtStmt); + maybe_whole!(self, NtStmt, |x| Some(x)); let attrs = self.parse_outer_attributes()?; let lo = self.span.lo; @@ -4086,7 +4045,7 @@ impl<'a> Parser<'a> { /// Parse a block. No inner attrs are allowed. pub fn parse_block(&mut self) -> PResult<'a, P<Block>> { - maybe_whole!(no_clone self, NtBlock); + maybe_whole!(self, NtBlock, |x| x); let lo = self.span.lo; @@ -4124,7 +4083,7 @@ impl<'a> Parser<'a> { /// Parse a block. Inner attrs are allowed. fn parse_inner_attrs_and_block(&mut self) -> PResult<'a, (Vec<Attribute>, P<Block>)> { - maybe_whole!(pair_empty self, NtBlock); + maybe_whole!(self, NtBlock, |x| (Vec::new(), x)); let lo = self.span.lo; self.expect(&token::OpenDelim(token::Brace))?; @@ -4299,7 +4258,7 @@ impl<'a> Parser<'a> { /// | ( < lifetimes , typaramseq ( , )? > ) /// where typaramseq = ( typaram ) | ( typaram , typaramseq ) pub fn parse_generics(&mut self) -> PResult<'a, ast::Generics> { - maybe_whole!(self, NtGenerics); + maybe_whole!(self, NtGenerics, |x| x); let span_lo = self.span.lo; if self.eat(&token::Lt) { @@ -4440,7 +4399,7 @@ impl<'a> Parser<'a> { /// where T : Trait<U, V> + 'b, 'a : 'b /// ``` pub fn parse_where_clause(&mut self) -> PResult<'a, ast::WhereClause> { - maybe_whole!(self, NtWhereClause); + maybe_whole!(self, NtWhereClause, |x| x); let mut where_clause = WhereClause { id: ast::DUMMY_NODE_ID, @@ -4848,7 +4807,7 @@ impl<'a> Parser<'a> { /// Parse an impl item. pub fn parse_impl_item(&mut self) -> PResult<'a, ImplItem> { - maybe_whole!(no_clone_from_p self, NtImplItem); + maybe_whole!(self, NtImplItem, |x| x); let mut attrs = self.parse_outer_attributes()?; let lo = self.span.lo; @@ -5716,19 +5675,13 @@ impl<'a> Parser<'a> { /// extern crate. fn parse_item_(&mut self, attrs: Vec<Attribute>, macros_allowed: bool, attributes_allowed: bool) -> PResult<'a, Option<P<Item>>> { - let nt_item = match self.token { - token::Interpolated(token::NtItem(ref item)) => { - Some((**item).clone()) - } - _ => None - }; - if let Some(mut item) = nt_item { - self.bump(); + maybe_whole!(self, NtItem, |item| { + let mut item = item.unwrap(); let mut attrs = attrs; mem::swap(&mut item.attrs, &mut attrs); item.attrs.extend(attrs); - return Ok(Some(P(item))); - } + Some(P(item)) + }); let lo = self.span.lo; diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index 4d0da660302..0198ee073d2 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -123,7 +123,7 @@ pub enum Token { Lifetime(ast::Ident), /* For interpolation */ - Interpolated(Nonterminal), + Interpolated(Rc<Nonterminal>), // Can be expanded into several tokens. /// Doc comment DocComment(ast::Name), @@ -172,12 +172,15 @@ impl Token { DotDot | DotDotDot => true, // range notation Lt | BinOp(Shl) => true, // associated path ModSep => true, - Interpolated(NtExpr(..)) => true, - Interpolated(NtIdent(..)) => true, - Interpolated(NtBlock(..)) => true, - Interpolated(NtPath(..)) => true, Pound => true, // for expression attributes - _ => false, + Interpolated(ref nt) => match **nt { + NtExpr(..) => true, + NtIdent(..) => true, + NtBlock(..) => true, + NtPath(..) => true, + _ => false, + }, + _ => false, } } @@ -215,10 +218,12 @@ impl Token { /// Returns `true` if the token is an interpolated path. pub fn is_path(&self) -> bool { - match *self { - Interpolated(NtPath(..)) => true, - _ => false, + if let Interpolated(ref nt) = *self { + if let NtPath(..) = **nt { + return true; + } } + false } /// Returns `true` if the token is a lifetime. @@ -290,19 +295,19 @@ impl Token { pub enum Nonterminal { NtItem(P<ast::Item>), NtBlock(P<ast::Block>), - NtStmt(P<ast::Stmt>), + NtStmt(ast::Stmt), NtPat(P<ast::Pat>), NtExpr(P<ast::Expr>), NtTy(P<ast::Ty>), - NtIdent(Box<ast::SpannedIdent>), + NtIdent(ast::SpannedIdent), /// Stuff inside brackets for attributes NtMeta(P<ast::MetaItem>), - NtPath(Box<ast::Path>), - NtTT(P<tokenstream::TokenTree>), // needs P'ed to break a circularity + NtPath(ast::Path), + NtTT(tokenstream::TokenTree), // These are not exposed to macros, but are used by quasiquote. NtArm(ast::Arm), - NtImplItem(P<ast::ImplItem>), - NtTraitItem(P<ast::TraitItem>), + NtImplItem(ast::ImplItem), + NtTraitItem(ast::TraitItem), NtGenerics(ast::Generics), NtWhereClause(ast::WhereClause), NtArg(ast::Arg), |