// Copyright 2017 The Rust Project Developers. See the COPYRIGHT // file at the top-level directory of this distribution and at // http://rust-lang.org/COPYRIGHT. // // Licensed under the Apache License, Version 2.0 or the MIT license // , at your // option. This file may not be copied, modified, or distributed // except according to those terms. use ast; use ext::tt::macro_parser; use parse::{ParseSess, token}; use print::pprust; use symbol::keywords; use syntax_pos::{DUMMY_SP, Span, BytePos}; use tokenstream; use std::rc::Rc; #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub struct Delimited { pub delim: token::DelimToken, pub tts: Vec, } impl Delimited { pub fn open_token(&self) -> token::Token { token::OpenDelim(self.delim) } pub fn close_token(&self) -> token::Token { token::CloseDelim(self.delim) } pub fn open_tt(&self, span: Span) -> TokenTree { let open_span = if span == DUMMY_SP { DUMMY_SP } else { span.with_lo(span.lo() + BytePos(self.delim.len() as u32)) }; TokenTree::Token(open_span, self.open_token()) } pub fn close_tt(&self, span: Span) -> TokenTree { let close_span = if span == DUMMY_SP { DUMMY_SP } else { span.with_lo(span.hi() - BytePos(self.delim.len() as u32)) }; TokenTree::Token(close_span, self.close_token()) } } #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub struct SequenceRepetition { /// The sequence of token trees pub tts: Vec, /// The optional separator pub separator: Option, /// Whether the sequence can be repeated zero (*), or one or more times (+) pub op: KleeneOp, /// The number of `Match`s that appear in the sequence (and subsequences) pub num_captures: usize, } /// A Kleene-style [repetition operator](http://en.wikipedia.org/wiki/Kleene_star) /// for token sequences. #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] pub enum KleeneOp { ZeroOrMore, OneOrMore, } /// Similar to `tokenstream::TokenTree`, except that `$i`, `$i:ident`, and `$(...)` /// are "first-class" token trees. #[derive(Debug, Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash)] pub enum TokenTree { Token(Span, token::Token), Delimited(Span, Rc), /// A kleene-style repetition sequence Sequence(Span, Rc), /// E.g. `$var` MetaVar(Span, ast::Ident), /// E.g. `$var:expr`. This is only used in the left hand side of MBE macros. MetaVarDecl(Span, ast::Ident /* name to bind */, ast::Ident /* kind of nonterminal */), } impl TokenTree { pub fn len(&self) -> usize { match *self { TokenTree::Delimited(_, ref delimed) => match delimed.delim { token::NoDelim => delimed.tts.len(), _ => delimed.tts.len() + 2, }, TokenTree::Sequence(_, ref seq) => seq.tts.len(), _ => 0, } } pub fn is_empty(&self) -> bool { match *self { TokenTree::Delimited(_, ref delimed) => match delimed.delim { token::NoDelim => delimed.tts.is_empty(), _ => false, }, TokenTree::Sequence(_, ref seq) => seq.tts.is_empty(), _ => true, } } pub fn get_tt(&self, index: usize) -> TokenTree { match (self, index) { (&TokenTree::Delimited(_, ref delimed), _) if delimed.delim == token::NoDelim => { delimed.tts[index].clone() } (&TokenTree::Delimited(span, ref delimed), _) => { if index == 0 { return delimed.open_tt(span); } if index == delimed.tts.len() + 1 { return delimed.close_tt(span); } delimed.tts[index - 1].clone() } (&TokenTree::Sequence(_, ref seq), _) => seq.tts[index].clone(), _ => panic!("Cannot expand a token tree"), } } /// Retrieve the `TokenTree`'s span. pub fn span(&self) -> Span { match *self { TokenTree::Token(sp, _) | TokenTree::MetaVar(sp, _) | TokenTree::MetaVarDecl(sp, _, _) | TokenTree::Delimited(sp, _) | TokenTree::Sequence(sp, _) => sp, } } } pub fn parse(input: tokenstream::TokenStream, expect_matchers: bool, sess: &ParseSess) -> Vec { let mut result = Vec::new(); let mut trees = input.trees(); while let Some(tree) = trees.next() { let tree = parse_tree(tree, &mut trees, expect_matchers, sess); match tree { TokenTree::MetaVar(start_sp, ident) if expect_matchers => { let span = match trees.next() { Some(tokenstream::TokenTree::Token(span, token::Colon)) => match trees.next() { Some(tokenstream::TokenTree::Token(end_sp, ref tok)) => match tok.ident() { Some(kind) => { let span = end_sp.with_lo(start_sp.lo()); result.push(TokenTree::MetaVarDecl(span, ident, kind)); continue } _ => end_sp, }, tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span), }, tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(start_sp), }; sess.missing_fragment_specifiers.borrow_mut().insert(span); result.push(TokenTree::MetaVarDecl(span, ident, keywords::Invalid.ident())); } _ => result.push(tree), } } result } fn parse_tree(tree: tokenstream::TokenTree, trees: &mut I, expect_matchers: bool, sess: &ParseSess) -> TokenTree where I: Iterator, { match tree { tokenstream::TokenTree::Token(span, token::Dollar) => match trees.next() { Some(tokenstream::TokenTree::Delimited(span, delimited)) => { if delimited.delim != token::Paren { let tok = pprust::token_to_string(&token::OpenDelim(delimited.delim)); let msg = format!("expected `(`, found `{}`", tok); sess.span_diagnostic.span_err(span, &msg); } let sequence = parse(delimited.tts.into(), expect_matchers, sess); let (separator, op) = parse_sep_and_kleene_op(trees, span, sess); let name_captures = macro_parser::count_names(&sequence); TokenTree::Sequence(span, Rc::new(SequenceRepetition { tts: sequence, separator, op, num_captures: name_captures, })) } Some(tokenstream::TokenTree::Token(ident_span, ref token)) if token.is_ident() => { let ident = token.ident().unwrap(); let span = ident_span.with_lo(span.lo()); if ident.name == keywords::Crate.name() { let ident = ast::Ident { name: keywords::DollarCrate.name(), ..ident }; TokenTree::Token(span, token::Ident(ident)) } else { TokenTree::MetaVar(span, ident) } } Some(tokenstream::TokenTree::Token(span, tok)) => { let msg = format!("expected identifier, found `{}`", pprust::token_to_string(&tok)); sess.span_diagnostic.span_err(span, &msg); TokenTree::MetaVar(span, keywords::Invalid.ident()) } None => TokenTree::Token(span, token::Dollar), }, tokenstream::TokenTree::Token(span, tok) => TokenTree::Token(span, tok), tokenstream::TokenTree::Delimited(span, delimited) => { TokenTree::Delimited(span, Rc::new(Delimited { delim: delimited.delim, tts: parse(delimited.tts.into(), expect_matchers, sess), })) } } } fn parse_sep_and_kleene_op(input: &mut I, span: Span, sess: &ParseSess) -> (Option, KleeneOp) where I: Iterator, { fn kleene_op(token: &token::Token) -> Option { match *token { token::BinOp(token::Star) => Some(KleeneOp::ZeroOrMore), token::BinOp(token::Plus) => Some(KleeneOp::OneOrMore), _ => None, } } let span = match input.next() { Some(tokenstream::TokenTree::Token(span, tok)) => match kleene_op(&tok) { Some(op) => return (None, op), None => match input.next() { Some(tokenstream::TokenTree::Token(span, tok2)) => match kleene_op(&tok2) { Some(op) => return (Some(tok), op), None => span, }, tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span), } }, tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span), }; sess.span_diagnostic.span_err(span, "expected `*` or `+`"); (None, KleeneOp::ZeroOrMore) }