From ce616a7d6ad838aacd080b47566c15e82ad8dd6d Mon Sep 17 00:00:00 2001 From: Jeffrey Seyfried Date: Tue, 14 Mar 2017 22:04:46 +0000 Subject: Improve the `TokenStream` quoter. --- src/libproc_macro_plugin/lib.rs | 78 ++++++------- src/libproc_macro_plugin/qquote.rs | 216 ------------------------------------ src/libproc_macro_plugin/quote.rs | 220 +++++++++++++++++++++++++++++++++++++ 3 files changed, 260 insertions(+), 254 deletions(-) delete mode 100644 src/libproc_macro_plugin/qquote.rs create mode 100644 src/libproc_macro_plugin/quote.rs (limited to 'src/libproc_macro_plugin') diff --git a/src/libproc_macro_plugin/lib.rs b/src/libproc_macro_plugin/lib.rs index e9042909576..a6dad641253 100644 --- a/src/libproc_macro_plugin/lib.rs +++ b/src/libproc_macro_plugin/lib.rs @@ -13,62 +13,64 @@ //! A library for procedural macro writers. //! //! ## Usage -//! This crate provides the `qquote!` macro for syntax creation. +//! This crate provides the `quote!` macro for syntax creation. //! -//! The `qquote!` macro uses the crate `syntax`, so users must declare `extern crate syntax;` +//! The `quote!` macro uses the crate `syntax`, so users must declare `extern crate syntax;` //! at the crate root. This is a temporary solution until we have better hygiene. //! //! ## Quasiquotation //! //! The quasiquoter creates output that, when run, constructs the tokenstream specified as -//! input. For example, `qquote!(5 + 5)` will produce a program, that, when run, will +//! input. For example, `quote!(5 + 5)` will produce a program, that, when run, will //! construct the TokenStream `5 | + | 5`. //! //! ### Unquoting //! -//! Unquoting is currently done as `unquote`, and works by taking the single next -//! TokenTree in the TokenStream as the unquoted term. Ergonomically, `unquote(foo)` works -//! fine, but `unquote foo` is also supported. +//! Unquoting is done with `$`, and works by taking the single next ident as the unquoted term. +//! To quote `$` itself, use `$$`. //! -//! A simple example might be: +//! A simple example is: //! //!``` //!fn double(tmp: TokenStream) -> TokenStream { -//! qquote!(unquote(tmp) * 2) +//! quote!($tmp * 2) //!} //!``` //! -//! ### Large Example: Implementing Scheme's `cond` +//! ### Large example: Scheme's `cond` //! -//! Below is the full implementation of Scheme's `cond` operator. +//! Below is an example implementation of Scheme's `cond`. //! //! ``` -//! fn cond_rec(input: TokenStream) -> TokenStream { -//! if input.is_empty() { return quote!(); } -//! -//! let next = input.slice(0..1); -//! let rest = input.slice_from(1..); -//! -//! let clause : TokenStream = match next.maybe_delimited() { -//! Some(ts) => ts, -//! _ => panic!("Invalid input"), -//! }; -//! -//! // clause is ([test]) [rhs] -//! if clause.len() < 2 { panic!("Invalid macro usage in cond: {:?}", clause) } -//! -//! let test: TokenStream = clause.slice(0..1); -//! let rhs: TokenStream = clause.slice_from(1..); -//! -//! if ident_eq(&test[0], str_to_ident("else")) || rest.is_empty() { -//! quote!({unquote(rhs)}) -//! } else { -//! quote!({if unquote(test) { unquote(rhs) } else { cond!(unquote(rest)) } }) -//! } +//! fn cond(input: TokenStream) -> TokenStream { +//! let mut conds = Vec::new(); +//! let mut input = input.trees().peekable(); +//! while let Some(tree) = input.next() { +//! let mut cond = match tree { +//! TokenTree::Delimited(_, ref delimited) => delimited.stream(), +//! _ => panic!("Invalid input"), +//! }; +//! let mut trees = cond.trees(); +//! let test = trees.next(); +//! let rhs = trees.collect::(); +//! if rhs.is_empty() { +//! panic!("Invalid macro usage in cond: {}", cond); +//! } +//! let is_else = match test { +//! Some(TokenTree::Token(_, Token::Ident(ident))) if ident.name == "else" => true, +//! _ => false, +//! }; +//! conds.push(if is_else || input.peek().is_none() { +//! quote!({ $rhs }) +//! } else { +//! let test = test.unwrap(); +//! quote!(if $test { $rhs } else) +//! }); +//! } +//! +//! conds.into_iter().collect() //! } //! ``` -//! - #![crate_name = "proc_macro_plugin"] #![unstable(feature = "rustc_private", issue = "27812")] #![feature(plugin_registrar)] @@ -87,8 +89,8 @@ extern crate rustc_plugin; extern crate syntax; extern crate syntax_pos; -mod qquote; -use qquote::qquote; +mod quote; +use quote::quote; use rustc_plugin::Registry; use syntax::ext::base::SyntaxExtension; @@ -99,6 +101,6 @@ use syntax::symbol::Symbol; #[plugin_registrar] pub fn plugin_registrar(reg: &mut Registry) { - reg.register_syntax_extension(Symbol::intern("qquote"), - SyntaxExtension::ProcMacro(Box::new(qquote))); + reg.register_syntax_extension(Symbol::intern("quote"), + SyntaxExtension::ProcMacro(Box::new(quote))); } diff --git a/src/libproc_macro_plugin/qquote.rs b/src/libproc_macro_plugin/qquote.rs deleted file mode 100644 index 0276587ed52..00000000000 --- a/src/libproc_macro_plugin/qquote.rs +++ /dev/null @@ -1,216 +0,0 @@ -// Copyright 2016 The Rust Project Developers. See the COPYRIGHT -// file at the top-level directory of this distribution and at -// http://rust-lang.org/COPYRIGHT. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -//! # Quasiquoter -//! This file contains the implementation internals of the quasiquoter provided by `qquote!`. - -use syntax::ast::Ident; -use syntax::parse::token::{self, Token, Lit}; -use syntax::symbol::Symbol; -use syntax::tokenstream::{self, Delimited, TokenTree, TokenStream}; -use syntax_pos::DUMMY_SP; - -use std::iter; - -pub fn qquote<'cx>(stream: TokenStream) -> TokenStream { - stream.quote() -} - -trait Quote { - fn quote(&self) -> TokenStream; -} - -macro_rules! quote_tok { - (,) => { Token::Comma }; - (.) => { Token::Dot }; - (:) => { Token::Colon }; - (::) => { Token::ModSep }; - (!) => { Token::Not }; - (<) => { Token::Lt }; - (>) => { Token::Gt }; - (_) => { Token::Underscore }; - ($i:ident) => { Token::Ident(Ident::from_str(stringify!($i))) }; -} - -macro_rules! quote_tree { - ((unquote $($t:tt)*)) => { $($t)* }; - ((quote $($t:tt)*)) => { ($($t)*).quote() }; - (($($t:tt)*)) => { delimit(token::Paren, quote!($($t)*)) }; - ([$($t:tt)*]) => { delimit(token::Bracket, quote!($($t)*)) }; - ({$($t:tt)*}) => { delimit(token::Brace, quote!($($t)*)) }; - ($t:tt) => { TokenStream::from(TokenTree::Token(DUMMY_SP, quote_tok!($t))) }; -} - -fn delimit(delim: token::DelimToken, stream: TokenStream) -> TokenStream { - TokenTree::Delimited(DUMMY_SP, Delimited { delim: delim, tts: stream.into() }).into() -} - -macro_rules! quote { - () => { TokenStream::empty() }; - ($($t:tt)*) => { [ $( quote_tree!($t), )* ].iter().cloned().collect::() }; -} - -impl Quote for Option { - fn quote(&self) -> TokenStream { - match *self { - Some(ref t) => quote!(::std::option::Option::Some((quote t))), - None => quote!(::std::option::Option::None), - } - } -} - -impl Quote for TokenStream { - fn quote(&self) -> TokenStream { - if self.is_empty() { - return quote!(::syntax::tokenstream::TokenStream::empty()); - } - - struct Quote(iter::Peekable); - - impl Iterator for Quote { - type Item = TokenStream; - - fn next(&mut self) -> Option { - let is_unquote = match self.0.peek() { - Some(&TokenTree::Token(_, Token::Ident(ident))) if ident.name == "unquote" => { - self.0.next(); - true - } - _ => false, - }; - - self.0.next().map(|tree| { - let quoted_tree = if is_unquote { tree.into() } else { tree.quote() }; - quote!(::syntax::tokenstream::TokenStream::from((unquote quoted_tree)),) - }) - } - } - - let quoted = Quote(self.trees().peekable()).collect::(); - quote!([(unquote quoted)].iter().cloned().collect::<::syntax::tokenstream::TokenStream>()) - } -} - -impl Quote for TokenTree { - fn quote(&self) -> TokenStream { - match *self { - TokenTree::Token(_, ref token) => quote! { - ::syntax::tokenstream::TokenTree::Token(::syntax::ext::quote::rt::DUMMY_SP, - (quote token)) - }, - TokenTree::Delimited(_, ref delimited) => quote! { - ::syntax::tokenstream::TokenTree::Delimited(::syntax::ext::quote::rt::DUMMY_SP, - (quote delimited)) - }, - } - } -} - -impl Quote for Delimited { - fn quote(&self) -> TokenStream { - quote!(::syntax::tokenstream::Delimited { - delim: (quote self.delim), - tts: (quote self.stream()).into(), - }) - } -} - -impl<'a> Quote for &'a str { - fn quote(&self) -> TokenStream { - TokenTree::Token(DUMMY_SP, Token::Literal(token::Lit::Str_(Symbol::intern(self)), None)) - .into() - } -} - -impl Quote for Ident { - fn quote(&self) -> TokenStream { - // FIXME(jseyfried) quote hygiene - quote!(::syntax::ast::Ident::from_str((quote &*self.name.as_str()))) - } -} - -impl Quote for Symbol { - fn quote(&self) -> TokenStream { - quote!(::syntax::symbol::Symbol::intern((quote &*self.as_str()))) - } -} - -impl Quote for Token { - fn quote(&self) -> TokenStream { - macro_rules! gen_match { - ($($i:ident),*; $($t:tt)*) => { - match *self { - $( Token::$i => quote!(::syntax::parse::token::$i), )* - $( $t )* - } - } - } - - gen_match! { - Eq, Lt, Le, EqEq, Ne, Ge, Gt, AndAnd, OrOr, Not, Tilde, At, Dot, DotDot, DotDotDot, - Comma, Semi, Colon, ModSep, RArrow, LArrow, FatArrow, Pound, Dollar, Question, - Underscore; - - Token::OpenDelim(delim) => quote!(::syntax::parse::token::OpenDelim((quote delim))), - Token::CloseDelim(delim) => quote!(::syntax::parse::token::CloseDelim((quote delim))), - Token::BinOp(tok) => quote!(::syntax::parse::token::BinOp((quote tok))), - Token::BinOpEq(tok) => quote!(::syntax::parse::token::BinOpEq((quote tok))), - Token::Ident(ident) => quote!(::syntax::parse::token::Ident((quote ident))), - Token::Lifetime(ident) => quote!(::syntax::parse::token::Lifetime((quote ident))), - Token::Literal(lit, sfx) => quote! { - ::syntax::parse::token::Literal((quote lit), (quote sfx)) - }, - _ => panic!("Unhandled case!"), - } - } -} - -impl Quote for token::BinOpToken { - fn quote(&self) -> TokenStream { - macro_rules! gen_match { - ($($i:ident),*) => { - match *self { - $( token::BinOpToken::$i => quote!(::syntax::parse::token::BinOpToken::$i), )* - } - } - } - - gen_match!(Plus, Minus, Star, Slash, Percent, Caret, And, Or, Shl, Shr) - } -} - -impl Quote for Lit { - fn quote(&self) -> TokenStream { - macro_rules! gen_match { - ($($i:ident),*) => { - match *self { - $( Lit::$i(lit) => quote!(::syntax::parse::token::Lit::$i((quote lit))), )* - _ => panic!("Unsupported literal"), - } - } - } - - gen_match!(Byte, Char, Float, Str_, Integer, ByteStr) - } -} - -impl Quote for token::DelimToken { - fn quote(&self) -> TokenStream { - macro_rules! gen_match { - ($($i:ident),*) => { - match *self { - $(token::DelimToken::$i => { quote!(::syntax::parse::token::DelimToken::$i) })* - } - } - } - - gen_match!(Paren, Bracket, Brace, NoDelim) - } -} diff --git a/src/libproc_macro_plugin/quote.rs b/src/libproc_macro_plugin/quote.rs new file mode 100644 index 00000000000..ad71584b61a --- /dev/null +++ b/src/libproc_macro_plugin/quote.rs @@ -0,0 +1,220 @@ +// Copyright 2016 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +//! # Quasiquoter +//! This file contains the implementation internals of the quasiquoter provided by `qquote!`. + +use syntax::ast::Ident; +use syntax::parse::token::{self, Token, Lit}; +use syntax::symbol::Symbol; +use syntax::tokenstream::{self, Delimited, TokenTree, TokenStream}; +use syntax_pos::DUMMY_SP; + +use std::iter; + +pub fn quote<'cx>(stream: TokenStream) -> TokenStream { + stream.quote() +} + +trait Quote { + fn quote(&self) -> TokenStream; +} + +macro_rules! quote_tok { + (,) => { Token::Comma }; + (.) => { Token::Dot }; + (:) => { Token::Colon }; + (::) => { Token::ModSep }; + (!) => { Token::Not }; + (<) => { Token::Lt }; + (>) => { Token::Gt }; + (_) => { Token::Underscore }; + ($i:ident) => { Token::Ident(Ident::from_str(stringify!($i))) }; +} + +macro_rules! quote_tree { + ((unquote $($t:tt)*)) => { $($t)* }; + ((quote $($t:tt)*)) => { ($($t)*).quote() }; + (($($t:tt)*)) => { delimit(token::Paren, quote!($($t)*)) }; + ([$($t:tt)*]) => { delimit(token::Bracket, quote!($($t)*)) }; + ({$($t:tt)*}) => { delimit(token::Brace, quote!($($t)*)) }; + ($t:tt) => { TokenStream::from(TokenTree::Token(DUMMY_SP, quote_tok!($t))) }; +} + +fn delimit(delim: token::DelimToken, stream: TokenStream) -> TokenStream { + TokenTree::Delimited(DUMMY_SP, Delimited { delim: delim, tts: stream.into() }).into() +} + +macro_rules! quote { + () => { TokenStream::empty() }; + ($($t:tt)*) => { [ $( quote_tree!($t), )* ].iter().cloned().collect::() }; +} + +impl Quote for Option { + fn quote(&self) -> TokenStream { + match *self { + Some(ref t) => quote!(::std::option::Option::Some((quote t))), + None => quote!(::std::option::Option::None), + } + } +} + +impl Quote for TokenStream { + fn quote(&self) -> TokenStream { + if self.is_empty() { + return quote!(::syntax::tokenstream::TokenStream::empty()); + } + + struct Quoter(iter::Peekable); + + impl Iterator for Quoter { + type Item = TokenStream; + + fn next(&mut self) -> Option { + let quoted_tree = if let Some(&TokenTree::Token(_, Token::Dollar)) = self.0.peek() { + self.0.next(); + match self.0.next() { + Some(tree @ TokenTree::Token(_, Token::Ident(..))) => Some(tree.into()), + Some(tree @ TokenTree::Token(_, Token::Dollar)) => Some(tree.quote()), + // FIXME(jseyfried): improve these diagnostics + Some(..) => panic!("`$` must be followed by an ident or `$` in `quote!`"), + None => panic!("unexpected trailing `$` in `quote!`"), + } + } else { + self.0.next().as_ref().map(Quote::quote) + }; + + quoted_tree.map(|quoted_tree| { + quote!(::syntax::tokenstream::TokenStream::from((unquote quoted_tree)),) + }) + } + } + + let quoted = Quoter(self.trees().peekable()).collect::(); + quote!([(unquote quoted)].iter().cloned().collect::<::syntax::tokenstream::TokenStream>()) + } +} + +impl Quote for TokenTree { + fn quote(&self) -> TokenStream { + match *self { + TokenTree::Token(_, ref token) => quote! { + ::syntax::tokenstream::TokenTree::Token(::syntax::ext::quote::rt::DUMMY_SP, + (quote token)) + }, + TokenTree::Delimited(_, ref delimited) => quote! { + ::syntax::tokenstream::TokenTree::Delimited(::syntax::ext::quote::rt::DUMMY_SP, + (quote delimited)) + }, + } + } +} + +impl Quote for Delimited { + fn quote(&self) -> TokenStream { + quote!(::syntax::tokenstream::Delimited { + delim: (quote self.delim), + tts: (quote self.stream()).into(), + }) + } +} + +impl<'a> Quote for &'a str { + fn quote(&self) -> TokenStream { + TokenTree::Token(DUMMY_SP, Token::Literal(token::Lit::Str_(Symbol::intern(self)), None)) + .into() + } +} + +impl Quote for Ident { + fn quote(&self) -> TokenStream { + // FIXME(jseyfried) quote hygiene + quote!(::syntax::ast::Ident::from_str((quote &*self.name.as_str()))) + } +} + +impl Quote for Symbol { + fn quote(&self) -> TokenStream { + quote!(::syntax::symbol::Symbol::intern((quote &*self.as_str()))) + } +} + +impl Quote for Token { + fn quote(&self) -> TokenStream { + macro_rules! gen_match { + ($($i:ident),*; $($t:tt)*) => { + match *self { + $( Token::$i => quote!(::syntax::parse::token::$i), )* + $( $t )* + } + } + } + + gen_match! { + Eq, Lt, Le, EqEq, Ne, Ge, Gt, AndAnd, OrOr, Not, Tilde, At, Dot, DotDot, DotDotDot, + Comma, Semi, Colon, ModSep, RArrow, LArrow, FatArrow, Pound, Dollar, Question, + Underscore; + + Token::OpenDelim(delim) => quote!(::syntax::parse::token::OpenDelim((quote delim))), + Token::CloseDelim(delim) => quote!(::syntax::parse::token::CloseDelim((quote delim))), + Token::BinOp(tok) => quote!(::syntax::parse::token::BinOp((quote tok))), + Token::BinOpEq(tok) => quote!(::syntax::parse::token::BinOpEq((quote tok))), + Token::Ident(ident) => quote!(::syntax::parse::token::Ident((quote ident))), + Token::Lifetime(ident) => quote!(::syntax::parse::token::Lifetime((quote ident))), + Token::Literal(lit, sfx) => quote! { + ::syntax::parse::token::Literal((quote lit), (quote sfx)) + }, + _ => panic!("Unhandled case!"), + } + } +} + +impl Quote for token::BinOpToken { + fn quote(&self) -> TokenStream { + macro_rules! gen_match { + ($($i:ident),*) => { + match *self { + $( token::BinOpToken::$i => quote!(::syntax::parse::token::BinOpToken::$i), )* + } + } + } + + gen_match!(Plus, Minus, Star, Slash, Percent, Caret, And, Or, Shl, Shr) + } +} + +impl Quote for Lit { + fn quote(&self) -> TokenStream { + macro_rules! gen_match { + ($($i:ident),*) => { + match *self { + $( Lit::$i(lit) => quote!(::syntax::parse::token::Lit::$i((quote lit))), )* + _ => panic!("Unsupported literal"), + } + } + } + + gen_match!(Byte, Char, Float, Str_, Integer, ByteStr) + } +} + +impl Quote for token::DelimToken { + fn quote(&self) -> TokenStream { + macro_rules! gen_match { + ($($i:ident),*) => { + match *self { + $(token::DelimToken::$i => { quote!(::syntax::parse::token::DelimToken::$i) })* + } + } + } + + gen_match!(Paren, Bracket, Brace, NoDelim) + } +} -- cgit 1.4.1-3-g733a5