diff options
| author | Nick Cameron <ncameron@mozilla.com> | 2016-08-29 16:16:43 +1200 |
|---|---|---|
| committer | Nick Cameron <ncameron@mozilla.com> | 2016-09-22 08:47:57 +1200 |
| commit | 6a2d2c949581c710eeb505000e56ffa1e5a860b5 (patch) | |
| tree | 053157b72ce58708786d8356c07aaafbef95e5c1 /src | |
| parent | c772948b687488a087356cb91432425662e034b9 (diff) | |
| download | rust-6a2d2c949581c710eeb505000e56ffa1e5a860b5.tar.gz rust-6a2d2c949581c710eeb505000e56ffa1e5a860b5.zip | |
Adds a `ProcMacro` form of syntax extension
This commit adds syntax extension forms matching the types for procedural macros 2.0 (RFC #1566), these still require the usual syntax extension boiler plate, but this is a first step towards proper implementation and should be useful for macros 1.1 stuff too. Supports both attribute-like and function-like macros.
Diffstat (limited to 'src')
| -rw-r--r-- | src/librustc_plugin/registry.rs | 2 | ||||
| -rw-r--r-- | src/libsyntax/ext/base.rs | 203 | ||||
| -rw-r--r-- | src/libsyntax/ext/expand.rs | 53 | ||||
| -rw-r--r-- | src/libsyntax/ext/proc_macro_shim.rs | 6 | ||||
| -rw-r--r-- | src/libsyntax/parse/lexer/mod.rs | 58 | ||||
| -rw-r--r-- | src/libsyntax/parse/mod.rs | 7 | ||||
| -rw-r--r-- | src/libsyntax/tokenstream.rs | 7 | ||||
| -rw-r--r-- | src/test/run-pass-fulldeps/auxiliary/proc_macro_def.rs | 56 | ||||
| -rw-r--r-- | src/test/run-pass-fulldeps/proc_macro.rs | 48 |
9 files changed, 422 insertions, 18 deletions
diff --git a/src/librustc_plugin/registry.rs b/src/librustc_plugin/registry.rs index 8f0cc2c3d75..f8bce297a42 100644 --- a/src/librustc_plugin/registry.rs +++ b/src/librustc_plugin/registry.rs @@ -111,6 +111,8 @@ impl<'a> Registry<'a> { } MultiDecorator(ext) => MultiDecorator(ext), MultiModifier(ext) => MultiModifier(ext), + SyntaxExtension::ProcMacro(ext) => SyntaxExtension::ProcMacro(ext), + SyntaxExtension::AttrProcMacro(ext) => SyntaxExtension::AttrProcMacro(ext), })); } diff --git a/src/libsyntax/ext/base.rs b/src/libsyntax/ext/base.rs index 9d0d74138cd..82db9ffca83 100644 --- a/src/libsyntax/ext/base.rs +++ b/src/libsyntax/ext/base.rs @@ -8,7 +8,7 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. -pub use self::SyntaxExtension::*; +pub use self::SyntaxExtension::{MultiDecorator, MultiModifier, NormalTT, IdentTT, MacroRulesTT}; use ast::{self, Attribute, Name, PatKind}; use attr::HasAttrs; @@ -19,7 +19,7 @@ use ext::expand::{self, Invocation, Expansion}; use ext::hygiene::Mark; use ext::tt::macro_rules; use parse; -use parse::parser; +use parse::parser::{self, Parser}; use parse::token; use parse::token::{InternedString, str_to_ident}; use ptr::P; @@ -31,7 +31,8 @@ use feature_gate; use std::collections::HashMap; use std::path::PathBuf; use std::rc::Rc; -use tokenstream; +use std::default::Default; +use tokenstream::{self, TokenStream}; #[derive(Debug,Clone)] @@ -60,6 +61,14 @@ impl HasAttrs for Annotatable { } impl Annotatable { + pub fn span(&self) -> Span { + match *self { + Annotatable::Item(ref item) => item.span, + Annotatable::TraitItem(ref trait_item) => trait_item.span, + Annotatable::ImplItem(ref impl_item) => impl_item.span, + } + } + pub fn expect_item(self) -> P<ast::Item> { match self { Annotatable::Item(i) => i, @@ -146,6 +155,173 @@ impl Into<Vec<Annotatable>> for Annotatable { } } +pub trait ProcMacro { + fn expand<'cx>(&self, + ecx: &'cx mut ExtCtxt, + span: Span, + ts: TokenStream) + -> Box<MacResult+'cx>; +} + +impl<F> ProcMacro for F + where F: Fn(TokenStream) -> TokenStream +{ + fn expand<'cx>(&self, + ecx: &'cx mut ExtCtxt, + span: Span, + ts: TokenStream) + -> Box<MacResult+'cx> { + let result = (*self)(ts); + // FIXME setup implicit context in TLS before calling self. + let parser = ecx.new_parser_from_tts(&result.to_tts()); + Box::new(TokResult { parser: parser, span: span }) + } +} + +pub trait AttrProcMacro { + fn expand<'cx>(&self, + ecx: &'cx mut ExtCtxt, + span: Span, + annotation: TokenStream, + annotated: TokenStream) + -> Box<MacResult+'cx>; +} + +impl<F> AttrProcMacro for F + where F: Fn(TokenStream, TokenStream) -> TokenStream +{ + fn expand<'cx>(&self, + ecx: &'cx mut ExtCtxt, + span: Span, + annotation: TokenStream, + annotated: TokenStream) + -> Box<MacResult+'cx> { + // FIXME setup implicit context in TLS before calling self. + let parser = ecx.new_parser_from_tts(&(*self)(annotation, annotated).to_tts()); + Box::new(TokResult { parser: parser, span: span }) + } +} + +struct TokResult<'a> { + parser: Parser<'a>, + span: Span, +} + +impl<'a> MacResult for TokResult<'a> { + fn make_items(mut self: Box<Self>) -> Option<SmallVector<P<ast::Item>>> { + if self.parser.sess.span_diagnostic.has_errors() { + return None; + } + + let mut items = SmallVector::zero(); + loop { + match self.parser.parse_item() { + Ok(Some(item)) => { + // FIXME better span info. + let mut item = item.unwrap(); + item.span = self.span; + items.push(P(item)); + } + Ok(None) => { + return Some(items); + } + Err(mut e) => { + e.emit(); + return None; + } + } + } + } + + fn make_impl_items(mut self: Box<Self>) -> Option<SmallVector<ast::ImplItem>> { + let mut items = SmallVector::zero(); + loop { + match self.parser.parse_impl_item() { + Ok(mut item) => { + // FIXME better span info. + item.span = self.span; + items.push(item); + + return Some(items); + } + Err(mut e) => { + e.emit(); + return None; + } + } + } + } + + fn make_trait_items(mut self: Box<Self>) -> Option<SmallVector<ast::TraitItem>> { + let mut items = SmallVector::zero(); + loop { + match self.parser.parse_trait_item() { + Ok(mut item) => { + // FIXME better span info. + item.span = self.span; + items.push(item); + + return Some(items); + } + Err(mut e) => { + e.emit(); + return None; + } + } + } + } + + fn make_expr(mut self: Box<Self>) -> Option<P<ast::Expr>> { + match self.parser.parse_expr() { + Ok(e) => Some(e), + Err(mut e) => { + e.emit(); + return None; + } + } + } + + fn make_pat(mut self: Box<Self>) -> Option<P<ast::Pat>> { + match self.parser.parse_pat() { + Ok(e) => Some(e), + Err(mut e) => { + e.emit(); + return None; + } + } + } + + fn make_stmts(mut self: Box<Self>) -> Option<SmallVector<ast::Stmt>> { + let mut stmts = SmallVector::zero(); + loop { + if self.parser.token == token::Eof { + return Some(stmts); + } + match self.parser.parse_full_stmt(true) { + Ok(Some(mut stmt)) => { + stmt.span = self.span; + stmts.push(stmt); + } + Ok(None) => { /* continue */ } + Err(mut e) => { + e.emit(); + return None; + } + } + } + } + + fn make_ty(mut self: Box<Self>) -> Option<P<ast::Ty>> { + match self.parser.parse_ty() { + Ok(e) => Some(e), + Err(mut e) => { + e.emit(); + return None; + } + } + } +} + /// Represents a thing that maps token trees to Macro Results pub trait TTMacroExpander { fn expand<'cx>(&self, @@ -439,11 +615,22 @@ pub enum SyntaxExtension { /// based upon it. /// /// `#[derive(...)]` is a `MultiItemDecorator`. - MultiDecorator(Box<MultiItemDecorator + 'static>), + /// + /// Prefer ProcMacro or MultiModifier since they are more flexible. + MultiDecorator(Box<MultiItemDecorator>), /// A syntax extension that is attached to an item and modifies it - /// in-place. More flexible version than Modifier. - MultiModifier(Box<MultiItemModifier + 'static>), + /// in-place. Also allows decoration, i.e., creating new items. + MultiModifier(Box<MultiItemModifier>), + + /// A function-like procedural macro. TokenStream -> TokenStream. + ProcMacro(Box<ProcMacro>), + + /// An attribute-like procedural macro. TokenStream, TokenStream -> TokenStream. + /// The first TokenSteam is the attribute, the second is the annotated item. + /// Allows modification of the input items and adding new items, similar to + /// MultiModifier, but uses TokenStreams, rather than AST nodes. + AttrProcMacro(Box<AttrProcMacro>), /// A normal, function-like syntax extension. /// @@ -451,12 +638,12 @@ pub enum SyntaxExtension { /// /// The `bool` dictates whether the contents of the macro can /// directly use `#[unstable]` things (true == yes). - NormalTT(Box<TTMacroExpander + 'static>, Option<Span>, bool), + NormalTT(Box<TTMacroExpander>, Option<Span>, bool), /// A function-like syntax extension that has an extra ident before /// the block. /// - IdentTT(Box<IdentMacroExpander + 'static>, Option<Span>, bool), + IdentTT(Box<IdentMacroExpander>, Option<Span>, bool), } pub type NamedSyntaxExtension = (Name, SyntaxExtension); diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index 4e87d8ee9dd..f022dd3a08b 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -22,8 +22,9 @@ use feature_gate::{self, Features}; use fold; use fold::*; use parse::token::{intern, keywords}; +use parse::span_to_tts; use ptr::P; -use tokenstream::TokenTree; +use tokenstream::{TokenTree, TokenStream}; use util::small_vector::SmallVector; use visit::Visitor; @@ -308,6 +309,31 @@ impl<'a, 'b> MacroExpander<'a, 'b> { items.push(item); kind.expect_from_annotatables(items) } + SyntaxExtension::AttrProcMacro(ref mac) => { + let attr_toks = TokenStream::from_tts(span_to_tts(&fld.cx.parse_sess, + attr.span)); + let item_toks = TokenStream::from_tts(span_to_tts(&fld.cx.parse_sess, + item.span())); + let result = mac.expand(self.cx, attr.span, attr_toks, item_toks); + let items = match item { + Annotatable::Item(_) => result.make_items() + .unwrap_or(SmallVector::zero()) + .into_iter() + .map(|i| Annotatable::Item(i)) + .collect(), + Annotatable::TraitItem(_) => result.make_trait_items() + .unwrap_or(SmallVector::zero()) + .into_iter() + .map(|i| Annotatable::TraitItem(P(i))) + .collect(), + Annotatable::ImplItem(_) => result.make_impl_items() + .unwrap_or(SmallVector::zero()) + .into_iter() + .map(|i| Annotatable::ImplItem(P(i))) + .collect(), + }; + kind.expect_from_annotatables(items) + } _ => unreachable!(), } } @@ -377,11 +403,34 @@ impl<'a, 'b> MacroExpander<'a, 'b> { kind.make_from(expander.expand(self.cx, span, ident, marked_tts, attrs)) } - MultiDecorator(..) | MultiModifier(..) => { + MultiDecorator(..) | MultiModifier(..) | SyntaxExtension::AttrProcMacro(..) => { self.cx.span_err(path.span, &format!("`{}` can only be used in attributes", extname)); return kind.dummy(span); } + + SyntaxExtension::ProcMacro(ref expandfun) => { + if ident.name != keywords::Invalid.name() { + let msg = + format!("macro {}! expects no ident argument, given '{}'", extname, ident); + fld.cx.span_err(path.span, &msg); + return None; + } + + fld.cx.bt_push(ExpnInfo { + call_site: call_site, + callee: NameAndSpan { + format: MacroBang(extname), + // FIXME procedural macros do not have proper span info + // yet, when they do, we should use it here. + span: None, + // FIXME probably want to follow macro_rules macros here. + allow_internal_unstable: false, + }, + }); + + Some(expandfun.expand(fld.cx, call_site, TokenStream::from_tts(marked_tts))) + } }; let expanded = if let Some(expanded) = opt_expanded { diff --git a/src/libsyntax/ext/proc_macro_shim.rs b/src/libsyntax/ext/proc_macro_shim.rs index fa37e9b54e4..dc3a01f41bc 100644 --- a/src/libsyntax/ext/proc_macro_shim.rs +++ b/src/libsyntax/ext/proc_macro_shim.rs @@ -24,7 +24,9 @@ use ext::base::*; /// Take a `ExtCtxt`, `Span`, and `TokenStream`, and produce a Macro Result that parses /// the TokenStream as a block and returns it as an `Expr`. -pub fn build_block_emitter<'cx>(cx: &'cx mut ExtCtxt, sp: Span, output: TokenStream) +pub fn build_block_emitter<'cx>(cx: &'cx mut ExtCtxt, + sp: Span, + output: TokenStream) -> Box<MacResult + 'cx> { let parser = cx.new_parser_from_tts(&output.to_tts()); @@ -60,7 +62,7 @@ pub fn build_block_emitter<'cx>(cx: &'cx mut ExtCtxt, sp: Span, output: TokenStr } pub mod prelude { - pub use ext::proc_macro_shim::build_block_emitter; + pub use super::build_block_emitter; pub use ast::Ident; pub use codemap::{DUMMY_SP, Span}; pub use ext::base::{ExtCtxt, MacResult}; diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs index 9e9ea096460..53294e78710 100644 --- a/src/libsyntax/parse/lexer/mod.rs +++ b/src/libsyntax/parse/lexer/mod.rs @@ -85,6 +85,12 @@ pub struct StringReader<'a> { /// The last character to be read pub curr: Option<char>, pub filemap: Rc<syntax_pos::FileMap>, + /// If Some, stop reading the source at this position (inclusive). + pub terminator: Option<BytePos>, + /// Whether to record new-lines in filemap. This is only necessary the first + /// time a filemap is lexed. If part of a filemap is being re-lexed, this + /// should be set to false. + pub save_new_lines: bool, // cached: pub peek_tok: token::Token, pub peek_span: Span, @@ -96,7 +102,15 @@ pub struct StringReader<'a> { impl<'a> Reader for StringReader<'a> { fn is_eof(&self) -> bool { - self.curr.is_none() + if self.curr.is_none() { + return true; + } + + match self.terminator { + Some(t) => self.pos > t, + None => false, + } + } /// Return the next token. EFFECT: advances the string_reader. fn try_next_token(&mut self) -> Result<TokenAndSpan, ()> { @@ -164,6 +178,14 @@ impl<'a> StringReader<'a> { pub fn new_raw<'b>(span_diagnostic: &'b Handler, filemap: Rc<syntax_pos::FileMap>) -> StringReader<'b> { + let mut sr = StringReader::new_raw_internal(span_diagnostic, filemap); + sr.bump(); + sr + } + + fn new_raw_internal<'b>(span_diagnostic: &'b Handler, + filemap: Rc<syntax_pos::FileMap>) + -> StringReader<'b> { if filemap.src.is_none() { span_diagnostic.bug(&format!("Cannot lex filemap \ without source: {}", @@ -172,21 +194,21 @@ impl<'a> StringReader<'a> { let source_text = (*filemap.src.as_ref().unwrap()).clone(); - let mut sr = StringReader { + StringReader { span_diagnostic: span_diagnostic, pos: filemap.start_pos, last_pos: filemap.start_pos, col: CharPos(0), curr: Some('\n'), filemap: filemap, + terminator: None, + save_new_lines: true, // dummy values; not read peek_tok: token::Eof, peek_span: syntax_pos::DUMMY_SP, source_text: source_text, fatal_errs: Vec::new(), - }; - sr.bump(); - sr + } } pub fn new<'b>(span_diagnostic: &'b Handler, @@ -200,6 +222,28 @@ impl<'a> StringReader<'a> { sr } + pub fn from_span<'b>(span_diagnostic: &'b Handler, + span: Span, + codemap: &CodeMap) + -> StringReader<'b> { + let start_pos = codemap.lookup_byte_offset(span.lo); + let last_pos = codemap.lookup_byte_offset(span.hi); + assert!(start_pos.fm.name == last_pos.fm.name, "Attempt to lex span which crosses files"); + let mut sr = StringReader::new_raw_internal(span_diagnostic, start_pos.fm.clone()); + sr.pos = span.lo; + sr.last_pos = span.lo; + sr.terminator = Some(span.hi); + sr.save_new_lines = false; + + sr.bump(); + + if let Err(_) = sr.advance_token() { + sr.emit_fatal_errors(); + panic!(FatalError); + } + sr + } + pub fn curr_is(&self, c: char) -> bool { self.curr == Some(c) } @@ -405,7 +449,9 @@ impl<'a> StringReader<'a> { self.curr = Some(ch); self.col = self.col + CharPos(1); if last_char == '\n' { - self.filemap.next_line(self.last_pos); + if self.save_new_lines { + self.filemap.next_line(self.last_pos); + } self.col = CharPos(0); } diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index 5aa0efdec11..4ad8e227cbb 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -258,6 +258,13 @@ fn file_to_filemap(sess: &ParseSess, path: &Path, spanopt: Option<Span>) } } +pub fn span_to_tts(sess: &ParseSess, span: Span) -> Vec<tokenstream::TokenTree> { + let cfg = Vec::new(); + let srdr = lexer::StringReader::from_span(&sess.span_diagnostic, span, &sess.code_map); + let mut p1 = Parser::new(sess, cfg, Box::new(srdr)); + panictry!(p1.parse_all_token_trees()) +} + /// Given a filemap, produce a sequence of token-trees pub fn filemap_to_tts(sess: &ParseSess, filemap: Rc<FileMap>) -> Vec<tokenstream::TokenTree> { diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index 7b1df6f0e97..b35b4617ea1 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -33,6 +33,7 @@ use parse::lexer::comments::{doc_comment_style, strip_doc_comment_decoration}; use parse::lexer; use parse; use parse::token::{self, Token, Lit, Nonterminal}; +use print::pprust; use std::fmt; use std::iter::*; @@ -781,6 +782,12 @@ impl TokenStream { } } +impl fmt::Display for TokenStream { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str(&pprust::tts_to_string(&self.to_tts())) + } +} + // FIXME Reimplement this iterator to hold onto a slice iterator for a leaf, getting the // next leaf's iterator when the current one is exhausted. pub struct Iter<'a> { diff --git a/src/test/run-pass-fulldeps/auxiliary/proc_macro_def.rs b/src/test/run-pass-fulldeps/auxiliary/proc_macro_def.rs new file mode 100644 index 00000000000..52c38a6ee03 --- /dev/null +++ b/src/test/run-pass-fulldeps/auxiliary/proc_macro_def.rs @@ -0,0 +1,56 @@ +// Copyright 2016 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license +// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +#![feature(plugin, plugin_registrar, rustc_private)] + +extern crate proc_macro; +extern crate rustc_plugin; +extern crate syntax; + +use proc_macro::prelude::*; +use rustc_plugin::Registry; +use syntax::ext::base::SyntaxExtension; +use syntax::ext::proc_macro_shim::prelude::*; + +#[plugin_registrar] +pub fn plugin_registrar(reg: &mut Registry) { + reg.register_syntax_extension(token::intern("attr_tru"), + SyntaxExtension::AttrProcMacro(Box::new(attr_tru))); + reg.register_syntax_extension(token::intern("attr_identity"), + SyntaxExtension::AttrProcMacro(Box::new(attr_identity))); + reg.register_syntax_extension(token::intern("tru"), + SyntaxExtension::ProcMacro(Box::new(tru))); + reg.register_syntax_extension(token::intern("ret_tru"), + SyntaxExtension::ProcMacro(Box::new(ret_tru))); + reg.register_syntax_extension(token::intern("identity"), + SyntaxExtension::ProcMacro(Box::new(identity))); +} + +fn attr_tru(_attr: TokenStream, _item: TokenStream) -> TokenStream { + lex("fn f1() -> bool { true }") +} + +fn attr_identity(_attr: TokenStream, item: TokenStream) -> TokenStream { + let source = item.to_string(); + lex(&source) +} + +fn tru(_ts: TokenStream) -> TokenStream { + lex("true") +} + +fn ret_tru(_ts: TokenStream) -> TokenStream { + lex("return true;") +} + +fn identity(ts: TokenStream) -> TokenStream { + let source = ts.to_string(); + lex(&source) +} diff --git a/src/test/run-pass-fulldeps/proc_macro.rs b/src/test/run-pass-fulldeps/proc_macro.rs new file mode 100644 index 00000000000..22cc9f0f8d4 --- /dev/null +++ b/src/test/run-pass-fulldeps/proc_macro.rs @@ -0,0 +1,48 @@ +// Copyright 2016 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license +// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// aux-build:proc_macro_def.rs +// ignore-stage1 +// ignore-cross-compile + +#![feature(plugin, custom_attribute)] +#![feature(type_macros)] + +#![plugin(proc_macro_def)] + +#[attr_tru] +fn f1() -> bool { + return false; +} + +#[attr_identity] +fn f2() -> bool { + return identity!(true); +} + +fn f3() -> identity!(bool) { + ret_tru!(); +} + +fn f4(x: bool) -> bool { + match x { + identity!(true) => false, + identity!(false) => true, + } +} + +fn main() { + assert!(f1()); + assert!(f2()); + assert!(tru!()); + assert!(f3()); + assert!(identity!(5 == 5)); + assert!(f4(false)); +} |
