From 3832a634d3aa6a7c60448906e6656a22f7e35628 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Tue, 8 Oct 2019 10:34:53 +1100 Subject: Optimize `TokenStream::from_streams`. Currently, this function creates a new empty stream, and then appends the elements from each given stream onto that stream. This can cause quadratic behaviour. This commit changes the function so that it modifies the first stream (which can be long) by extending it with the elements from the subsequent streams (which are almost always short), which avoids the quadratic behaviour. --- src/libsyntax/tokenstream.rs | 47 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 10 deletions(-) (limited to 'src/libsyntax/tokenstream.rs') diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index 26cae2a8e7c..76983f8ef85 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -249,20 +249,47 @@ impl TokenStream { 0 => TokenStream::empty(), 1 => streams.pop().unwrap(), _ => { - // rust-lang/rust#57735: pre-allocate vector to avoid - // quadratic blow-up due to on-the-fly reallocations. - let tree_count = streams.iter() - .map(|ts| match &ts.0 { None => 0, Some(s) => s.len() }) + // We are going to extend the first stream in `streams` with + // the elements from the subsequent streams. This requires + // using `make_mut()` on the first stream, and in practice this + // doesn't cause cloning 99.9% of the time. + // + // One very common use case is when `streams` has two elements, + // where the first stream has any number of elements within + // (often 1, but sometimes many more) and the second stream has + // a single element within. + + // Determine how much the first stream will be extended. + // Needed to avoid quadratic blow up from on-the-fly + // reallocations (#57735). + let num_appends = streams.iter() + .skip(1) + .map(|ts| ts.len()) .sum(); - let mut vec = Vec::with_capacity(tree_count); - for stream in streams { - match stream.0 { - None => {}, - Some(stream2) => vec.extend(stream2.iter().cloned()), + // Get the first stream. If it's `None`, create an empty + // stream. + let mut iter = streams.drain(); + let mut first_stream_lrc = match iter.next().unwrap().0 { + Some(first_stream_lrc) => first_stream_lrc, + None => Lrc::new(vec![]), + }; + + // Append the elements to the first stream, after reserving + // space for them. + let first_vec_mut = Lrc::make_mut(&mut first_stream_lrc); + first_vec_mut.reserve(num_appends); + for stream in iter { + if let Some(stream) = stream.0 { + first_vec_mut.extend(stream.iter().cloned()); } } - TokenStream::new(vec) + + // Create the final `TokenStream`. + match first_vec_mut.len() { + 0 => TokenStream(None), + _ => TokenStream(Some(first_stream_lrc)), + } } } } -- cgit 1.4.1-3-g733a5 From 75e0078a1703448a19e25eac85daaa5a4e6e68ac Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Tue, 8 Oct 2019 13:32:59 +1100 Subject: Optimize `TokenStreamBuilder::push`. Currently, when two tokens must be glued together, this function duplicates large chunks of the existing streams. This can cause quadratic behaviour. This commit changes the function so that it overwrites the last token with a glued token, which avoids the quadratic behaviour. This removes the need for `TokenStreamBuilder::push_all_but_{first,last}_tree`. The commit also restructures `push` somewhat, by removing `TokenStream::{first_tree_and_joint,last_tree_if_joint}` in favour of more pattern matching and some comments. This makes the code shorter, and in my opinion, more readable. --- src/libsyntax/tokenstream.rs | 94 ++++++++++++++++++++------------------------ 1 file changed, 43 insertions(+), 51 deletions(-) (limited to 'src/libsyntax/tokenstream.rs') diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index 76983f8ef85..bef12ed4fad 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -390,25 +390,6 @@ impl TokenStream { .collect()) })) } - - fn first_tree_and_joint(&self) -> Option { - self.0.as_ref().map(|stream| { - stream.first().unwrap().clone() - }) - } - - fn last_tree_if_joint(&self) -> Option { - match self.0 { - None => None, - Some(ref stream) => { - if let (tree, Joint) = stream.last().unwrap() { - Some(tree.clone()) - } else { - None - } - } - } - } } // 99.5%+ of the time we have 1 or 2 elements in this vector. @@ -421,18 +402,49 @@ impl TokenStreamBuilder { } pub fn push>(&mut self, stream: T) { - let stream = stream.into(); - let last_tree_if_joint = self.0.last().and_then(TokenStream::last_tree_if_joint); - if let Some(TokenTree::Token(last_token)) = last_tree_if_joint { - if let Some((TokenTree::Token(token), is_joint)) = stream.first_tree_and_joint() { - if let Some(glued_tok) = last_token.glue(&token) { - let last_stream = self.0.pop().unwrap(); - self.push_all_but_last_tree(&last_stream); - let glued_tt = TokenTree::Token(glued_tok); - let glued_tokenstream = TokenStream::new(vec![(glued_tt, is_joint)]); - self.0.push(glued_tokenstream); - self.push_all_but_first_tree(&stream); - return + let mut stream = stream.into(); + + // If `self` is not empty and the last tree within the last stream is a + // token tree marked with `Joint`... + if let Some(TokenStream(Some(ref mut last_stream_lrc))) = self.0.last_mut() { + if let Some((TokenTree::Token(last_token), Joint)) = last_stream_lrc.last() { + + // ...and `stream` is not empty and the first tree within it is + // a token tree... + if let TokenStream(Some(ref mut stream_lrc)) = stream { + if let Some((TokenTree::Token(token), is_joint)) = stream_lrc.first() { + + // ...and the two tokens can be glued together... + if let Some(glued_tok) = last_token.glue(&token) { + + // ...then do so, by overwriting the last token + // tree in `self` and removing the first token tree + // from `stream`. This requires using `make_mut()` + // on the last stream in `self` and on `stream`, + // and in practice this doesn't cause cloning 99.9% + // of the time. + + // Overwrite the last token tree with the merged + // token. + let last_vec_mut = Lrc::make_mut(last_stream_lrc); + *last_vec_mut.last_mut().unwrap() = + (TokenTree::Token(glued_tok), *is_joint); + + // Remove the first token tree from `stream`. (This + // is almost always the only tree in `stream`.) + let stream_vec_mut = Lrc::make_mut(stream_lrc); + stream_vec_mut.remove(0); + + // Don't push `stream` if it's empty -- that could + // block subsequent token gluing, by getting + // between two token trees that should be glued + // together. + if !stream.is_empty() { + self.0.push(stream); + } + return; + } + } } } } @@ -442,26 +454,6 @@ impl TokenStreamBuilder { pub fn build(self) -> TokenStream { TokenStream::from_streams(self.0) } - - fn push_all_but_last_tree(&mut self, stream: &TokenStream) { - if let Some(ref streams) = stream.0 { - let len = streams.len(); - match len { - 1 => {} - _ => self.0.push(TokenStream(Some(Lrc::new(streams[0 .. len - 1].to_vec())))), - } - } - } - - fn push_all_but_first_tree(&mut self, stream: &TokenStream) { - if let Some(ref streams) = stream.0 { - let len = streams.len(); - match len { - 1 => {} - _ => self.0.push(TokenStream(Some(Lrc::new(streams[1 .. len].to_vec())))), - } - } - } } #[derive(Clone)] -- cgit 1.4.1-3-g733a5 From ab8105ee9703882534203237b3a6494842d65a75 Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Wed, 9 Oct 2019 01:45:40 +0200 Subject: tokenstream: don't depend on pprust --- src/libsyntax/ext/mbe/macro_rules.rs | 6 ++++-- src/libsyntax/ext/proc_macro_server.rs | 3 ++- src/libsyntax/tokenstream.rs | 9 +-------- 3 files changed, 7 insertions(+), 11 deletions(-) (limited to 'src/libsyntax/tokenstream.rs') diff --git a/src/libsyntax/ext/mbe/macro_rules.rs b/src/libsyntax/ext/mbe/macro_rules.rs index aec4a683141..e4fc269d147 100644 --- a/src/libsyntax/ext/mbe/macro_rules.rs +++ b/src/libsyntax/ext/mbe/macro_rules.rs @@ -174,7 +174,8 @@ fn generic_extension<'cx>( rhses: &[mbe::TokenTree], ) -> Box { if cx.trace_macros() { - trace_macros_note(cx, sp, format!("expanding `{}! {{ {} }}`", name, arg)); + let msg = format!("expanding `{}! {{ {} }}`", name, pprust::tts_to_string(arg.clone())); + trace_macros_note(cx, sp, msg); } // Which arm's failure should we report? (the one furthest along) @@ -212,7 +213,8 @@ fn generic_extension<'cx>( } if cx.trace_macros() { - trace_macros_note(cx, sp, format!("to `{}`", tts)); + let msg = format!("to `{}`", pprust::tts_to_string(tts.clone())); + trace_macros_note(cx, sp, msg); } let directory = Directory { diff --git a/src/libsyntax/ext/proc_macro_server.rs b/src/libsyntax/ext/proc_macro_server.rs index 021ec46d987..3ed6f4114ae 100644 --- a/src/libsyntax/ext/proc_macro_server.rs +++ b/src/libsyntax/ext/proc_macro_server.rs @@ -2,6 +2,7 @@ use crate::ast; use crate::ext::base::ExtCtxt; use crate::parse::{self, token, ParseSess}; use crate::parse::lexer::comments; +use crate::print::pprust; use crate::tokenstream::{self, DelimSpan, IsJoint::*, TokenStream, TreeAndJoint}; use errors::Diagnostic; @@ -407,7 +408,7 @@ impl server::TokenStream for Rustc<'_> { ) } fn to_string(&mut self, stream: &Self::TokenStream) -> String { - stream.to_string() + pprust::tts_to_string(stream.clone()) } fn from_token_tree( &mut self, diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index bef12ed4fad..970bacdde13 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -14,7 +14,6 @@ //! ownership of the original. use crate::parse::token::{self, DelimToken, Token, TokenKind}; -use crate::print::pprust; use syntax_pos::{BytePos, Span, DUMMY_SP}; #[cfg(target_arch = "x86_64")] @@ -23,7 +22,7 @@ use rustc_data_structures::sync::Lrc; use rustc_serialize::{Decoder, Decodable, Encoder, Encodable}; use smallvec::{SmallVec, smallvec}; -use std::{fmt, iter, mem}; +use std::{iter, mem}; #[cfg(test)] mod tests; @@ -507,12 +506,6 @@ impl Cursor { } } -impl fmt::Display for TokenStream { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(&pprust::tts_to_string(self.clone())) - } -} - impl Encodable for TokenStream { fn encode(&self, encoder: &mut E) -> Result<(), E::Error> { self.trees().collect::>().encode(encoder) -- cgit 1.4.1-3-g733a5 From 5c93492da914f972fb549e95db778adcbdf70480 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Thu, 10 Oct 2019 07:29:02 +1100 Subject: Remove the `Option` in `TokenStream`. It means an allocation is required to create an empty `TokenStream`, but all other operations are simpler and marginally faster due to not having to check for `None`. Overall it simplifies the code for a negligible performance effect. The commit also removes `TokenStream::empty` by implementing `Default`, which is now possible. --- src/libsyntax/attr/mod.rs | 2 +- src/libsyntax/ext/expand.rs | 4 +- src/libsyntax/ext/mbe/transcribe.rs | 2 +- src/libsyntax/ext/placeholders.rs | 2 +- src/libsyntax/ext/proc_macro_server.rs | 2 +- src/libsyntax/mut_visit.rs | 6 +- src/libsyntax/parse/attr.rs | 2 +- src/libsyntax/tokenstream.rs | 224 ++++++++++++++------------------- src/libsyntax_ext/plugin_macro_defs.rs | 2 +- 9 files changed, 101 insertions(+), 145 deletions(-) (limited to 'src/libsyntax/tokenstream.rs') diff --git a/src/libsyntax/attr/mod.rs b/src/libsyntax/attr/mod.rs index 7bef693a5be..c2c883fd20e 100644 --- a/src/libsyntax/attr/mod.rs +++ b/src/libsyntax/attr/mod.rs @@ -551,7 +551,7 @@ impl MetaItem { impl MetaItemKind { pub fn tokens(&self, span: Span) -> TokenStream { match *self { - MetaItemKind::Word => TokenStream::empty(), + MetaItemKind::Word => TokenStream::default(), MetaItemKind::NameValue(ref lit) => { let mut vec = vec![TokenTree::token(token::Eq, span).into()]; lit.tokens().append_to_tree_and_joint_vec(&mut vec); diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index bbd8da2acef..ec87451edbd 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -671,12 +671,12 @@ impl<'a, 'b> MacroExpander<'a, 'b> { } } Some(TokenTree::Token(..)) => {} - None => return TokenStream::empty(), + None => return TokenStream::default(), } self.cx.span_err(span, "custom attribute invocations must be \ of the form `#[foo]` or `#[foo(..)]`, the macro name must only be \ followed by a delimiter token"); - TokenStream::empty() + TokenStream::default() } fn gate_proc_macro_attr_item(&self, span: Span, item: &Annotatable) { diff --git a/src/libsyntax/ext/mbe/transcribe.rs b/src/libsyntax/ext/mbe/transcribe.rs index ba818ebd35c..da930436d81 100644 --- a/src/libsyntax/ext/mbe/transcribe.rs +++ b/src/libsyntax/ext/mbe/transcribe.rs @@ -95,7 +95,7 @@ pub(super) fn transcribe( ) -> TokenStream { // Nothing for us to transcribe... if src.is_empty() { - return TokenStream::empty(); + return TokenStream::default(); } // We descend into the RHS (`src`), expanding things as we go. This stack contains the things diff --git a/src/libsyntax/ext/placeholders.rs b/src/libsyntax/ext/placeholders.rs index 8eecef1020d..43a49dbeb51 100644 --- a/src/libsyntax/ext/placeholders.rs +++ b/src/libsyntax/ext/placeholders.rs @@ -15,7 +15,7 @@ pub fn placeholder(kind: AstFragmentKind, id: ast::NodeId) -> AstFragment { fn mac_placeholder() -> ast::Mac { ast::Mac { path: ast::Path { span: DUMMY_SP, segments: Vec::new() }, - tts: TokenStream::empty().into(), + tts: TokenStream::default().into(), delim: ast::MacDelimiter::Brace, span: DUMMY_SP, prior_type_ascription: None, diff --git a/src/libsyntax/ext/proc_macro_server.rs b/src/libsyntax/ext/proc_macro_server.rs index 021ec46d987..27ac8a2b78e 100644 --- a/src/libsyntax/ext/proc_macro_server.rs +++ b/src/libsyntax/ext/proc_macro_server.rs @@ -393,7 +393,7 @@ impl server::Types for Rustc<'_> { impl server::TokenStream for Rustc<'_> { fn new(&mut self) -> Self::TokenStream { - TokenStream::empty() + TokenStream::default() } fn is_empty(&mut self, stream: &Self::TokenStream) -> bool { stream.is_empty() diff --git a/src/libsyntax/mut_visit.rs b/src/libsyntax/mut_visit.rs index 3923b9f297b..60ee17d09b7 100644 --- a/src/libsyntax/mut_visit.rs +++ b/src/libsyntax/mut_visit.rs @@ -610,10 +610,8 @@ pub fn noop_visit_tt(tt: &mut TokenTree, vis: &mut T) { } pub fn noop_visit_tts(TokenStream(tts): &mut TokenStream, vis: &mut T) { - visit_opt(tts, |tts| { - let tts = Lrc::make_mut(tts); - visit_vec(tts, |(tree, _is_joint)| vis.visit_tt(tree)); - }) + let tts = Lrc::make_mut(tts); + visit_vec(tts, |(tree, _is_joint)| vis.visit_tt(tree)); } // Applies ident visitor if it's an ident; applies other visits to interpolated nodes. diff --git a/src/libsyntax/parse/attr.rs b/src/libsyntax/parse/attr.rs index e74f3045db8..0963efcfc8a 100644 --- a/src/libsyntax/parse/attr.rs +++ b/src/libsyntax/parse/attr.rs @@ -203,7 +203,7 @@ impl<'a> Parser<'a> { }; TokenStream::from_streams(smallvec![eq.into(), tokens]) } else { - TokenStream::empty() + TokenStream::default() }; ast::AttrItem { path, tokens } }) diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index bef12ed4fad..d2def5630e6 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -137,13 +137,8 @@ impl TokenTree { /// The goal is for procedural macros to work with `TokenStream`s and `TokenTree`s /// instead of a representation of the abstract syntax tree. /// Today's `TokenTree`s can still contain AST via `token::Interpolated` for back-compat. -/// -/// The use of `Option` is an optimization that avoids the need for an -/// allocation when the stream is empty. However, it is not guaranteed that an -/// empty stream is represented with `None`; it may be represented as a `Some` -/// around an empty `Vec`. -#[derive(Clone, Debug)] -pub struct TokenStream(pub Option>>); +#[derive(Clone, Debug, Default)] +pub struct TokenStream(pub Lrc>); pub type TreeAndJoint = (TokenTree, IsJoint); @@ -164,36 +159,34 @@ impl TokenStream { /// separating the two arguments with a comma for diagnostic suggestions. pub(crate) fn add_comma(&self) -> Option<(TokenStream, Span)> { // Used to suggest if a user writes `foo!(a b);` - if let Some(ref stream) = self.0 { - let mut suggestion = None; - let mut iter = stream.iter().enumerate().peekable(); - while let Some((pos, ts)) = iter.next() { - if let Some((_, next)) = iter.peek() { - let sp = match (&ts, &next) { - (_, (TokenTree::Token(Token { kind: token::Comma, .. }), _)) => continue, - ((TokenTree::Token(token_left), NonJoint), - (TokenTree::Token(token_right), _)) - if ((token_left.is_ident() && !token_left.is_reserved_ident()) - || token_left.is_lit()) && - ((token_right.is_ident() && !token_right.is_reserved_ident()) - || token_right.is_lit()) => token_left.span, - ((TokenTree::Delimited(sp, ..), NonJoint), _) => sp.entire(), - _ => continue, - }; - let sp = sp.shrink_to_hi(); - let comma = (TokenTree::token(token::Comma, sp), NonJoint); - suggestion = Some((pos, comma, sp)); - } - } - if let Some((pos, comma, sp)) = suggestion { - let mut new_stream = vec![]; - let parts = stream.split_at(pos + 1); - new_stream.extend_from_slice(parts.0); - new_stream.push(comma); - new_stream.extend_from_slice(parts.1); - return Some((TokenStream::new(new_stream), sp)); + let mut suggestion = None; + let mut iter = self.0.iter().enumerate().peekable(); + while let Some((pos, ts)) = iter.next() { + if let Some((_, next)) = iter.peek() { + let sp = match (&ts, &next) { + (_, (TokenTree::Token(Token { kind: token::Comma, .. }), _)) => continue, + ((TokenTree::Token(token_left), NonJoint), + (TokenTree::Token(token_right), _)) + if ((token_left.is_ident() && !token_left.is_reserved_ident()) + || token_left.is_lit()) && + ((token_right.is_ident() && !token_right.is_reserved_ident()) + || token_right.is_lit()) => token_left.span, + ((TokenTree::Delimited(sp, ..), NonJoint), _) => sp.entire(), + _ => continue, + }; + let sp = sp.shrink_to_hi(); + let comma = (TokenTree::token(token::Comma, sp), NonJoint); + suggestion = Some((pos, comma, sp)); } } + if let Some((pos, comma, sp)) = suggestion { + let mut new_stream = vec![]; + let parts = self.0.split_at(pos + 1); + new_stream.extend_from_slice(parts.0); + new_stream.push(comma); + new_stream.extend_from_slice(parts.1); + return Some((TokenStream::new(new_stream), sp)); + } None } } @@ -225,28 +218,21 @@ impl PartialEq for TokenStream { } impl TokenStream { - pub fn len(&self) -> usize { - if let Some(ref slice) = self.0 { - slice.len() - } else { - 0 - } + pub fn new(streams: Vec) -> TokenStream { + TokenStream(Lrc::new(streams)) } - pub fn empty() -> TokenStream { - TokenStream(None) + pub fn is_empty(&self) -> bool { + self.0.is_empty() } - pub fn is_empty(&self) -> bool { - match self.0 { - None => true, - Some(ref stream) => stream.is_empty(), - } + pub fn len(&self) -> usize { + self.0.len() } pub(crate) fn from_streams(mut streams: SmallVec<[TokenStream; 2]>) -> TokenStream { match streams.len() { - 0 => TokenStream::empty(), + 0 => TokenStream::default(), 1 => streams.pop().unwrap(), _ => { // We are going to extend the first stream in `streams` with @@ -270,41 +256,24 @@ impl TokenStream { // Get the first stream. If it's `None`, create an empty // stream. let mut iter = streams.drain(); - let mut first_stream_lrc = match iter.next().unwrap().0 { - Some(first_stream_lrc) => first_stream_lrc, - None => Lrc::new(vec![]), - }; + let mut first_stream_lrc = iter.next().unwrap().0; // Append the elements to the first stream, after reserving // space for them. let first_vec_mut = Lrc::make_mut(&mut first_stream_lrc); first_vec_mut.reserve(num_appends); for stream in iter { - if let Some(stream) = stream.0 { - first_vec_mut.extend(stream.iter().cloned()); - } + first_vec_mut.extend(stream.0.iter().cloned()); } // Create the final `TokenStream`. - match first_vec_mut.len() { - 0 => TokenStream(None), - _ => TokenStream(Some(first_stream_lrc)), - } + TokenStream(first_stream_lrc) } } } - pub fn new(streams: Vec) -> TokenStream { - match streams.len() { - 0 => TokenStream(None), - _ => TokenStream(Some(Lrc::new(streams))), - } - } - pub fn append_to_tree_and_joint_vec(self, vec: &mut Vec) { - if let Some(stream) = self.0 { - vec.extend(stream.iter().cloned()); - } + vec.extend(self.0.iter().cloned()); } pub fn trees(&self) -> Cursor { @@ -371,24 +340,22 @@ impl TokenStream { } pub fn map_enumerated TokenTree>(self, mut f: F) -> TokenStream { - TokenStream(self.0.map(|stream| { - Lrc::new( - stream - .iter() - .enumerate() - .map(|(i, (tree, is_joint))| (f(i, tree.clone()), *is_joint)) - .collect()) - })) + TokenStream(Lrc::new( + self.0 + .iter() + .enumerate() + .map(|(i, (tree, is_joint))| (f(i, tree.clone()), *is_joint)) + .collect() + )) } pub fn map TokenTree>(self, mut f: F) -> TokenStream { - TokenStream(self.0.map(|stream| { - Lrc::new( - stream - .iter() - .map(|(tree, is_joint)| (f(tree.clone()), *is_joint)) - .collect()) - })) + TokenStream(Lrc::new( + self.0 + .iter() + .map(|(tree, is_joint)| (f(tree.clone()), *is_joint)) + .collect() + )) } } @@ -406,44 +373,43 @@ impl TokenStreamBuilder { // If `self` is not empty and the last tree within the last stream is a // token tree marked with `Joint`... - if let Some(TokenStream(Some(ref mut last_stream_lrc))) = self.0.last_mut() { + if let Some(TokenStream(ref mut last_stream_lrc)) = self.0.last_mut() { if let Some((TokenTree::Token(last_token), Joint)) = last_stream_lrc.last() { // ...and `stream` is not empty and the first tree within it is // a token tree... - if let TokenStream(Some(ref mut stream_lrc)) = stream { - if let Some((TokenTree::Token(token), is_joint)) = stream_lrc.first() { - - // ...and the two tokens can be glued together... - if let Some(glued_tok) = last_token.glue(&token) { - - // ...then do so, by overwriting the last token - // tree in `self` and removing the first token tree - // from `stream`. This requires using `make_mut()` - // on the last stream in `self` and on `stream`, - // and in practice this doesn't cause cloning 99.9% - // of the time. - - // Overwrite the last token tree with the merged - // token. - let last_vec_mut = Lrc::make_mut(last_stream_lrc); - *last_vec_mut.last_mut().unwrap() = - (TokenTree::Token(glued_tok), *is_joint); - - // Remove the first token tree from `stream`. (This - // is almost always the only tree in `stream`.) - let stream_vec_mut = Lrc::make_mut(stream_lrc); - stream_vec_mut.remove(0); - - // Don't push `stream` if it's empty -- that could - // block subsequent token gluing, by getting - // between two token trees that should be glued - // together. - if !stream.is_empty() { - self.0.push(stream); - } - return; + let TokenStream(ref mut stream_lrc) = stream; + if let Some((TokenTree::Token(token), is_joint)) = stream_lrc.first() { + + // ...and the two tokens can be glued together... + if let Some(glued_tok) = last_token.glue(&token) { + + // ...then do so, by overwriting the last token + // tree in `self` and removing the first token tree + // from `stream`. This requires using `make_mut()` + // on the last stream in `self` and on `stream`, + // and in practice this doesn't cause cloning 99.9% + // of the time. + + // Overwrite the last token tree with the merged + // token. + let last_vec_mut = Lrc::make_mut(last_stream_lrc); + *last_vec_mut.last_mut().unwrap() = + (TokenTree::Token(glued_tok), *is_joint); + + // Remove the first token tree from `stream`. (This + // is almost always the only tree in `stream`.) + let stream_vec_mut = Lrc::make_mut(stream_lrc); + stream_vec_mut.remove(0); + + // Don't push `stream` if it's empty -- that could + // block subsequent token gluing, by getting + // between two token trees that should be glued + // together. + if !stream.is_empty() { + self.0.push(stream); } + return; } } } @@ -476,16 +442,11 @@ impl Cursor { } pub fn next_with_joint(&mut self) -> Option { - match self.stream.0 { - None => None, - Some(ref stream) => { - if self.index < stream.len() { - self.index += 1; - Some(stream[self.index - 1].clone()) - } else { - None - } - } + if self.index < self.stream.len() { + self.index += 1; + Some(self.stream.0[self.index - 1].clone()) + } else { + None } } @@ -494,16 +455,13 @@ impl Cursor { return; } let index = self.index; - let stream = mem::replace(&mut self.stream, TokenStream(None)); + let stream = mem::take(&mut self.stream); *self = TokenStream::from_streams(smallvec![stream, new_stream]).into_trees(); self.index = index; } pub fn look_ahead(&self, n: usize) -> Option { - match self.stream.0 { - None => None, - Some(ref stream) => stream[self.index ..].get(n).map(|(tree, _)| tree.clone()), - } + self.stream.0[self.index ..].get(n).map(|(tree, _)| tree.clone()) } } diff --git a/src/libsyntax_ext/plugin_macro_defs.rs b/src/libsyntax_ext/plugin_macro_defs.rs index 315babceae3..62c7e188eba 100644 --- a/src/libsyntax_ext/plugin_macro_defs.rs +++ b/src/libsyntax_ext/plugin_macro_defs.rs @@ -20,7 +20,7 @@ fn plugin_macro_def(name: Name, span: Span) -> P { attr::mk_word_item(Ident::new(sym::rustc_builtin_macro, span))); let parens: TreeAndJoint = TokenTree::Delimited( - DelimSpan::from_single(span), token::Paren, TokenStream::empty() + DelimSpan::from_single(span), token::Paren, TokenStream::default() ).into(); let trees = vec![parens.clone(), TokenTree::token(token::FatArrow, span).into(), parens]; -- cgit 1.4.1-3-g733a5 From d420d719c4c44c3c6d02b5fafba4f2cf5e837dba Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Wed, 16 Oct 2019 10:59:30 +0200 Subject: move syntax::ext to new crate syntax_expand --- Cargo.lock | 27 + src/librustc/Cargo.toml | 1 + src/librustc/hir/def.rs | 2 +- src/librustc/hir/lowering.rs | 6 +- src/librustc/hir/lowering/item.rs | 2 +- src/librustc/hir/map/def_collector.rs | 2 +- src/librustc/hir/map/definitions.rs | 2 +- src/librustc/hir/map/mod.rs | 2 +- src/librustc/ich/hcx.rs | 2 +- src/librustc/ich/impls_syntax.rs | 2 +- src/librustc/lint/mod.rs | 2 +- src/librustc/session/mod.rs | 2 +- src/librustc/ty/mod.rs | 2 +- src/librustc_codegen_llvm/allocator.rs | 2 +- src/librustc_codegen_llvm/lib.rs | 3 +- src/librustc_codegen_ssa/Cargo.toml | 1 + src/librustc_codegen_ssa/back/symbol_export.rs | 2 +- src/librustc_codegen_ssa/back/write.rs | 2 +- src/librustc_codegen_ssa/traits/backend.rs | 2 +- src/librustc_interface/Cargo.toml | 1 + src/librustc_interface/passes.rs | 8 +- src/librustc_metadata/Cargo.toml | 1 + src/librustc_metadata/creader.rs | 2 +- src/librustc_metadata/cstore.rs | 2 +- src/librustc_metadata/decoder.rs | 4 +- src/librustc_metadata/encoder.rs | 2 +- src/librustc_passes/Cargo.toml | 1 + src/librustc_passes/ast_validation.rs | 2 +- src/librustc_plugin/Cargo.toml | 1 + src/librustc_plugin/lib.rs | 2 +- src/librustc_plugin/registry.rs | 4 +- src/librustc_resolve/Cargo.toml | 1 + src/librustc_resolve/build_reduced_graph.rs | 6 +- src/librustc_resolve/diagnostics.rs | 2 +- src/librustc_resolve/late/diagnostics.rs | 2 +- src/librustc_resolve/lib.rs | 14 +- src/librustc_resolve/macros.rs | 10 +- src/librustc_resolve/resolve_imports.rs | 2 +- src/librustdoc/clean/inline.rs | 2 +- src/librustdoc/clean/mod.rs | 2 +- src/librustdoc/doctree.rs | 2 +- src/librustdoc/html/item_type.rs | 2 +- src/librustdoc/html/render.rs | 4 +- src/librustdoc/lib.rs | 1 + src/librustdoc/passes/collect_intra_doc_links.rs | 2 +- src/librustdoc/visit_ast.rs | 2 +- src/libsyntax/attr/mod.rs | 4 +- src/libsyntax/config.rs | 1 + src/libsyntax/ext/allocator.rs | 75 -- src/libsyntax/ext/base.rs | 1188 ----------------- src/libsyntax/ext/build.rs | 639 --------- src/libsyntax/ext/expand.rs | 1543 --------------------- src/libsyntax/ext/mbe.rs | 166 --- src/libsyntax/ext/mbe/macro_check.rs | 626 --------- src/libsyntax/ext/mbe/macro_parser.rs | 943 ------------- src/libsyntax/ext/mbe/macro_rules.rs | 1194 ----------------- src/libsyntax/ext/mbe/quoted.rs | 263 ---- src/libsyntax/ext/mbe/transcribe.rs | 398 ------ src/libsyntax/ext/placeholders.rs | 349 ----- src/libsyntax/ext/proc_macro.rs | 219 --- src/libsyntax/ext/proc_macro_server.rs | 712 ---------- src/libsyntax/feature_gate/check.rs | 2 +- src/libsyntax/feature_gate/mod.rs | 3 +- src/libsyntax/lib.rs | 41 +- src/libsyntax/parse/literal.rs | 2 +- src/libsyntax/parse/parser.rs | 14 +- src/libsyntax/parse/parser/diagnostics.rs | 6 +- src/libsyntax/parse/parser/expr.rs | 2 +- src/libsyntax/parse/parser/item.rs | 6 +- src/libsyntax/parse/parser/pat.rs | 2 +- src/libsyntax/parse/parser/path.rs | 4 +- src/libsyntax/parse/parser/stmt.rs | 6 +- src/libsyntax/parse/parser/ty.rs | 4 +- src/libsyntax/parse/token.rs | 14 +- src/libsyntax/tokenstream.rs | 2 +- src/libsyntax_expand/Cargo.toml | 26 + src/libsyntax_expand/allocator.rs | 75 ++ src/libsyntax_expand/base.rs | 1189 +++++++++++++++++ src/libsyntax_expand/build.rs | 640 +++++++++ src/libsyntax_expand/expand.rs | 1551 ++++++++++++++++++++++ src/libsyntax_expand/lib.rs | 38 + src/libsyntax_expand/mbe.rs | 166 +++ src/libsyntax_expand/mbe/macro_check.rs | 626 +++++++++ src/libsyntax_expand/mbe/macro_parser.rs | 944 +++++++++++++ src/libsyntax_expand/mbe/macro_rules.rs | 1192 +++++++++++++++++ src/libsyntax_expand/mbe/quoted.rs | 264 ++++ src/libsyntax_expand/mbe/transcribe.rs | 399 ++++++ src/libsyntax_expand/placeholders.rs | 350 +++++ src/libsyntax_expand/proc_macro.rs | 215 +++ src/libsyntax_expand/proc_macro_server.rs | 713 ++++++++++ src/libsyntax_ext/Cargo.toml | 1 + src/libsyntax_ext/asm.rs | 2 +- src/libsyntax_ext/assert.rs | 2 +- src/libsyntax_ext/cfg.rs | 2 +- src/libsyntax_ext/cmdline_attrs.rs | 2 +- src/libsyntax_ext/compile_error.rs | 2 +- src/libsyntax_ext/concat.rs | 2 +- src/libsyntax_ext/concat_idents.rs | 2 +- src/libsyntax_ext/deriving/bounds.rs | 2 +- src/libsyntax_ext/deriving/clone.rs | 2 +- src/libsyntax_ext/deriving/cmp/eq.rs | 2 +- src/libsyntax_ext/deriving/cmp/ord.rs | 2 +- src/libsyntax_ext/deriving/cmp/partial_eq.rs | 2 +- src/libsyntax_ext/deriving/cmp/partial_ord.rs | 2 +- src/libsyntax_ext/deriving/debug.rs | 2 +- src/libsyntax_ext/deriving/decodable.rs | 2 +- src/libsyntax_ext/deriving/default.rs | 2 +- src/libsyntax_ext/deriving/encodable.rs | 2 +- src/libsyntax_ext/deriving/generic/mod.rs | 2 +- src/libsyntax_ext/deriving/generic/ty.rs | 2 +- src/libsyntax_ext/deriving/hash.rs | 2 +- src/libsyntax_ext/deriving/mod.rs | 2 +- src/libsyntax_ext/env.rs | 2 +- src/libsyntax_ext/format.rs | 2 +- src/libsyntax_ext/global_allocator.rs | 4 +- src/libsyntax_ext/global_asm.rs | 2 +- src/libsyntax_ext/lib.rs | 6 +- src/libsyntax_ext/log_syntax.rs | 2 +- src/libsyntax_ext/plugin_macro_defs.rs | 2 +- src/libsyntax_ext/proc_macro_harness.rs | 8 +- src/libsyntax_ext/source_util.rs | 5 +- src/libsyntax_ext/standard_library_imports.rs | 6 +- src/libsyntax_ext/test.rs | 2 +- src/libsyntax_ext/test_harness.rs | 4 +- src/libsyntax_ext/trace_macros.rs | 2 +- src/libsyntax_ext/util.rs | 2 +- 126 files changed, 8564 insertions(+), 8494 deletions(-) delete mode 100644 src/libsyntax/ext/allocator.rs delete mode 100644 src/libsyntax/ext/base.rs delete mode 100644 src/libsyntax/ext/build.rs delete mode 100644 src/libsyntax/ext/expand.rs delete mode 100644 src/libsyntax/ext/mbe.rs delete mode 100644 src/libsyntax/ext/mbe/macro_check.rs delete mode 100644 src/libsyntax/ext/mbe/macro_parser.rs delete mode 100644 src/libsyntax/ext/mbe/macro_rules.rs delete mode 100644 src/libsyntax/ext/mbe/quoted.rs delete mode 100644 src/libsyntax/ext/mbe/transcribe.rs delete mode 100644 src/libsyntax/ext/placeholders.rs delete mode 100644 src/libsyntax/ext/proc_macro.rs delete mode 100644 src/libsyntax/ext/proc_macro_server.rs create mode 100644 src/libsyntax_expand/Cargo.toml create mode 100644 src/libsyntax_expand/allocator.rs create mode 100644 src/libsyntax_expand/base.rs create mode 100644 src/libsyntax_expand/build.rs create mode 100644 src/libsyntax_expand/expand.rs create mode 100644 src/libsyntax_expand/lib.rs create mode 100644 src/libsyntax_expand/mbe.rs create mode 100644 src/libsyntax_expand/mbe/macro_check.rs create mode 100644 src/libsyntax_expand/mbe/macro_parser.rs create mode 100644 src/libsyntax_expand/mbe/macro_rules.rs create mode 100644 src/libsyntax_expand/mbe/quoted.rs create mode 100644 src/libsyntax_expand/mbe/transcribe.rs create mode 100644 src/libsyntax_expand/placeholders.rs create mode 100644 src/libsyntax_expand/proc_macro.rs create mode 100644 src/libsyntax_expand/proc_macro_server.rs (limited to 'src/libsyntax/tokenstream.rs') diff --git a/Cargo.lock b/Cargo.lock index 6d40b198927..a5a7521abde 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3112,6 +3112,7 @@ dependencies = [ "serialize", "smallvec", "syntax", + "syntax_expand", "syntax_pos", ] @@ -3427,6 +3428,7 @@ dependencies = [ "rustc_target", "serialize", "syntax", + "syntax_expand", "syntax_pos", "tempfile", ] @@ -3559,6 +3561,7 @@ dependencies = [ "serialize", "smallvec", "syntax", + "syntax_expand", "syntax_ext", "syntax_pos", "tempfile", @@ -3630,6 +3633,7 @@ dependencies = [ "smallvec", "stable_deref_trait", "syntax", + "syntax_expand", "syntax_pos", ] @@ -3678,6 +3682,7 @@ dependencies = [ "rustc_index", "rustc_target", "syntax", + "syntax_expand", "syntax_pos", ] @@ -3695,6 +3700,7 @@ dependencies = [ "rustc", "rustc_metadata", "syntax", + "syntax_expand", "syntax_pos", ] @@ -3723,6 +3729,7 @@ dependencies = [ "rustc_metadata", "smallvec", "syntax", + "syntax_expand", "syntax_pos", ] @@ -4336,6 +4343,25 @@ dependencies = [ "syntax_pos", ] +[[package]] +name = "syntax_expand" +version = "0.0.0" +dependencies = [ + "bitflags", + "lazy_static 1.3.0", + "log", + "rustc_data_structures", + "rustc_errors", + "rustc_index", + "rustc_lexer", + "rustc_target", + "scoped-tls", + "serialize", + "smallvec", + "syntax", + "syntax_pos", +] + [[package]] name = "syntax_ext" version = "0.0.0" @@ -4347,6 +4373,7 @@ dependencies = [ "rustc_target", "smallvec", "syntax", + "syntax_expand", "syntax_pos", ] diff --git a/src/librustc/Cargo.toml b/src/librustc/Cargo.toml index 9b3609eca3e..cf9f36ca37c 100644 --- a/src/librustc/Cargo.toml +++ b/src/librustc/Cargo.toml @@ -29,6 +29,7 @@ rustc_index = { path = "../librustc_index" } errors = { path = "../librustc_errors", package = "rustc_errors" } rustc_serialize = { path = "../libserialize", package = "serialize" } syntax = { path = "../libsyntax" } +syntax_expand = { path = "../libsyntax_expand" } syntax_pos = { path = "../libsyntax_pos" } backtrace = "0.3.3" parking_lot = "0.9" diff --git a/src/librustc/hir/def.rs b/src/librustc/hir/def.rs index f7d31ca06ee..a071a539e01 100644 --- a/src/librustc/hir/def.rs +++ b/src/librustc/hir/def.rs @@ -6,7 +6,7 @@ use crate::ty; use crate::util::nodemap::DefIdMap; use syntax::ast; -use syntax::ext::base::MacroKind; +use syntax_expand::base::MacroKind; use syntax::ast::NodeId; use syntax_pos::Span; use rustc_macros::HashStable; diff --git a/src/librustc/hir/lowering.rs b/src/librustc/hir/lowering.rs index e9788a55812..d5287fd415b 100644 --- a/src/librustc/hir/lowering.rs +++ b/src/librustc/hir/lowering.rs @@ -64,15 +64,15 @@ use syntax::ast; use syntax::ptr::P as AstP; use syntax::ast::*; use syntax::errors; -use syntax::ext::base::SpecialDerives; -use syntax::ext::hygiene::ExpnId; +use syntax_expand::base::SpecialDerives; use syntax::print::pprust; -use syntax::tokenstream::{TokenStream, TokenTree}; use syntax::parse::token::{self, Nonterminal, Token}; +use syntax::tokenstream::{TokenStream, TokenTree}; use syntax::sess::ParseSess; use syntax::source_map::{respan, ExpnData, ExpnKind, DesugaringKind, Spanned}; use syntax::symbol::{kw, sym, Symbol}; use syntax::visit::{self, Visitor}; +use syntax_pos::hygiene::ExpnId; use syntax_pos::Span; const HIR_ID_COUNTER_LOCKED: u32 = 0xFFFFFFFF; diff --git a/src/librustc/hir/lowering/item.rs b/src/librustc/hir/lowering/item.rs index 548a2fedfff..73d2ac5c134 100644 --- a/src/librustc/hir/lowering/item.rs +++ b/src/librustc/hir/lowering/item.rs @@ -18,7 +18,7 @@ use smallvec::SmallVec; use syntax::attr; use syntax::ast::*; use syntax::visit::{self, Visitor}; -use syntax::ext::base::SpecialDerives; +use syntax_expand::base::SpecialDerives; use syntax::source_map::{respan, DesugaringKind, Spanned}; use syntax::symbol::{kw, sym}; use syntax_pos::Span; diff --git a/src/librustc/hir/map/def_collector.rs b/src/librustc/hir/map/def_collector.rs index 1997e2aab35..fbef95fec7d 100644 --- a/src/librustc/hir/map/def_collector.rs +++ b/src/librustc/hir/map/def_collector.rs @@ -2,7 +2,7 @@ use crate::hir::map::definitions::*; use crate::hir::def_id::DefIndex; use syntax::ast::*; -use syntax::ext::hygiene::ExpnId; +use syntax_expand::hygiene::ExpnId; use syntax::visit; use syntax::symbol::{kw, sym}; use syntax::parse::token::{self, Token}; diff --git a/src/librustc/hir/map/definitions.rs b/src/librustc/hir/map/definitions.rs index 71bf230e37d..cc099fcc40f 100644 --- a/src/librustc/hir/map/definitions.rs +++ b/src/librustc/hir/map/definitions.rs @@ -17,7 +17,7 @@ use std::borrow::Borrow; use std::fmt::Write; use std::hash::Hash; use syntax::ast; -use syntax::ext::hygiene::ExpnId; +use syntax_expand::hygiene::ExpnId; use syntax::symbol::{Symbol, sym, InternedString}; use syntax_pos::{Span, DUMMY_SP}; diff --git a/src/librustc/hir/map/mod.rs b/src/librustc/hir/map/mod.rs index 16c4ab7187d..f839087ec02 100644 --- a/src/librustc/hir/map/mod.rs +++ b/src/librustc/hir/map/mod.rs @@ -20,7 +20,7 @@ use rustc_data_structures::svh::Svh; use rustc_index::vec::IndexVec; use syntax::ast::{self, Name, NodeId}; use syntax::source_map::Spanned; -use syntax::ext::base::MacroKind; +use syntax_expand::base::MacroKind; use syntax_pos::{Span, DUMMY_SP}; pub mod blocks; diff --git a/src/librustc/ich/hcx.rs b/src/librustc/ich/hcx.rs index 3e6b271b834..14d0673ecc0 100644 --- a/src/librustc/ich/hcx.rs +++ b/src/librustc/ich/hcx.rs @@ -13,7 +13,7 @@ use std::cell::RefCell; use syntax::ast; use syntax::source_map::SourceMap; -use syntax::ext::hygiene::SyntaxContext; +use syntax_expand::hygiene::SyntaxContext; use syntax::symbol::Symbol; use syntax::tokenstream::DelimSpan; use syntax_pos::{Span, DUMMY_SP}; diff --git a/src/librustc/ich/impls_syntax.rs b/src/librustc/ich/impls_syntax.rs index 23a2f115e05..defc3fb25bc 100644 --- a/src/librustc/ich/impls_syntax.rs +++ b/src/librustc/ich/impls_syntax.rs @@ -59,7 +59,7 @@ impl_stable_hash_for!(enum ::syntax::ast::AsmDialect { Intel }); -impl_stable_hash_for!(enum ::syntax::ext::base::MacroKind { +impl_stable_hash_for!(enum ::syntax_expand::base::MacroKind { Bang, Attr, Derive, diff --git a/src/librustc/lint/mod.rs b/src/librustc/lint/mod.rs index b31efc24e52..7443cca822a 100644 --- a/src/librustc/lint/mod.rs +++ b/src/librustc/lint/mod.rs @@ -39,7 +39,7 @@ use syntax::ast; use syntax::source_map::{MultiSpan, ExpnKind, DesugaringKind}; use syntax::early_buffered_lints::BufferedEarlyLintId; use syntax::edition::Edition; -use syntax::ext::base::MacroKind; +use syntax_expand::base::MacroKind; use syntax::symbol::{Symbol, sym}; use syntax_pos::Span; diff --git a/src/librustc/session/mod.rs b/src/librustc/session/mod.rs index c59df146629..b65bf2230b3 100644 --- a/src/librustc/session/mod.rs +++ b/src/librustc/session/mod.rs @@ -24,7 +24,7 @@ use errors::emitter::HumanReadableErrorType; use errors::annotate_snippet_emitter_writer::{AnnotateSnippetEmitterWriter}; use syntax::ast::{self, NodeId}; use syntax::edition::Edition; -use syntax::ext::allocator::AllocatorKind; +use syntax_expand::allocator::AllocatorKind; use syntax::feature_gate::{self, AttributeType}; use syntax::json::JsonEmitter; use syntax::source_map; diff --git a/src/librustc/ty/mod.rs b/src/librustc/ty/mod.rs index 65aea7b459f..ab9fb1df682 100644 --- a/src/librustc/ty/mod.rs +++ b/src/librustc/ty/mod.rs @@ -45,7 +45,7 @@ use std::{mem, ptr}; use std::ops::Range; use syntax::ast::{self, Name, Ident, NodeId}; use syntax::attr; -use syntax::ext::hygiene::ExpnId; +use syntax_expand::hygiene::ExpnId; use syntax::symbol::{kw, sym, Symbol, InternedString}; use syntax_pos::Span; diff --git a/src/librustc_codegen_llvm/allocator.rs b/src/librustc_codegen_llvm/allocator.rs index f31765cea4f..8c60c030eac 100644 --- a/src/librustc_codegen_llvm/allocator.rs +++ b/src/librustc_codegen_llvm/allocator.rs @@ -3,7 +3,7 @@ use std::ffi::CString; use crate::attributes; use libc::c_uint; use rustc::ty::TyCtxt; -use syntax::ext::allocator::{AllocatorKind, AllocatorTy, ALLOCATOR_METHODS}; +use syntax_expand::allocator::{AllocatorKind, AllocatorTy, ALLOCATOR_METHODS}; use crate::ModuleLlvm; use crate::llvm::{self, False, True}; diff --git a/src/librustc_codegen_llvm/lib.rs b/src/librustc_codegen_llvm/lib.rs index fde04a68457..9b55bef0c51 100644 --- a/src/librustc_codegen_llvm/lib.rs +++ b/src/librustc_codegen_llvm/lib.rs @@ -39,6 +39,7 @@ extern crate rustc_driver as _; #[macro_use] extern crate log; extern crate syntax; +extern crate syntax_expand; extern crate syntax_pos; extern crate rustc_errors as errors; @@ -48,7 +49,7 @@ use rustc_codegen_ssa::back::lto::{SerializedModule, LtoModuleCodegen, ThinModul use rustc_codegen_ssa::CompiledModule; use errors::{FatalError, Handler}; use rustc::dep_graph::WorkProduct; -use syntax::ext::allocator::AllocatorKind; +use syntax_expand::allocator::AllocatorKind; use syntax_pos::symbol::InternedString; pub use llvm_util::target_features; use std::any::Any; diff --git a/src/librustc_codegen_ssa/Cargo.toml b/src/librustc_codegen_ssa/Cargo.toml index c7d09a423d5..6992f93d999 100644 --- a/src/librustc_codegen_ssa/Cargo.toml +++ b/src/librustc_codegen_ssa/Cargo.toml @@ -21,6 +21,7 @@ tempfile = "3.1" rustc_serialize = { path = "../libserialize", package = "serialize" } syntax = { path = "../libsyntax" } +syntax_expand = { path = "../libsyntax_expand" } syntax_pos = { path = "../libsyntax_pos" } rustc = { path = "../librustc" } rustc_apfloat = { path = "../librustc_apfloat" } diff --git a/src/librustc_codegen_ssa/back/symbol_export.rs b/src/librustc_codegen_ssa/back/symbol_export.rs index 9078f77f1f7..d866a10f069 100644 --- a/src/librustc_codegen_ssa/back/symbol_export.rs +++ b/src/librustc_codegen_ssa/back/symbol_export.rs @@ -14,7 +14,7 @@ use rustc::ty::query::Providers; use rustc::ty::subst::SubstsRef; use rustc::util::nodemap::{FxHashMap, DefIdMap}; use rustc_index::vec::IndexVec; -use syntax::ext::allocator::ALLOCATOR_METHODS; +use syntax_expand::allocator::ALLOCATOR_METHODS; pub type ExportedSymbols = FxHashMap< CrateNum, diff --git a/src/librustc_codegen_ssa/back/write.rs b/src/librustc_codegen_ssa/back/write.rs index 5339134a2a8..856c280cac0 100644 --- a/src/librustc_codegen_ssa/back/write.rs +++ b/src/librustc_codegen_ssa/back/write.rs @@ -27,7 +27,7 @@ use rustc_errors::{Handler, Level, FatalError, DiagnosticId, SourceMapperDyn}; use rustc_errors::emitter::{Emitter}; use rustc_target::spec::MergeFunctions; use syntax::attr; -use syntax::ext::hygiene::ExpnId; +use syntax_expand::hygiene::ExpnId; use syntax_pos::symbol::{Symbol, sym}; use jobserver::{Client, Acquired}; diff --git a/src/librustc_codegen_ssa/traits/backend.rs b/src/librustc_codegen_ssa/traits/backend.rs index a7faf4eaf10..7cae3e9ade5 100644 --- a/src/librustc_codegen_ssa/traits/backend.rs +++ b/src/librustc_codegen_ssa/traits/backend.rs @@ -9,7 +9,7 @@ use rustc::ty::TyCtxt; use rustc_codegen_utils::codegen_backend::CodegenBackend; use std::sync::Arc; use std::sync::mpsc; -use syntax::ext::allocator::AllocatorKind; +use syntax_expand::allocator::AllocatorKind; use syntax_pos::symbol::InternedString; pub trait BackendTypes { diff --git a/src/librustc_interface/Cargo.toml b/src/librustc_interface/Cargo.toml index bed5febb72e..7a0d931ca73 100644 --- a/src/librustc_interface/Cargo.toml +++ b/src/librustc_interface/Cargo.toml @@ -15,6 +15,7 @@ rayon = { version = "0.3.0", package = "rustc-rayon" } smallvec = { version = "0.6.7", features = ["union", "may_dangle"] } syntax = { path = "../libsyntax" } syntax_ext = { path = "../libsyntax_ext" } +syntax_expand = { path = "../libsyntax_expand" } syntax_pos = { path = "../libsyntax_pos" } rustc_serialize = { path = "../libserialize", package = "serialize" } rustc = { path = "../librustc" } diff --git a/src/librustc_interface/passes.rs b/src/librustc_interface/passes.rs index 396c5610251..89de5714695 100644 --- a/src/librustc_interface/passes.rs +++ b/src/librustc_interface/passes.rs @@ -35,7 +35,7 @@ use rustc_traits; use rustc_typeck as typeck; use syntax::{self, ast, visit}; use syntax::early_buffered_lints::BufferedEarlyLint; -use syntax::ext::base::{NamedSyntaxExtension, ExtCtxt}; +use syntax_expand::base::{NamedSyntaxExtension, ExtCtxt}; use syntax::mut_visit::MutVisitor; use syntax::parse::{self, PResult}; use syntax::util::node_count::NodeCounter; @@ -397,12 +397,12 @@ fn configure_and_expand_inner<'a>( // Create the config for macro expansion let features = sess.features_untracked(); - let cfg = syntax::ext::expand::ExpansionConfig { + let cfg = syntax_expand::expand::ExpansionConfig { features: Some(&features), recursion_limit: *sess.recursion_limit.get(), trace_mac: sess.opts.debugging_opts.trace_macros, should_test: sess.opts.test, - ..syntax::ext::expand::ExpansionConfig::default(crate_name.to_string()) + ..syntax_expand::expand::ExpansionConfig::default(crate_name.to_string()) }; let mut ecx = ExtCtxt::new(&sess.parse_sess, cfg, &mut resolver); @@ -559,7 +559,7 @@ pub fn lower_to_hir( // Discard hygiene data, which isn't required after lowering to HIR. if !sess.opts.debugging_opts.keep_hygiene_data { - syntax::ext::hygiene::clear_syntax_context_map(); + syntax_expand::hygiene::clear_syntax_context_map(); } Ok(hir_forest) diff --git a/src/librustc_metadata/Cargo.toml b/src/librustc_metadata/Cargo.toml index 032470e1400..18192e35f8a 100644 --- a/src/librustc_metadata/Cargo.toml +++ b/src/librustc_metadata/Cargo.toml @@ -22,4 +22,5 @@ rustc_index = { path = "../librustc_index" } rustc_serialize = { path = "../libserialize", package = "serialize" } stable_deref_trait = "1.0.0" syntax = { path = "../libsyntax" } +syntax_expand = { path = "../libsyntax_expand" } syntax_pos = { path = "../libsyntax_pos" } diff --git a/src/librustc_metadata/creader.rs b/src/librustc_metadata/creader.rs index 9850121d2ce..6a2da5d4988 100644 --- a/src/librustc_metadata/creader.rs +++ b/src/librustc_metadata/creader.rs @@ -26,7 +26,7 @@ use std::{cmp, fs}; use syntax::ast; use syntax::attr; -use syntax::ext::allocator::{global_allocator_spans, AllocatorKind}; +use syntax_expand::allocator::{global_allocator_spans, AllocatorKind}; use syntax::symbol::{Symbol, sym}; use syntax::{span_err, span_fatal}; use syntax_pos::{Span, DUMMY_SP}; diff --git a/src/librustc_metadata/cstore.rs b/src/librustc_metadata/cstore.rs index a5a458e49a3..9a0b98ffb73 100644 --- a/src/librustc_metadata/cstore.rs +++ b/src/librustc_metadata/cstore.rs @@ -11,7 +11,7 @@ use rustc_index::vec::IndexVec; use rustc::util::nodemap::FxHashMap; use rustc_data_structures::sync::{Lrc, RwLock, Lock, MetadataRef, AtomicCell}; use syntax::ast; -use syntax::ext::base::SyntaxExtension; +use syntax_expand::base::SyntaxExtension; use syntax_pos; use proc_macro::bridge::client::ProcMacro; diff --git a/src/librustc_metadata/decoder.rs b/src/librustc_metadata/decoder.rs index 6969d608d76..37f3726124a 100644 --- a/src/librustc_metadata/decoder.rs +++ b/src/librustc_metadata/decoder.rs @@ -32,11 +32,11 @@ use syntax::attr; use syntax::ast::{self, Ident}; use syntax::source_map::{self, respan, Spanned}; use syntax::symbol::{Symbol, sym}; -use syntax::ext::base::{MacroKind, SyntaxExtensionKind, SyntaxExtension}; +use syntax_expand::base::{MacroKind, SyntaxExtensionKind, SyntaxExtension}; use syntax_pos::{self, Span, BytePos, Pos, DUMMY_SP, symbol::{InternedString}}; use log::debug; use proc_macro::bridge::client::ProcMacro; -use syntax::ext::proc_macro::{AttrProcMacro, ProcMacroDerive, BangProcMacro}; +use syntax_expand::proc_macro::{AttrProcMacro, ProcMacroDerive, BangProcMacro}; crate struct DecodeContext<'a, 'tcx> { opaque: opaque::Decoder<'a>, diff --git a/src/librustc_metadata/encoder.rs b/src/librustc_metadata/encoder.rs index 03a14f88645..ac20284233a 100644 --- a/src/librustc_metadata/encoder.rs +++ b/src/librustc_metadata/encoder.rs @@ -31,7 +31,7 @@ use rustc_data_structures::sync::Lrc; use std::u32; use syntax::ast; use syntax::attr; -use syntax::ext::proc_macro::is_proc_macro_attr; +use syntax_expand::proc_macro::is_proc_macro_attr; use syntax::source_map::Spanned; use syntax::symbol::{kw, sym, Ident, Symbol}; use syntax_pos::{self, FileName, SourceFile, Span}; diff --git a/src/librustc_passes/Cargo.toml b/src/librustc_passes/Cargo.toml index 9d29a230314..118deb560d6 100644 --- a/src/librustc_passes/Cargo.toml +++ b/src/librustc_passes/Cargo.toml @@ -13,6 +13,7 @@ log = "0.4" rustc = { path = "../librustc" } rustc_data_structures = { path = "../librustc_data_structures" } syntax = { path = "../libsyntax" } +syntax_expand = { path = "../libsyntax_expand" } syntax_pos = { path = "../libsyntax_pos" } errors = { path = "../librustc_errors", package = "rustc_errors" } rustc_target = { path = "../librustc_target" } diff --git a/src/librustc_passes/ast_validation.rs b/src/librustc_passes/ast_validation.rs index 43c4f720ad4..74de31263d3 100644 --- a/src/librustc_passes/ast_validation.rs +++ b/src/librustc_passes/ast_validation.rs @@ -14,7 +14,7 @@ use rustc::session::Session; use rustc_data_structures::fx::FxHashMap; use syntax::ast::*; use syntax::attr; -use syntax::ext::proc_macro::is_proc_macro_attr; +use syntax_expand::proc_macro::is_proc_macro_attr; use syntax::feature_gate::is_builtin_attr; use syntax::source_map::Spanned; use syntax::symbol::{kw, sym}; diff --git a/src/librustc_plugin/Cargo.toml b/src/librustc_plugin/Cargo.toml index 3f11430dc82..e8bf4e7ea8f 100644 --- a/src/librustc_plugin/Cargo.toml +++ b/src/librustc_plugin/Cargo.toml @@ -14,4 +14,5 @@ doctest = false rustc = { path = "../librustc" } rustc_metadata = { path = "../librustc_metadata" } syntax = { path = "../libsyntax" } +syntax_expand = { path = "../libsyntax_expand" } syntax_pos = { path = "../libsyntax_pos" } diff --git a/src/librustc_plugin/lib.rs b/src/librustc_plugin/lib.rs index 4e1a47c503e..38738e20630 100644 --- a/src/librustc_plugin/lib.rs +++ b/src/librustc_plugin/lib.rs @@ -21,7 +21,7 @@ //! extern crate syntax_pos; //! //! use rustc_driver::plugin::Registry; -//! use syntax::ext::base::{ExtCtxt, MacResult}; +//! use syntax_expand::base::{ExtCtxt, MacResult}; //! use syntax_pos::Span; //! use syntax::tokenstream::TokenTree; //! diff --git a/src/librustc_plugin/registry.rs b/src/librustc_plugin/registry.rs index bb3c950edae..b826dd91198 100644 --- a/src/librustc_plugin/registry.rs +++ b/src/librustc_plugin/registry.rs @@ -4,8 +4,8 @@ use rustc::lint::{EarlyLintPassObject, LateLintPassObject, LintId, Lint}; use rustc::session::Session; use rustc::util::nodemap::FxHashMap; -use syntax::ext::base::{SyntaxExtension, SyntaxExtensionKind, NamedSyntaxExtension}; -use syntax::ext::base::MacroExpanderFn; +use syntax_expand::base::{SyntaxExtension, SyntaxExtensionKind, NamedSyntaxExtension}; +use syntax_expand::base::MacroExpanderFn; use syntax::symbol::Symbol; use syntax::ast; use syntax::feature_gate::AttributeType; diff --git a/src/librustc_resolve/Cargo.toml b/src/librustc_resolve/Cargo.toml index 936e72ef2c5..06bf3085989 100644 --- a/src/librustc_resolve/Cargo.toml +++ b/src/librustc_resolve/Cargo.toml @@ -14,6 +14,7 @@ doctest = false bitflags = "1.0" log = "0.4" syntax = { path = "../libsyntax" } +syntax_expand = { path = "../libsyntax_expand" } rustc = { path = "../librustc" } arena = { path = "../libarena" } errors = { path = "../librustc_errors", package = "rustc_errors" } diff --git a/src/librustc_resolve/build_reduced_graph.rs b/src/librustc_resolve/build_reduced_graph.rs index 0d5361fe8d8..eadae52c250 100644 --- a/src/librustc_resolve/build_reduced_graph.rs +++ b/src/librustc_resolve/build_reduced_graph.rs @@ -32,9 +32,9 @@ use syntax::attr; use syntax::ast::{self, Block, ForeignItem, ForeignItemKind, Item, ItemKind, NodeId}; use syntax::ast::{MetaItemKind, StmtKind, TraitItem, TraitItemKind}; -use syntax::ext::base::{MacroKind, SyntaxExtension}; -use syntax::ext::expand::AstFragment; -use syntax::ext::hygiene::ExpnId; +use syntax_expand::base::{MacroKind, SyntaxExtension}; +use syntax_expand::expand::AstFragment; +use syntax_expand::hygiene::ExpnId; use syntax::feature_gate::is_builtin_attr; use syntax::parse::token::{self, Token}; use syntax::print::pprust; diff --git a/src/librustc_resolve/diagnostics.rs b/src/librustc_resolve/diagnostics.rs index de875808670..5647d5b2794 100644 --- a/src/librustc_resolve/diagnostics.rs +++ b/src/librustc_resolve/diagnostics.rs @@ -10,7 +10,7 @@ use rustc::session::Session; use rustc::ty::{self, DefIdTree}; use rustc::util::nodemap::FxHashSet; use syntax::ast::{self, Ident, Path}; -use syntax::ext::base::MacroKind; +use syntax_expand::base::MacroKind; use syntax::feature_gate::BUILTIN_ATTRIBUTES; use syntax::source_map::SourceMap; use syntax::struct_span_err; diff --git a/src/librustc_resolve/late/diagnostics.rs b/src/librustc_resolve/late/diagnostics.rs index ace9903a835..412734eabe0 100644 --- a/src/librustc_resolve/late/diagnostics.rs +++ b/src/librustc_resolve/late/diagnostics.rs @@ -13,7 +13,7 @@ use rustc::hir::PrimTy; use rustc::session::config::nightly_options; use rustc::util::nodemap::FxHashSet; use syntax::ast::{self, Expr, ExprKind, Ident, NodeId, Path, Ty, TyKind}; -use syntax::ext::base::MacroKind; +use syntax_expand::base::MacroKind; use syntax::symbol::kw; use syntax::util::lev_distance::find_best_match_for_name; use syntax_pos::Span; diff --git a/src/librustc_resolve/lib.rs b/src/librustc_resolve/lib.rs index e5b0ef89a41..17d8f0f211a 100644 --- a/src/librustc_resolve/lib.rs +++ b/src/librustc_resolve/lib.rs @@ -35,18 +35,16 @@ use rustc::span_bug; use rustc_metadata::creader::CrateLoader; use rustc_metadata::cstore::CStore; -use syntax::ext::hygiene::{ExpnId, Transparency, SyntaxContext}; +use syntax_expand::hygiene::{ExpnId, Transparency, SyntaxContext}; +use syntax_expand::base::{SyntaxExtension, MacroKind, SpecialDerives}; +use syntax::{struct_span_err, unwrap_or}; +use syntax::attr; use syntax::ast::{self, Name, NodeId, Ident, FloatTy, IntTy, UintTy}; -use syntax::ext::base::{SyntaxExtension, MacroKind, SpecialDerives}; +use syntax::ast::{ItemKind, Path, CRATE_NODE_ID, Crate}; use syntax::print::pprust; use syntax::symbol::{kw, sym}; - -use syntax::visit::{self, Visitor}; -use syntax::attr; -use syntax::ast::{CRATE_NODE_ID, Crate}; -use syntax::ast::{ItemKind, Path}; -use syntax::{struct_span_err, unwrap_or}; use syntax::source_map::Spanned; +use syntax::visit::{self, Visitor}; use syntax_pos::{Span, DUMMY_SP}; use errors::{Applicability, DiagnosticBuilder}; diff --git a/src/librustc_resolve/macros.rs b/src/librustc_resolve/macros.rs index 49186088fa0..c91a0b2ed98 100644 --- a/src/librustc_resolve/macros.rs +++ b/src/librustc_resolve/macros.rs @@ -14,11 +14,11 @@ use rustc::{ty, lint, span_bug}; use syntax::ast::{self, NodeId, Ident}; use syntax::attr::StabilityLevel; use syntax::edition::Edition; -use syntax::ext::base::{self, InvocationRes, Indeterminate, SpecialDerives}; -use syntax::ext::base::{MacroKind, SyntaxExtension}; -use syntax::ext::expand::{AstFragment, AstFragmentKind, Invocation, InvocationKind}; -use syntax::ext::hygiene::{self, ExpnId, ExpnData, ExpnKind}; -use syntax::ext::compile_declarative_macro; +use syntax_expand::base::{self, InvocationRes, Indeterminate, SpecialDerives}; +use syntax_expand::base::{MacroKind, SyntaxExtension}; +use syntax_expand::expand::{AstFragment, AstFragmentKind, Invocation, InvocationKind}; +use syntax_expand::hygiene::{self, ExpnId, ExpnData, ExpnKind}; +use syntax_expand::compile_declarative_macro; use syntax::feature_gate::{emit_feature_err, is_builtin_attr_name}; use syntax::feature_gate::GateIssue; use syntax::print::pprust; diff --git a/src/librustc_resolve/resolve_imports.rs b/src/librustc_resolve/resolve_imports.rs index 56fd2da2576..424bf31a785 100644 --- a/src/librustc_resolve/resolve_imports.rs +++ b/src/librustc_resolve/resolve_imports.rs @@ -28,7 +28,7 @@ use rustc::util::nodemap::FxHashSet; use rustc::{bug, span_bug}; use syntax::ast::{Ident, Name, NodeId, CRATE_NODE_ID}; -use syntax::ext::hygiene::ExpnId; +use syntax_expand::hygiene::ExpnId; use syntax::symbol::kw; use syntax::util::lev_distance::find_best_match_for_name; use syntax::{struct_span_err, unwrap_or}; diff --git a/src/librustdoc/clean/inline.rs b/src/librustdoc/clean/inline.rs index 532c5f67bf3..63350b2a5d2 100644 --- a/src/librustdoc/clean/inline.rs +++ b/src/librustdoc/clean/inline.rs @@ -3,7 +3,7 @@ use std::iter::once; use syntax::ast; -use syntax::ext::base::MacroKind; +use syntax_expand::base::MacroKind; use syntax::symbol::sym; use syntax_pos::Span; diff --git a/src/librustdoc/clean/mod.rs b/src/librustdoc/clean/mod.rs index 212a09ee6e6..ba00631dc6c 100644 --- a/src/librustdoc/clean/mod.rs +++ b/src/librustdoc/clean/mod.rs @@ -28,7 +28,7 @@ use rustc::ty::layout::VariantIdx; use rustc::util::nodemap::{FxHashMap, FxHashSet}; use syntax::ast::{self, AttrStyle, Ident}; use syntax::attr; -use syntax::ext::base::MacroKind; +use syntax_expand::base::MacroKind; use syntax::source_map::DUMMY_SP; use syntax::symbol::{Symbol, kw, sym}; use syntax::symbol::InternedString; diff --git a/src/librustdoc/doctree.rs b/src/librustdoc/doctree.rs index 6e453561f6d..0dc094ae329 100644 --- a/src/librustdoc/doctree.rs +++ b/src/librustdoc/doctree.rs @@ -4,7 +4,7 @@ pub use self::StructType::*; use syntax::ast; use syntax::ast::Name; -use syntax::ext::base::MacroKind; +use syntax_expand::base::MacroKind; use syntax_pos::{self, Span}; use rustc::hir; diff --git a/src/librustdoc/html/item_type.rs b/src/librustdoc/html/item_type.rs index 5fb9afd6c49..e015739b03c 100644 --- a/src/librustdoc/html/item_type.rs +++ b/src/librustdoc/html/item_type.rs @@ -1,7 +1,7 @@ //! Item types. use std::fmt; -use syntax::ext::base::MacroKind; +use syntax_expand::base::MacroKind; use crate::clean; /// Item type. Corresponds to `clean::ItemEnum` variants. diff --git a/src/librustdoc/html/render.rs b/src/librustdoc/html/render.rs index b726ad1e0d6..72a72e89281 100644 --- a/src/librustdoc/html/render.rs +++ b/src/librustdoc/html/render.rs @@ -45,11 +45,11 @@ use errors; use serialize::json::{ToJson, Json, as_json}; use syntax::ast; use syntax::edition::Edition; -use syntax::ext::base::MacroKind; +use syntax::feature_gate::UnstableFeatures; use syntax::print::pprust; use syntax::source_map::FileName; -use syntax::feature_gate::UnstableFeatures; use syntax::symbol::{Symbol, sym}; +use syntax_expand::base::MacroKind; use rustc::hir::def_id::DefId; use rustc::middle::privacy::AccessLevels; use rustc::middle::stability; diff --git a/src/librustdoc/lib.rs b/src/librustdoc/lib.rs index 6bcb4a817d7..8cd32a3d1b5 100644 --- a/src/librustdoc/lib.rs +++ b/src/librustdoc/lib.rs @@ -34,6 +34,7 @@ extern crate rustc_typeck; extern crate rustc_lexer; extern crate serialize; extern crate syntax; +extern crate syntax_expand; extern crate syntax_pos; extern crate test as testing; #[macro_use] extern crate log; diff --git a/src/librustdoc/passes/collect_intra_doc_links.rs b/src/librustdoc/passes/collect_intra_doc_links.rs index 9186ed51420..4270b162baf 100644 --- a/src/librustdoc/passes/collect_intra_doc_links.rs +++ b/src/librustdoc/passes/collect_intra_doc_links.rs @@ -7,7 +7,7 @@ use rustc::ty; use rustc_resolve::ParentScope; use syntax; use syntax::ast::{self, Ident}; -use syntax::ext::base::SyntaxExtensionKind; +use syntax_expand::base::SyntaxExtensionKind; use syntax::feature_gate::UnstableFeatures; use syntax::symbol::Symbol; use syntax_pos::DUMMY_SP; diff --git a/src/librustdoc/visit_ast.rs b/src/librustdoc/visit_ast.rs index b6a90e1fb98..70c30687dab 100644 --- a/src/librustdoc/visit_ast.rs +++ b/src/librustdoc/visit_ast.rs @@ -8,7 +8,7 @@ use rustc::middle::privacy::AccessLevel; use rustc::util::nodemap::{FxHashSet, FxHashMap}; use rustc::ty::TyCtxt; use syntax::ast; -use syntax::ext::base::MacroKind; +use syntax_expand::base::MacroKind; use syntax::source_map::Spanned; use syntax::symbol::sym; use syntax_pos::{self, Span}; diff --git a/src/libsyntax/attr/mod.rs b/src/libsyntax/attr/mod.rs index 65257c7e985..402c2cad72f 100644 --- a/src/libsyntax/attr/mod.rs +++ b/src/libsyntax/attr/mod.rs @@ -298,14 +298,14 @@ impl Attribute { Ok(result) } - crate fn parse_derive_paths<'a>(&self, sess: &'a ParseSess) -> PResult<'a, Vec> { + pub fn parse_derive_paths<'a>(&self, sess: &'a ParseSess) -> PResult<'a, Vec> { if self.tokens.is_empty() { return Ok(Vec::new()); } self.parse(sess, |p| p.parse_derive_paths()) } - crate fn parse_meta<'a>(&self, sess: &'a ParseSess) -> PResult<'a, MetaItem> { + pub fn parse_meta<'a>(&self, sess: &'a ParseSess) -> PResult<'a, MetaItem> { Ok(MetaItem { path: self.path.clone(), kind: self.parse(sess, |parser| parser.parse_meta_item_kind())?, diff --git a/src/libsyntax/config.rs b/src/libsyntax/config.rs index ee5a4eeb3a3..54dc95291d6 100644 --- a/src/libsyntax/config.rs +++ b/src/libsyntax/config.rs @@ -56,6 +56,7 @@ pub fn features(mut krate: ast::Crate, sess: &ParseSess, edition: Edition, (krate, features) } +#[macro_export] macro_rules! configure { ($this:ident, $node:ident) => { match $this.configure($node) { diff --git a/src/libsyntax/ext/allocator.rs b/src/libsyntax/ext/allocator.rs deleted file mode 100644 index 99aeb5414c5..00000000000 --- a/src/libsyntax/ext/allocator.rs +++ /dev/null @@ -1,75 +0,0 @@ -use crate::{ast, attr, visit}; -use crate::symbol::{sym, Symbol}; -use syntax_pos::Span; - -#[derive(Clone, Copy)] -pub enum AllocatorKind { - Global, - DefaultLib, - DefaultExe, -} - -impl AllocatorKind { - pub fn fn_name(&self, base: &str) -> String { - match *self { - AllocatorKind::Global => format!("__rg_{}", base), - AllocatorKind::DefaultLib => format!("__rdl_{}", base), - AllocatorKind::DefaultExe => format!("__rde_{}", base), - } - } -} - -pub enum AllocatorTy { - Layout, - Ptr, - ResultPtr, - Unit, - Usize, -} - -pub struct AllocatorMethod { - pub name: &'static str, - pub inputs: &'static [AllocatorTy], - pub output: AllocatorTy, -} - -pub static ALLOCATOR_METHODS: &[AllocatorMethod] = &[ - AllocatorMethod { - name: "alloc", - inputs: &[AllocatorTy::Layout], - output: AllocatorTy::ResultPtr, - }, - AllocatorMethod { - name: "dealloc", - inputs: &[AllocatorTy::Ptr, AllocatorTy::Layout], - output: AllocatorTy::Unit, - }, - AllocatorMethod { - name: "realloc", - inputs: &[AllocatorTy::Ptr, AllocatorTy::Layout, AllocatorTy::Usize], - output: AllocatorTy::ResultPtr, - }, - AllocatorMethod { - name: "alloc_zeroed", - inputs: &[AllocatorTy::Layout], - output: AllocatorTy::ResultPtr, - }, -]; - -pub fn global_allocator_spans(krate: &ast::Crate) -> Vec { - struct Finder { name: Symbol, spans: Vec } - impl<'ast> visit::Visitor<'ast> for Finder { - fn visit_item(&mut self, item: &'ast ast::Item) { - if item.ident.name == self.name && - attr::contains_name(&item.attrs, sym::rustc_std_internal_symbol) { - self.spans.push(item.span); - } - visit::walk_item(self, item) - } - } - - let name = Symbol::intern(&AllocatorKind::Global.fn_name("alloc")); - let mut f = Finder { name, spans: Vec::new() }; - visit::walk_crate(&mut f, krate); - f.spans -} diff --git a/src/libsyntax/ext/base.rs b/src/libsyntax/ext/base.rs deleted file mode 100644 index 01be5642d5c..00000000000 --- a/src/libsyntax/ext/base.rs +++ /dev/null @@ -1,1188 +0,0 @@ -use crate::ast::{self, NodeId, Attribute, Name, PatKind}; -use crate::attr::{self, HasAttrs, Stability, Deprecation}; -use crate::source_map::SourceMap; -use crate::edition::Edition; -use crate::ext::expand::{self, AstFragment, Invocation}; -use crate::ext::hygiene::ExpnId; -use crate::mut_visit::{self, MutVisitor}; -use crate::parse::{self, parser, DirectoryOwnership}; -use crate::parse::token; -use crate::ptr::P; -use crate::sess::ParseSess; -use crate::symbol::{kw, sym, Ident, Symbol}; -use crate::{ThinVec, MACRO_ARGUMENTS}; -use crate::tokenstream::{self, TokenStream}; -use crate::visit::Visitor; - -use errors::{DiagnosticBuilder, DiagnosticId}; -use smallvec::{smallvec, SmallVec}; -use syntax_pos::{FileName, Span, MultiSpan, DUMMY_SP}; -use syntax_pos::hygiene::{AstPass, ExpnData, ExpnKind}; - -use rustc_data_structures::fx::FxHashMap; -use rustc_data_structures::sync::{self, Lrc}; -use std::iter; -use std::path::PathBuf; -use std::rc::Rc; -use std::default::Default; - -pub use syntax_pos::hygiene::MacroKind; - -#[derive(Debug,Clone)] -pub enum Annotatable { - Item(P), - TraitItem(P), - ImplItem(P), - ForeignItem(P), - Stmt(P), - Expr(P), - Arm(ast::Arm), - Field(ast::Field), - FieldPat(ast::FieldPat), - GenericParam(ast::GenericParam), - Param(ast::Param), - StructField(ast::StructField), - Variant(ast::Variant), -} - -impl HasAttrs for Annotatable { - fn attrs(&self) -> &[Attribute] { - match *self { - Annotatable::Item(ref item) => &item.attrs, - Annotatable::TraitItem(ref trait_item) => &trait_item.attrs, - Annotatable::ImplItem(ref impl_item) => &impl_item.attrs, - Annotatable::ForeignItem(ref foreign_item) => &foreign_item.attrs, - Annotatable::Stmt(ref stmt) => stmt.attrs(), - Annotatable::Expr(ref expr) => &expr.attrs, - Annotatable::Arm(ref arm) => &arm.attrs, - Annotatable::Field(ref field) => &field.attrs, - Annotatable::FieldPat(ref fp) => &fp.attrs, - Annotatable::GenericParam(ref gp) => &gp.attrs, - Annotatable::Param(ref p) => &p.attrs, - Annotatable::StructField(ref sf) => &sf.attrs, - Annotatable::Variant(ref v) => &v.attrs(), - } - } - - fn visit_attrs)>(&mut self, f: F) { - match self { - Annotatable::Item(item) => item.visit_attrs(f), - Annotatable::TraitItem(trait_item) => trait_item.visit_attrs(f), - Annotatable::ImplItem(impl_item) => impl_item.visit_attrs(f), - Annotatable::ForeignItem(foreign_item) => foreign_item.visit_attrs(f), - Annotatable::Stmt(stmt) => stmt.visit_attrs(f), - Annotatable::Expr(expr) => expr.visit_attrs(f), - Annotatable::Arm(arm) => arm.visit_attrs(f), - Annotatable::Field(field) => field.visit_attrs(f), - Annotatable::FieldPat(fp) => fp.visit_attrs(f), - Annotatable::GenericParam(gp) => gp.visit_attrs(f), - Annotatable::Param(p) => p.visit_attrs(f), - Annotatable::StructField(sf) => sf.visit_attrs(f), - Annotatable::Variant(v) => v.visit_attrs(f), - } - } -} - -impl Annotatable { - pub fn span(&self) -> Span { - match *self { - Annotatable::Item(ref item) => item.span, - Annotatable::TraitItem(ref trait_item) => trait_item.span, - Annotatable::ImplItem(ref impl_item) => impl_item.span, - Annotatable::ForeignItem(ref foreign_item) => foreign_item.span, - Annotatable::Stmt(ref stmt) => stmt.span, - Annotatable::Expr(ref expr) => expr.span, - Annotatable::Arm(ref arm) => arm.span, - Annotatable::Field(ref field) => field.span, - Annotatable::FieldPat(ref fp) => fp.pat.span, - Annotatable::GenericParam(ref gp) => gp.ident.span, - Annotatable::Param(ref p) => p.span, - Annotatable::StructField(ref sf) => sf.span, - Annotatable::Variant(ref v) => v.span, - } - } - - pub fn visit_with<'a, V: Visitor<'a>>(&'a self, visitor: &mut V) { - match self { - Annotatable::Item(item) => visitor.visit_item(item), - Annotatable::TraitItem(trait_item) => visitor.visit_trait_item(trait_item), - Annotatable::ImplItem(impl_item) => visitor.visit_impl_item(impl_item), - Annotatable::ForeignItem(foreign_item) => visitor.visit_foreign_item(foreign_item), - Annotatable::Stmt(stmt) => visitor.visit_stmt(stmt), - Annotatable::Expr(expr) => visitor.visit_expr(expr), - Annotatable::Arm(arm) => visitor.visit_arm(arm), - Annotatable::Field(field) => visitor.visit_field(field), - Annotatable::FieldPat(fp) => visitor.visit_field_pattern(fp), - Annotatable::GenericParam(gp) => visitor.visit_generic_param(gp), - Annotatable::Param(p) => visitor.visit_param(p), - Annotatable::StructField(sf) =>visitor.visit_struct_field(sf), - Annotatable::Variant(v) => visitor.visit_variant(v), - } - } - - pub fn expect_item(self) -> P { - match self { - Annotatable::Item(i) => i, - _ => panic!("expected Item") - } - } - - pub fn map_item_or(self, mut f: F, mut or: G) -> Annotatable - where F: FnMut(P) -> P, - G: FnMut(Annotatable) -> Annotatable - { - match self { - Annotatable::Item(i) => Annotatable::Item(f(i)), - _ => or(self) - } - } - - pub fn expect_trait_item(self) -> ast::TraitItem { - match self { - Annotatable::TraitItem(i) => i.into_inner(), - _ => panic!("expected Item") - } - } - - pub fn expect_impl_item(self) -> ast::ImplItem { - match self { - Annotatable::ImplItem(i) => i.into_inner(), - _ => panic!("expected Item") - } - } - - pub fn expect_foreign_item(self) -> ast::ForeignItem { - match self { - Annotatable::ForeignItem(i) => i.into_inner(), - _ => panic!("expected foreign item") - } - } - - pub fn expect_stmt(self) -> ast::Stmt { - match self { - Annotatable::Stmt(stmt) => stmt.into_inner(), - _ => panic!("expected statement"), - } - } - - pub fn expect_expr(self) -> P { - match self { - Annotatable::Expr(expr) => expr, - _ => panic!("expected expression"), - } - } - - pub fn expect_arm(self) -> ast::Arm { - match self { - Annotatable::Arm(arm) => arm, - _ => panic!("expected match arm") - } - } - - pub fn expect_field(self) -> ast::Field { - match self { - Annotatable::Field(field) => field, - _ => panic!("expected field") - } - } - - pub fn expect_field_pattern(self) -> ast::FieldPat { - match self { - Annotatable::FieldPat(fp) => fp, - _ => panic!("expected field pattern") - } - } - - pub fn expect_generic_param(self) -> ast::GenericParam { - match self { - Annotatable::GenericParam(gp) => gp, - _ => panic!("expected generic parameter") - } - } - - pub fn expect_param(self) -> ast::Param { - match self { - Annotatable::Param(param) => param, - _ => panic!("expected parameter") - } - } - - pub fn expect_struct_field(self) -> ast::StructField { - match self { - Annotatable::StructField(sf) => sf, - _ => panic!("expected struct field") - } - } - - pub fn expect_variant(self) -> ast::Variant { - match self { - Annotatable::Variant(v) => v, - _ => panic!("expected variant") - } - } - - pub fn derive_allowed(&self) -> bool { - match *self { - Annotatable::Item(ref item) => match item.kind { - ast::ItemKind::Struct(..) | - ast::ItemKind::Enum(..) | - ast::ItemKind::Union(..) => true, - _ => false, - }, - _ => false, - } - } -} - -// `meta_item` is the annotation, and `item` is the item being modified. -// FIXME Decorators should follow the same pattern too. -pub trait MultiItemModifier { - fn expand(&self, - ecx: &mut ExtCtxt<'_>, - span: Span, - meta_item: &ast::MetaItem, - item: Annotatable) - -> Vec; -} - -impl MultiItemModifier for F - where F: Fn(&mut ExtCtxt<'_>, Span, &ast::MetaItem, Annotatable) -> T, - T: Into>, -{ - fn expand(&self, - ecx: &mut ExtCtxt<'_>, - span: Span, - meta_item: &ast::MetaItem, - item: Annotatable) - -> Vec { - (*self)(ecx, span, meta_item, item).into() - } -} - -impl Into> for Annotatable { - fn into(self) -> Vec { - vec![self] - } -} - -pub trait ProcMacro { - fn expand<'cx>(&self, - ecx: &'cx mut ExtCtxt<'_>, - span: Span, - ts: TokenStream) - -> TokenStream; -} - -impl ProcMacro for F - where F: Fn(TokenStream) -> TokenStream -{ - fn expand<'cx>(&self, - _ecx: &'cx mut ExtCtxt<'_>, - _span: Span, - ts: TokenStream) - -> TokenStream { - // FIXME setup implicit context in TLS before calling self. - (*self)(ts) - } -} - -pub trait AttrProcMacro { - fn expand<'cx>(&self, - ecx: &'cx mut ExtCtxt<'_>, - span: Span, - annotation: TokenStream, - annotated: TokenStream) - -> TokenStream; -} - -impl AttrProcMacro for F - where F: Fn(TokenStream, TokenStream) -> TokenStream -{ - fn expand<'cx>(&self, - _ecx: &'cx mut ExtCtxt<'_>, - _span: Span, - annotation: TokenStream, - annotated: TokenStream) - -> TokenStream { - // FIXME setup implicit context in TLS before calling self. - (*self)(annotation, annotated) - } -} - -/// Represents a thing that maps token trees to Macro Results -pub trait TTMacroExpander { - fn expand<'cx>( - &self, - ecx: &'cx mut ExtCtxt<'_>, - span: Span, - input: TokenStream, - ) -> Box; -} - -pub type MacroExpanderFn = - for<'cx> fn(&'cx mut ExtCtxt<'_>, Span, TokenStream) - -> Box; - -impl TTMacroExpander for F - where F: for<'cx> Fn(&'cx mut ExtCtxt<'_>, Span, TokenStream) - -> Box -{ - fn expand<'cx>( - &self, - ecx: &'cx mut ExtCtxt<'_>, - span: Span, - mut input: TokenStream, - ) -> Box { - struct AvoidInterpolatedIdents; - - impl MutVisitor for AvoidInterpolatedIdents { - fn visit_tt(&mut self, tt: &mut tokenstream::TokenTree) { - if let tokenstream::TokenTree::Token(token) = tt { - if let token::Interpolated(nt) = &token.kind { - if let token::NtIdent(ident, is_raw) = **nt { - *tt = tokenstream::TokenTree::token( - token::Ident(ident.name, is_raw), ident.span - ); - } - } - } - mut_visit::noop_visit_tt(tt, self) - } - - fn visit_mac(&mut self, mac: &mut ast::Mac) { - mut_visit::noop_visit_mac(mac, self) - } - } - AvoidInterpolatedIdents.visit_tts(&mut input); - (*self)(ecx, span, input) - } -} - -// Use a macro because forwarding to a simple function has type system issues -macro_rules! make_stmts_default { - ($me:expr) => { - $me.make_expr().map(|e| smallvec![ast::Stmt { - id: ast::DUMMY_NODE_ID, - span: e.span, - kind: ast::StmtKind::Expr(e), - }]) - } -} - -/// The result of a macro expansion. The return values of the various -/// methods are spliced into the AST at the callsite of the macro. -pub trait MacResult { - /// Creates an expression. - fn make_expr(self: Box) -> Option> { - None - } - /// Creates zero or more items. - fn make_items(self: Box) -> Option; 1]>> { - None - } - - /// Creates zero or more impl items. - fn make_impl_items(self: Box) -> Option> { - None - } - - /// Creates zero or more trait items. - fn make_trait_items(self: Box) -> Option> { - None - } - - /// Creates zero or more items in an `extern {}` block - fn make_foreign_items(self: Box) -> Option> { None } - - /// Creates a pattern. - fn make_pat(self: Box) -> Option> { - None - } - - /// Creates zero or more statements. - /// - /// By default this attempts to create an expression statement, - /// returning None if that fails. - fn make_stmts(self: Box) -> Option> { - make_stmts_default!(self) - } - - fn make_ty(self: Box) -> Option> { - None - } - - fn make_arms(self: Box) -> Option> { - None - } - - fn make_fields(self: Box) -> Option> { - None - } - - fn make_field_patterns(self: Box) -> Option> { - None - } - - fn make_generic_params(self: Box) -> Option> { - None - } - - fn make_params(self: Box) -> Option> { - None - } - - fn make_struct_fields(self: Box) -> Option> { - None - } - - fn make_variants(self: Box) -> Option> { - None - } -} - -macro_rules! make_MacEager { - ( $( $fld:ident: $t:ty, )* ) => { - /// `MacResult` implementation for the common case where you've already - /// built each form of AST that you might return. - #[derive(Default)] - pub struct MacEager { - $( - pub $fld: Option<$t>, - )* - } - - impl MacEager { - $( - pub fn $fld(v: $t) -> Box { - Box::new(MacEager { - $fld: Some(v), - ..Default::default() - }) - } - )* - } - } -} - -make_MacEager! { - expr: P, - pat: P, - items: SmallVec<[P; 1]>, - impl_items: SmallVec<[ast::ImplItem; 1]>, - trait_items: SmallVec<[ast::TraitItem; 1]>, - foreign_items: SmallVec<[ast::ForeignItem; 1]>, - stmts: SmallVec<[ast::Stmt; 1]>, - ty: P, -} - -impl MacResult for MacEager { - fn make_expr(self: Box) -> Option> { - self.expr - } - - fn make_items(self: Box) -> Option; 1]>> { - self.items - } - - fn make_impl_items(self: Box) -> Option> { - self.impl_items - } - - fn make_trait_items(self: Box) -> Option> { - self.trait_items - } - - fn make_foreign_items(self: Box) -> Option> { - self.foreign_items - } - - fn make_stmts(self: Box) -> Option> { - match self.stmts.as_ref().map_or(0, |s| s.len()) { - 0 => make_stmts_default!(self), - _ => self.stmts, - } - } - - fn make_pat(self: Box) -> Option> { - if let Some(p) = self.pat { - return Some(p); - } - if let Some(e) = self.expr { - if let ast::ExprKind::Lit(_) = e.kind { - return Some(P(ast::Pat { - id: ast::DUMMY_NODE_ID, - span: e.span, - kind: PatKind::Lit(e), - })); - } - } - None - } - - fn make_ty(self: Box) -> Option> { - self.ty - } -} - -/// Fill-in macro expansion result, to allow compilation to continue -/// after hitting errors. -#[derive(Copy, Clone)] -pub struct DummyResult { - is_error: bool, - span: Span, -} - -impl DummyResult { - /// Creates a default MacResult that can be anything. - /// - /// Use this as a return value after hitting any errors and - /// calling `span_err`. - pub fn any(span: Span) -> Box { - Box::new(DummyResult { is_error: true, span }) - } - - /// Same as `any`, but must be a valid fragment, not error. - pub fn any_valid(span: Span) -> Box { - Box::new(DummyResult { is_error: false, span }) - } - - /// A plain dummy expression. - pub fn raw_expr(sp: Span, is_error: bool) -> P { - P(ast::Expr { - id: ast::DUMMY_NODE_ID, - kind: if is_error { ast::ExprKind::Err } else { ast::ExprKind::Tup(Vec::new()) }, - span: sp, - attrs: ThinVec::new(), - }) - } - - /// A plain dummy pattern. - pub fn raw_pat(sp: Span) -> ast::Pat { - ast::Pat { - id: ast::DUMMY_NODE_ID, - kind: PatKind::Wild, - span: sp, - } - } - - /// A plain dummy type. - pub fn raw_ty(sp: Span, is_error: bool) -> P { - P(ast::Ty { - id: ast::DUMMY_NODE_ID, - kind: if is_error { ast::TyKind::Err } else { ast::TyKind::Tup(Vec::new()) }, - span: sp - }) - } -} - -impl MacResult for DummyResult { - fn make_expr(self: Box) -> Option> { - Some(DummyResult::raw_expr(self.span, self.is_error)) - } - - fn make_pat(self: Box) -> Option> { - Some(P(DummyResult::raw_pat(self.span))) - } - - fn make_items(self: Box) -> Option; 1]>> { - Some(SmallVec::new()) - } - - fn make_impl_items(self: Box) -> Option> { - Some(SmallVec::new()) - } - - fn make_trait_items(self: Box) -> Option> { - Some(SmallVec::new()) - } - - fn make_foreign_items(self: Box) -> Option> { - Some(SmallVec::new()) - } - - fn make_stmts(self: Box) -> Option> { - Some(smallvec![ast::Stmt { - id: ast::DUMMY_NODE_ID, - kind: ast::StmtKind::Expr(DummyResult::raw_expr(self.span, self.is_error)), - span: self.span, - }]) - } - - fn make_ty(self: Box) -> Option> { - Some(DummyResult::raw_ty(self.span, self.is_error)) - } - - fn make_arms(self: Box) -> Option> { - Some(SmallVec::new()) - } - - fn make_fields(self: Box) -> Option> { - Some(SmallVec::new()) - } - - fn make_field_patterns(self: Box) -> Option> { - Some(SmallVec::new()) - } - - fn make_generic_params(self: Box) -> Option> { - Some(SmallVec::new()) - } - - fn make_params(self: Box) -> Option> { - Some(SmallVec::new()) - } - - fn make_struct_fields(self: Box) -> Option> { - Some(SmallVec::new()) - } - - fn make_variants(self: Box) -> Option> { - Some(SmallVec::new()) - } -} - -/// A syntax extension kind. -pub enum SyntaxExtensionKind { - /// A token-based function-like macro. - Bang( - /// An expander with signature TokenStream -> TokenStream. - Box, - ), - - /// An AST-based function-like macro. - LegacyBang( - /// An expander with signature TokenStream -> AST. - Box, - ), - - /// A token-based attribute macro. - Attr( - /// An expander with signature (TokenStream, TokenStream) -> TokenStream. - /// The first TokenSteam is the attribute itself, the second is the annotated item. - /// The produced TokenSteam replaces the input TokenSteam. - Box, - ), - - /// An AST-based attribute macro. - LegacyAttr( - /// An expander with signature (AST, AST) -> AST. - /// The first AST fragment is the attribute itself, the second is the annotated item. - /// The produced AST fragment replaces the input AST fragment. - Box, - ), - - /// A trivial attribute "macro" that does nothing, - /// only keeps the attribute and marks it as inert, - /// thus making it ineligible for further expansion. - NonMacroAttr { - /// Suppresses the `unused_attributes` lint for this attribute. - mark_used: bool, - }, - - /// A token-based derive macro. - Derive( - /// An expander with signature TokenStream -> TokenStream (not yet). - /// The produced TokenSteam is appended to the input TokenSteam. - Box, - ), - - /// An AST-based derive macro. - LegacyDerive( - /// An expander with signature AST -> AST. - /// The produced AST fragment is appended to the input AST fragment. - Box, - ), -} - -/// A struct representing a macro definition in "lowered" form ready for expansion. -pub struct SyntaxExtension { - /// A syntax extension kind. - pub kind: SyntaxExtensionKind, - /// Span of the macro definition. - pub span: Span, - /// Whitelist of unstable features that are treated as stable inside this macro. - pub allow_internal_unstable: Option>, - /// Suppresses the `unsafe_code` lint for code produced by this macro. - pub allow_internal_unsafe: bool, - /// Enables the macro helper hack (`ident!(...)` -> `$crate::ident!(...)`) for this macro. - pub local_inner_macros: bool, - /// The macro's stability info. - pub stability: Option, - /// The macro's deprecation info. - pub deprecation: Option, - /// Names of helper attributes registered by this macro. - pub helper_attrs: Vec, - /// Edition of the crate in which this macro is defined. - pub edition: Edition, - /// Built-in macros have a couple of special properties like availability - /// in `#[no_implicit_prelude]` modules, so we have to keep this flag. - pub is_builtin: bool, - /// We have to identify macros providing a `Copy` impl early for compatibility reasons. - pub is_derive_copy: bool, -} - -impl SyntaxExtension { - /// Returns which kind of macro calls this syntax extension. - pub fn macro_kind(&self) -> MacroKind { - match self.kind { - SyntaxExtensionKind::Bang(..) | - SyntaxExtensionKind::LegacyBang(..) => MacroKind::Bang, - SyntaxExtensionKind::Attr(..) | - SyntaxExtensionKind::LegacyAttr(..) | - SyntaxExtensionKind::NonMacroAttr { .. } => MacroKind::Attr, - SyntaxExtensionKind::Derive(..) | - SyntaxExtensionKind::LegacyDerive(..) => MacroKind::Derive, - } - } - - /// Constructs a syntax extension with default properties. - pub fn default(kind: SyntaxExtensionKind, edition: Edition) -> SyntaxExtension { - SyntaxExtension { - span: DUMMY_SP, - allow_internal_unstable: None, - allow_internal_unsafe: false, - local_inner_macros: false, - stability: None, - deprecation: None, - helper_attrs: Vec::new(), - edition, - is_builtin: false, - is_derive_copy: false, - kind, - } - } - - /// Constructs a syntax extension with the given properties - /// and other properties converted from attributes. - pub fn new( - sess: &ParseSess, - kind: SyntaxExtensionKind, - span: Span, - helper_attrs: Vec, - edition: Edition, - name: Name, - attrs: &[ast::Attribute], - ) -> SyntaxExtension { - let allow_internal_unstable = attr::allow_internal_unstable( - &attrs, &sess.span_diagnostic, - ).map(|features| features.collect::>().into()); - - let mut local_inner_macros = false; - if let Some(macro_export) = attr::find_by_name(attrs, sym::macro_export) { - if let Some(l) = macro_export.meta_item_list() { - local_inner_macros = attr::list_contains_name(&l, sym::local_inner_macros); - } - } - - let is_builtin = attr::contains_name(attrs, sym::rustc_builtin_macro); - - SyntaxExtension { - kind, - span, - allow_internal_unstable, - allow_internal_unsafe: attr::contains_name(attrs, sym::allow_internal_unsafe), - local_inner_macros, - stability: attr::find_stability(&sess, attrs, span), - deprecation: attr::find_deprecation(&sess, attrs, span), - helper_attrs, - edition, - is_builtin, - is_derive_copy: is_builtin && name == sym::Copy, - } - } - - pub fn dummy_bang(edition: Edition) -> SyntaxExtension { - fn expander<'cx>(_: &'cx mut ExtCtxt<'_>, span: Span, _: TokenStream) - -> Box { - DummyResult::any(span) - } - SyntaxExtension::default(SyntaxExtensionKind::LegacyBang(Box::new(expander)), edition) - } - - pub fn dummy_derive(edition: Edition) -> SyntaxExtension { - fn expander(_: &mut ExtCtxt<'_>, _: Span, _: &ast::MetaItem, _: Annotatable) - -> Vec { - Vec::new() - } - SyntaxExtension::default(SyntaxExtensionKind::Derive(Box::new(expander)), edition) - } - - pub fn non_macro_attr(mark_used: bool, edition: Edition) -> SyntaxExtension { - SyntaxExtension::default(SyntaxExtensionKind::NonMacroAttr { mark_used }, edition) - } - - pub fn expn_data(&self, parent: ExpnId, call_site: Span, descr: Symbol) -> ExpnData { - ExpnData { - kind: ExpnKind::Macro(self.macro_kind(), descr), - parent, - call_site, - def_site: self.span, - allow_internal_unstable: self.allow_internal_unstable.clone(), - allow_internal_unsafe: self.allow_internal_unsafe, - local_inner_macros: self.local_inner_macros, - edition: self.edition, - } - } -} - -pub type NamedSyntaxExtension = (Name, SyntaxExtension); - -/// Result of resolving a macro invocation. -pub enum InvocationRes { - Single(Lrc), - DeriveContainer(Vec>), -} - -/// Error type that denotes indeterminacy. -pub struct Indeterminate; - -bitflags::bitflags! { - /// Built-in derives that need some extra tracking beyond the usual macro functionality. - #[derive(Default)] - pub struct SpecialDerives: u8 { - const PARTIAL_EQ = 1 << 0; - const EQ = 1 << 1; - const COPY = 1 << 2; - } -} - -pub trait Resolver { - fn next_node_id(&mut self) -> NodeId; - - fn resolve_dollar_crates(&mut self); - fn visit_ast_fragment_with_placeholders(&mut self, expn_id: ExpnId, fragment: &AstFragment, - extra_placeholders: &[NodeId]); - fn register_builtin_macro(&mut self, ident: ast::Ident, ext: SyntaxExtension); - - fn expansion_for_ast_pass( - &mut self, - call_site: Span, - pass: AstPass, - features: &[Symbol], - parent_module_id: Option, - ) -> ExpnId; - - fn resolve_imports(&mut self); - - fn resolve_macro_invocation( - &mut self, invoc: &Invocation, eager_expansion_root: ExpnId, force: bool - ) -> Result; - - fn check_unused_macros(&self); - - fn has_derives(&self, expn_id: ExpnId, derives: SpecialDerives) -> bool; - fn add_derives(&mut self, expn_id: ExpnId, derives: SpecialDerives); -} - -#[derive(Clone)] -pub struct ModuleData { - pub mod_path: Vec, - pub directory: PathBuf, -} - -#[derive(Clone)] -pub struct ExpansionData { - pub id: ExpnId, - pub depth: usize, - pub module: Rc, - pub directory_ownership: DirectoryOwnership, - pub prior_type_ascription: Option<(Span, bool)>, -} - -/// One of these is made during expansion and incrementally updated as we go; -/// when a macro expansion occurs, the resulting nodes have the `backtrace() -/// -> expn_data` of their expansion context stored into their span. -pub struct ExtCtxt<'a> { - pub parse_sess: &'a ParseSess, - pub ecfg: expand::ExpansionConfig<'a>, - pub root_path: PathBuf, - pub resolver: &'a mut dyn Resolver, - pub current_expansion: ExpansionData, - pub expansions: FxHashMap>, -} - -impl<'a> ExtCtxt<'a> { - pub fn new(parse_sess: &'a ParseSess, - ecfg: expand::ExpansionConfig<'a>, - resolver: &'a mut dyn Resolver) - -> ExtCtxt<'a> { - ExtCtxt { - parse_sess, - ecfg, - root_path: PathBuf::new(), - resolver, - current_expansion: ExpansionData { - id: ExpnId::root(), - depth: 0, - module: Rc::new(ModuleData { mod_path: Vec::new(), directory: PathBuf::new() }), - directory_ownership: DirectoryOwnership::Owned { relative: None }, - prior_type_ascription: None, - }, - expansions: FxHashMap::default(), - } - } - - /// Returns a `Folder` for deeply expanding all macros in an AST node. - pub fn expander<'b>(&'b mut self) -> expand::MacroExpander<'b, 'a> { - expand::MacroExpander::new(self, false) - } - - /// Returns a `Folder` that deeply expands all macros and assigns all `NodeId`s in an AST node. - /// Once `NodeId`s are assigned, the node may not be expanded, removed, or otherwise modified. - pub fn monotonic_expander<'b>(&'b mut self) -> expand::MacroExpander<'b, 'a> { - expand::MacroExpander::new(self, true) - } - pub fn new_parser_from_tts(&self, stream: TokenStream) -> parser::Parser<'a> { - parse::stream_to_parser(self.parse_sess, stream, MACRO_ARGUMENTS) - } - pub fn source_map(&self) -> &'a SourceMap { self.parse_sess.source_map() } - pub fn parse_sess(&self) -> &'a ParseSess { self.parse_sess } - pub fn cfg(&self) -> &ast::CrateConfig { &self.parse_sess.config } - pub fn call_site(&self) -> Span { - self.current_expansion.id.expn_data().call_site - } - - /// Equivalent of `Span::def_site` from the proc macro API, - /// except that the location is taken from the span passed as an argument. - pub fn with_def_site_ctxt(&self, span: Span) -> Span { - span.with_def_site_ctxt(self.current_expansion.id) - } - - /// Equivalent of `Span::call_site` from the proc macro API, - /// except that the location is taken from the span passed as an argument. - pub fn with_call_site_ctxt(&self, span: Span) -> Span { - span.with_call_site_ctxt(self.current_expansion.id) - } - - /// Equivalent of `Span::mixed_site` from the proc macro API, - /// except that the location is taken from the span passed as an argument. - pub fn with_mixed_site_ctxt(&self, span: Span) -> Span { - span.with_mixed_site_ctxt(self.current_expansion.id) - } - - /// Returns span for the macro which originally caused the current expansion to happen. - /// - /// Stops backtracing at include! boundary. - pub fn expansion_cause(&self) -> Option { - let mut expn_id = self.current_expansion.id; - let mut last_macro = None; - loop { - let expn_data = expn_id.expn_data(); - // Stop going up the backtrace once include! is encountered - if expn_data.is_root() || expn_data.kind.descr() == sym::include { - break; - } - expn_id = expn_data.call_site.ctxt().outer_expn(); - last_macro = Some(expn_data.call_site); - } - last_macro - } - - pub fn struct_span_warn>(&self, - sp: S, - msg: &str) - -> DiagnosticBuilder<'a> { - self.parse_sess.span_diagnostic.struct_span_warn(sp, msg) - } - pub fn struct_span_err>(&self, - sp: S, - msg: &str) - -> DiagnosticBuilder<'a> { - self.parse_sess.span_diagnostic.struct_span_err(sp, msg) - } - pub fn struct_span_fatal>(&self, - sp: S, - msg: &str) - -> DiagnosticBuilder<'a> { - self.parse_sess.span_diagnostic.struct_span_fatal(sp, msg) - } - - /// Emit `msg` attached to `sp`, and stop compilation immediately. - /// - /// `span_err` should be strongly preferred where-ever possible: - /// this should *only* be used when: - /// - /// - continuing has a high risk of flow-on errors (e.g., errors in - /// declaring a macro would cause all uses of that macro to - /// complain about "undefined macro"), or - /// - there is literally nothing else that can be done (however, - /// in most cases one can construct a dummy expression/item to - /// substitute; we never hit resolve/type-checking so the dummy - /// value doesn't have to match anything) - pub fn span_fatal>(&self, sp: S, msg: &str) -> ! { - self.parse_sess.span_diagnostic.span_fatal(sp, msg).raise(); - } - - /// Emit `msg` attached to `sp`, without immediately stopping - /// compilation. - /// - /// Compilation will be stopped in the near future (at the end of - /// the macro expansion phase). - pub fn span_err>(&self, sp: S, msg: &str) { - self.parse_sess.span_diagnostic.span_err(sp, msg); - } - pub fn span_err_with_code>(&self, sp: S, msg: &str, code: DiagnosticId) { - self.parse_sess.span_diagnostic.span_err_with_code(sp, msg, code); - } - pub fn span_warn>(&self, sp: S, msg: &str) { - self.parse_sess.span_diagnostic.span_warn(sp, msg); - } - pub fn span_bug>(&self, sp: S, msg: &str) -> ! { - self.parse_sess.span_diagnostic.span_bug(sp, msg); - } - pub fn trace_macros_diag(&mut self) { - for (sp, notes) in self.expansions.iter() { - let mut db = self.parse_sess.span_diagnostic.span_note_diag(*sp, "trace_macro"); - for note in notes { - db.note(note); - } - db.emit(); - } - // Fixme: does this result in errors? - self.expansions.clear(); - } - pub fn bug(&self, msg: &str) -> ! { - self.parse_sess.span_diagnostic.bug(msg); - } - pub fn trace_macros(&self) -> bool { - self.ecfg.trace_mac - } - pub fn set_trace_macros(&mut self, x: bool) { - self.ecfg.trace_mac = x - } - pub fn ident_of(&self, st: &str, sp: Span) -> ast::Ident { - ast::Ident::from_str_and_span(st, sp) - } - pub fn std_path(&self, components: &[Symbol]) -> Vec { - let def_site = self.with_def_site_ctxt(DUMMY_SP); - iter::once(Ident::new(kw::DollarCrate, def_site)) - .chain(components.iter().map(|&s| Ident::with_dummy_span(s))) - .collect() - } - pub fn name_of(&self, st: &str) -> ast::Name { - Symbol::intern(st) - } - - pub fn check_unused_macros(&self) { - self.resolver.check_unused_macros(); - } - - /// Resolves a path mentioned inside Rust code. - /// - /// This unifies the logic used for resolving `include_X!`, and `#[doc(include)]` file paths. - /// - /// Returns an absolute path to the file that `path` refers to. - pub fn resolve_path(&self, path: impl Into, span: Span) -> PathBuf { - let path = path.into(); - - // Relative paths are resolved relative to the file in which they are found - // after macro expansion (that is, they are unhygienic). - if !path.is_absolute() { - let callsite = span.source_callsite(); - let mut result = match self.source_map().span_to_unmapped_path(callsite) { - FileName::Real(path) => path, - FileName::DocTest(path, _) => path, - other => panic!("cannot resolve relative path in non-file source `{}`", other), - }; - result.pop(); - result.push(path); - result - } else { - path - } - } -} - -/// Extracts a string literal from the macro expanded version of `expr`, -/// emitting `err_msg` if `expr` is not a string literal. This does not stop -/// compilation on error, merely emits a non-fatal error and returns `None`. -pub fn expr_to_spanned_string<'a>( - cx: &'a mut ExtCtxt<'_>, - expr: P, - err_msg: &str, -) -> Result<(Symbol, ast::StrStyle, Span), Option>> { - // Perform eager expansion on the expression. - // We want to be able to handle e.g., `concat!("foo", "bar")`. - let expr = cx.expander().fully_expand_fragment(AstFragment::Expr(expr)).make_expr(); - - Err(match expr.kind { - ast::ExprKind::Lit(ref l) => match l.kind { - ast::LitKind::Str(s, style) => return Ok((s, style, expr.span)), - ast::LitKind::Err(_) => None, - _ => Some(cx.struct_span_err(l.span, err_msg)) - }, - ast::ExprKind::Err => None, - _ => Some(cx.struct_span_err(expr.span, err_msg)) - }) -} - -pub fn expr_to_string(cx: &mut ExtCtxt<'_>, expr: P, err_msg: &str) - -> Option<(Symbol, ast::StrStyle)> { - expr_to_spanned_string(cx, expr, err_msg) - .map_err(|err| err.map(|mut err| err.emit())) - .ok() - .map(|(symbol, style, _)| (symbol, style)) -} - -/// Non-fatally assert that `tts` is empty. Note that this function -/// returns even when `tts` is non-empty, macros that *need* to stop -/// compilation should call -/// `cx.parse_sess.span_diagnostic.abort_if_errors()` (this should be -/// done as rarely as possible). -pub fn check_zero_tts(cx: &ExtCtxt<'_>, - sp: Span, - tts: TokenStream, - name: &str) { - if !tts.is_empty() { - cx.span_err(sp, &format!("{} takes no arguments", name)); - } -} - -/// Interpreting `tts` as a comma-separated sequence of expressions, -/// expect exactly one string literal, or emit an error and return `None`. -pub fn get_single_str_from_tts(cx: &mut ExtCtxt<'_>, - sp: Span, - tts: TokenStream, - name: &str) - -> Option { - let mut p = cx.new_parser_from_tts(tts); - if p.token == token::Eof { - cx.span_err(sp, &format!("{} takes 1 argument", name)); - return None - } - let ret = panictry!(p.parse_expr()); - let _ = p.eat(&token::Comma); - - if p.token != token::Eof { - cx.span_err(sp, &format!("{} takes 1 argument", name)); - } - expr_to_string(cx, ret, "argument must be a string literal").map(|(s, _)| { - s.to_string() - }) -} - -/// Extracts comma-separated expressions from `tts`. If there is a -/// parsing error, emit a non-fatal error and return `None`. -pub fn get_exprs_from_tts(cx: &mut ExtCtxt<'_>, - sp: Span, - tts: TokenStream) -> Option>> { - let mut p = cx.new_parser_from_tts(tts); - let mut es = Vec::new(); - while p.token != token::Eof { - let expr = panictry!(p.parse_expr()); - - // Perform eager expansion on the expression. - // We want to be able to handle e.g., `concat!("foo", "bar")`. - let expr = cx.expander().fully_expand_fragment(AstFragment::Expr(expr)).make_expr(); - - es.push(expr); - if p.eat(&token::Comma) { - continue; - } - if p.token != token::Eof { - cx.span_err(sp, "expected token: `,`"); - return None; - } - } - Some(es) -} diff --git a/src/libsyntax/ext/build.rs b/src/libsyntax/ext/build.rs deleted file mode 100644 index 8c5289671c9..00000000000 --- a/src/libsyntax/ext/build.rs +++ /dev/null @@ -1,639 +0,0 @@ -use crate::ast::{self, Ident, Expr, BlockCheckMode, UnOp, PatKind}; -use crate::attr; -use crate::source_map::{respan, Spanned}; -use crate::ext::base::ExtCtxt; -use crate::ptr::P; -use crate::symbol::{kw, sym, Symbol}; -use crate::ThinVec; - -use syntax_pos::{Pos, Span}; - -impl<'a> ExtCtxt<'a> { - pub fn path(&self, span: Span, strs: Vec ) -> ast::Path { - self.path_all(span, false, strs, vec![]) - } - pub fn path_ident(&self, span: Span, id: ast::Ident) -> ast::Path { - self.path(span, vec![id]) - } - pub fn path_global(&self, span: Span, strs: Vec ) -> ast::Path { - self.path_all(span, true, strs, vec![]) - } - pub fn path_all(&self, - span: Span, - global: bool, - mut idents: Vec , - args: Vec) - -> ast::Path { - assert!(!idents.is_empty()); - let add_root = global && !idents[0].is_path_segment_keyword(); - let mut segments = Vec::with_capacity(idents.len() + add_root as usize); - if add_root { - segments.push(ast::PathSegment::path_root(span)); - } - let last_ident = idents.pop().unwrap(); - segments.extend(idents.into_iter().map(|ident| { - ast::PathSegment::from_ident(ident.with_span_pos(span)) - })); - let args = if !args.is_empty() { - ast::AngleBracketedArgs { args, constraints: Vec::new(), span }.into() - } else { - None - }; - segments.push(ast::PathSegment { - ident: last_ident.with_span_pos(span), - id: ast::DUMMY_NODE_ID, - args, - }); - ast::Path { span, segments } - } - - pub fn ty_mt(&self, ty: P, mutbl: ast::Mutability) -> ast::MutTy { - ast::MutTy { - ty, - mutbl, - } - } - - pub fn ty(&self, span: Span, kind: ast::TyKind) -> P { - P(ast::Ty { - id: ast::DUMMY_NODE_ID, - span, - kind, - }) - } - - pub fn ty_path(&self, path: ast::Path) -> P { - self.ty(path.span, ast::TyKind::Path(None, path)) - } - - // Might need to take bounds as an argument in the future, if you ever want - // to generate a bounded existential trait type. - pub fn ty_ident(&self, span: Span, ident: ast::Ident) - -> P { - self.ty_path(self.path_ident(span, ident)) - } - - pub fn anon_const(&self, span: Span, kind: ast::ExprKind) -> ast::AnonConst { - ast::AnonConst { - id: ast::DUMMY_NODE_ID, - value: P(ast::Expr { - id: ast::DUMMY_NODE_ID, - kind, - span, - attrs: ThinVec::new(), - }) - } - } - - pub fn const_ident(&self, span: Span, ident: ast::Ident) -> ast::AnonConst { - self.anon_const(span, ast::ExprKind::Path(None, self.path_ident(span, ident))) - } - - pub fn ty_rptr(&self, - span: Span, - ty: P, - lifetime: Option, - mutbl: ast::Mutability) - -> P { - self.ty(span, - ast::TyKind::Rptr(lifetime, self.ty_mt(ty, mutbl))) - } - - pub fn ty_ptr(&self, - span: Span, - ty: P, - mutbl: ast::Mutability) - -> P { - self.ty(span, - ast::TyKind::Ptr(self.ty_mt(ty, mutbl))) - } - - pub fn typaram(&self, - span: Span, - ident: ast::Ident, - attrs: Vec, - bounds: ast::GenericBounds, - default: Option>) -> ast::GenericParam { - ast::GenericParam { - ident: ident.with_span_pos(span), - id: ast::DUMMY_NODE_ID, - attrs: attrs.into(), - bounds, - kind: ast::GenericParamKind::Type { - default, - }, - is_placeholder: false - } - } - - pub fn trait_ref(&self, path: ast::Path) -> ast::TraitRef { - ast::TraitRef { - path, - ref_id: ast::DUMMY_NODE_ID, - } - } - - pub fn poly_trait_ref(&self, span: Span, path: ast::Path) -> ast::PolyTraitRef { - ast::PolyTraitRef { - bound_generic_params: Vec::new(), - trait_ref: self.trait_ref(path), - span, - } - } - - pub fn trait_bound(&self, path: ast::Path) -> ast::GenericBound { - ast::GenericBound::Trait(self.poly_trait_ref(path.span, path), - ast::TraitBoundModifier::None) - } - - pub fn lifetime(&self, span: Span, ident: ast::Ident) -> ast::Lifetime { - ast::Lifetime { id: ast::DUMMY_NODE_ID, ident: ident.with_span_pos(span) } - } - - pub fn lifetime_def(&self, - span: Span, - ident: ast::Ident, - attrs: Vec, - bounds: ast::GenericBounds) - -> ast::GenericParam { - let lifetime = self.lifetime(span, ident); - ast::GenericParam { - ident: lifetime.ident, - id: lifetime.id, - attrs: attrs.into(), - bounds, - kind: ast::GenericParamKind::Lifetime, - is_placeholder: false - } - } - - pub fn stmt_expr(&self, expr: P) -> ast::Stmt { - ast::Stmt { - id: ast::DUMMY_NODE_ID, - span: expr.span, - kind: ast::StmtKind::Expr(expr), - } - } - - pub fn stmt_let(&self, sp: Span, mutbl: bool, ident: ast::Ident, - ex: P) -> ast::Stmt { - let pat = if mutbl { - let binding_mode = ast::BindingMode::ByValue(ast::Mutability::Mutable); - self.pat_ident_binding_mode(sp, ident, binding_mode) - } else { - self.pat_ident(sp, ident) - }; - let local = P(ast::Local { - pat, - ty: None, - init: Some(ex), - id: ast::DUMMY_NODE_ID, - span: sp, - attrs: ThinVec::new(), - }); - ast::Stmt { - id: ast::DUMMY_NODE_ID, - kind: ast::StmtKind::Local(local), - span: sp, - } - } - - // Generates `let _: Type;`, which is usually used for type assertions. - pub fn stmt_let_type_only(&self, span: Span, ty: P) -> ast::Stmt { - let local = P(ast::Local { - pat: self.pat_wild(span), - ty: Some(ty), - init: None, - id: ast::DUMMY_NODE_ID, - span, - attrs: ThinVec::new(), - }); - ast::Stmt { - id: ast::DUMMY_NODE_ID, - kind: ast::StmtKind::Local(local), - span, - } - } - - pub fn stmt_item(&self, sp: Span, item: P) -> ast::Stmt { - ast::Stmt { - id: ast::DUMMY_NODE_ID, - kind: ast::StmtKind::Item(item), - span: sp, - } - } - - pub fn block_expr(&self, expr: P) -> P { - self.block(expr.span, vec![ast::Stmt { - id: ast::DUMMY_NODE_ID, - span: expr.span, - kind: ast::StmtKind::Expr(expr), - }]) - } - pub fn block(&self, span: Span, stmts: Vec) -> P { - P(ast::Block { - stmts, - id: ast::DUMMY_NODE_ID, - rules: BlockCheckMode::Default, - span, - }) - } - - pub fn expr(&self, span: Span, kind: ast::ExprKind) -> P { - P(ast::Expr { - id: ast::DUMMY_NODE_ID, - kind, - span, - attrs: ThinVec::new(), - }) - } - - pub fn expr_path(&self, path: ast::Path) -> P { - self.expr(path.span, ast::ExprKind::Path(None, path)) - } - - pub fn expr_ident(&self, span: Span, id: ast::Ident) -> P { - self.expr_path(self.path_ident(span, id)) - } - pub fn expr_self(&self, span: Span) -> P { - self.expr_ident(span, Ident::with_dummy_span(kw::SelfLower)) - } - - pub fn expr_binary(&self, sp: Span, op: ast::BinOpKind, - lhs: P, rhs: P) -> P { - self.expr(sp, ast::ExprKind::Binary(Spanned { node: op, span: sp }, lhs, rhs)) - } - - pub fn expr_deref(&self, sp: Span, e: P) -> P { - self.expr(sp, ast::ExprKind::Unary(UnOp::Deref, e)) - } - - pub fn expr_addr_of(&self, sp: Span, e: P) -> P { - self.expr(sp, ast::ExprKind::AddrOf(ast::Mutability::Immutable, e)) - } - - pub fn expr_call( - &self, span: Span, expr: P, args: Vec>, - ) -> P { - self.expr(span, ast::ExprKind::Call(expr, args)) - } - pub fn expr_call_ident(&self, span: Span, id: ast::Ident, - args: Vec>) -> P { - self.expr(span, ast::ExprKind::Call(self.expr_ident(span, id), args)) - } - pub fn expr_call_global(&self, sp: Span, fn_path: Vec , - args: Vec> ) -> P { - let pathexpr = self.expr_path(self.path_global(sp, fn_path)); - self.expr_call(sp, pathexpr, args) - } - pub fn expr_method_call(&self, span: Span, - expr: P, - ident: ast::Ident, - mut args: Vec> ) -> P { - args.insert(0, expr); - let segment = ast::PathSegment::from_ident(ident.with_span_pos(span)); - self.expr(span, ast::ExprKind::MethodCall(segment, args)) - } - pub fn expr_block(&self, b: P) -> P { - self.expr(b.span, ast::ExprKind::Block(b, None)) - } - pub fn field_imm(&self, span: Span, ident: Ident, e: P) -> ast::Field { - ast::Field { - ident: ident.with_span_pos(span), - expr: e, - span, - is_shorthand: false, - attrs: ThinVec::new(), - id: ast::DUMMY_NODE_ID, - is_placeholder: false, - } - } - pub fn expr_struct( - &self, span: Span, path: ast::Path, fields: Vec - ) -> P { - self.expr(span, ast::ExprKind::Struct(path, fields, None)) - } - pub fn expr_struct_ident(&self, span: Span, - id: ast::Ident, fields: Vec) -> P { - self.expr_struct(span, self.path_ident(span, id), fields) - } - - pub fn expr_lit(&self, span: Span, lit_kind: ast::LitKind) -> P { - let lit = ast::Lit::from_lit_kind(lit_kind, span); - self.expr(span, ast::ExprKind::Lit(lit)) - } - pub fn expr_usize(&self, span: Span, i: usize) -> P { - self.expr_lit(span, ast::LitKind::Int(i as u128, - ast::LitIntType::Unsigned(ast::UintTy::Usize))) - } - pub fn expr_u32(&self, sp: Span, u: u32) -> P { - self.expr_lit(sp, ast::LitKind::Int(u as u128, - ast::LitIntType::Unsigned(ast::UintTy::U32))) - } - pub fn expr_bool(&self, sp: Span, value: bool) -> P { - self.expr_lit(sp, ast::LitKind::Bool(value)) - } - - pub fn expr_vec(&self, sp: Span, exprs: Vec>) -> P { - self.expr(sp, ast::ExprKind::Array(exprs)) - } - pub fn expr_vec_slice(&self, sp: Span, exprs: Vec>) -> P { - self.expr_addr_of(sp, self.expr_vec(sp, exprs)) - } - pub fn expr_str(&self, sp: Span, s: Symbol) -> P { - self.expr_lit(sp, ast::LitKind::Str(s, ast::StrStyle::Cooked)) - } - - pub fn expr_cast(&self, sp: Span, expr: P, ty: P) -> P { - self.expr(sp, ast::ExprKind::Cast(expr, ty)) - } - - pub fn expr_some(&self, sp: Span, expr: P) -> P { - let some = self.std_path(&[sym::option, sym::Option, sym::Some]); - self.expr_call_global(sp, some, vec![expr]) - } - - pub fn expr_tuple(&self, sp: Span, exprs: Vec>) -> P { - self.expr(sp, ast::ExprKind::Tup(exprs)) - } - - pub fn expr_fail(&self, span: Span, msg: Symbol) -> P { - let loc = self.source_map().lookup_char_pos(span.lo()); - let expr_file = self.expr_str(span, Symbol::intern(&loc.file.name.to_string())); - let expr_line = self.expr_u32(span, loc.line as u32); - let expr_col = self.expr_u32(span, loc.col.to_usize() as u32 + 1); - let expr_loc_tuple = self.expr_tuple(span, vec![expr_file, expr_line, expr_col]); - let expr_loc_ptr = self.expr_addr_of(span, expr_loc_tuple); - self.expr_call_global( - span, - [sym::std, sym::rt, sym::begin_panic].iter().map(|s| Ident::new(*s, span)).collect(), - vec![ - self.expr_str(span, msg), - expr_loc_ptr]) - } - - pub fn expr_unreachable(&self, span: Span) -> P { - self.expr_fail(span, Symbol::intern("internal error: entered unreachable code")) - } - - pub fn expr_ok(&self, sp: Span, expr: P) -> P { - let ok = self.std_path(&[sym::result, sym::Result, sym::Ok]); - self.expr_call_global(sp, ok, vec![expr]) - } - - pub fn expr_try(&self, sp: Span, head: P) -> P { - let ok = self.std_path(&[sym::result, sym::Result, sym::Ok]); - let ok_path = self.path_global(sp, ok); - let err = self.std_path(&[sym::result, sym::Result, sym::Err]); - let err_path = self.path_global(sp, err); - - let binding_variable = self.ident_of("__try_var", sp); - let binding_pat = self.pat_ident(sp, binding_variable); - let binding_expr = self.expr_ident(sp, binding_variable); - - // `Ok(__try_var)` pattern - let ok_pat = self.pat_tuple_struct(sp, ok_path, vec![binding_pat.clone()]); - - // `Err(__try_var)` (pattern and expression respectively) - let err_pat = self.pat_tuple_struct(sp, err_path.clone(), vec![binding_pat]); - let err_inner_expr = self.expr_call(sp, self.expr_path(err_path), - vec![binding_expr.clone()]); - // `return Err(__try_var)` - let err_expr = self.expr(sp, ast::ExprKind::Ret(Some(err_inner_expr))); - - // `Ok(__try_var) => __try_var` - let ok_arm = self.arm(sp, ok_pat, binding_expr); - // `Err(__try_var) => return Err(__try_var)` - let err_arm = self.arm(sp, err_pat, err_expr); - - // `match head { Ok() => ..., Err() => ... }` - self.expr_match(sp, head, vec![ok_arm, err_arm]) - } - - - pub fn pat(&self, span: Span, kind: PatKind) -> P { - P(ast::Pat { id: ast::DUMMY_NODE_ID, kind, span }) - } - pub fn pat_wild(&self, span: Span) -> P { - self.pat(span, PatKind::Wild) - } - pub fn pat_lit(&self, span: Span, expr: P) -> P { - self.pat(span, PatKind::Lit(expr)) - } - pub fn pat_ident(&self, span: Span, ident: ast::Ident) -> P { - let binding_mode = ast::BindingMode::ByValue(ast::Mutability::Immutable); - self.pat_ident_binding_mode(span, ident, binding_mode) - } - - pub fn pat_ident_binding_mode(&self, - span: Span, - ident: ast::Ident, - bm: ast::BindingMode) -> P { - let pat = PatKind::Ident(bm, ident.with_span_pos(span), None); - self.pat(span, pat) - } - pub fn pat_path(&self, span: Span, path: ast::Path) -> P { - self.pat(span, PatKind::Path(None, path)) - } - pub fn pat_tuple_struct(&self, span: Span, path: ast::Path, - subpats: Vec>) -> P { - self.pat(span, PatKind::TupleStruct(path, subpats)) - } - pub fn pat_struct(&self, span: Span, path: ast::Path, - field_pats: Vec) -> P { - self.pat(span, PatKind::Struct(path, field_pats, false)) - } - pub fn pat_tuple(&self, span: Span, pats: Vec>) -> P { - self.pat(span, PatKind::Tuple(pats)) - } - - pub fn pat_some(&self, span: Span, pat: P) -> P { - let some = self.std_path(&[sym::option, sym::Option, sym::Some]); - let path = self.path_global(span, some); - self.pat_tuple_struct(span, path, vec![pat]) - } - - pub fn pat_none(&self, span: Span) -> P { - let some = self.std_path(&[sym::option, sym::Option, sym::None]); - let path = self.path_global(span, some); - self.pat_path(span, path) - } - - pub fn pat_ok(&self, span: Span, pat: P) -> P { - let some = self.std_path(&[sym::result, sym::Result, sym::Ok]); - let path = self.path_global(span, some); - self.pat_tuple_struct(span, path, vec![pat]) - } - - pub fn pat_err(&self, span: Span, pat: P) -> P { - let some = self.std_path(&[sym::result, sym::Result, sym::Err]); - let path = self.path_global(span, some); - self.pat_tuple_struct(span, path, vec![pat]) - } - - pub fn arm(&self, span: Span, pat: P, expr: P) -> ast::Arm { - ast::Arm { - attrs: vec![], - pat, - guard: None, - body: expr, - span, - id: ast::DUMMY_NODE_ID, - is_placeholder: false, - } - } - - pub fn arm_unreachable(&self, span: Span) -> ast::Arm { - self.arm(span, self.pat_wild(span), self.expr_unreachable(span)) - } - - pub fn expr_match(&self, span: Span, arg: P, arms: Vec) -> P { - self.expr(span, ast::ExprKind::Match(arg, arms)) - } - - pub fn expr_if(&self, span: Span, cond: P, - then: P, els: Option>) -> P { - let els = els.map(|x| self.expr_block(self.block_expr(x))); - self.expr(span, ast::ExprKind::If(cond, self.block_expr(then), els)) - } - - pub fn lambda_fn_decl(&self, - span: Span, - fn_decl: P, - body: P, - fn_decl_span: Span) // span of the `|...|` part - -> P { - self.expr(span, ast::ExprKind::Closure(ast::CaptureBy::Ref, - ast::IsAsync::NotAsync, - ast::Movability::Movable, - fn_decl, - body, - fn_decl_span)) - } - - pub fn lambda(&self, - span: Span, - ids: Vec, - body: P) - -> P { - let fn_decl = self.fn_decl( - ids.iter().map(|id| self.param(span, *id, self.ty(span, ast::TyKind::Infer))).collect(), - ast::FunctionRetTy::Default(span)); - - // FIXME -- We are using `span` as the span of the `|...|` - // part of the lambda, but it probably (maybe?) corresponds to - // the entire lambda body. Probably we should extend the API - // here, but that's not entirely clear. - self.expr(span, ast::ExprKind::Closure(ast::CaptureBy::Ref, - ast::IsAsync::NotAsync, - ast::Movability::Movable, - fn_decl, - body, - span)) - } - - pub fn lambda0(&self, span: Span, body: P) -> P { - self.lambda(span, Vec::new(), body) - } - - pub fn lambda1(&self, span: Span, body: P, ident: ast::Ident) -> P { - self.lambda(span, vec![ident], body) - } - - pub fn lambda_stmts_1(&self, span: Span, stmts: Vec, - ident: ast::Ident) -> P { - self.lambda1(span, self.expr_block(self.block(span, stmts)), ident) - } - - pub fn param(&self, span: Span, ident: ast::Ident, ty: P) -> ast::Param { - let arg_pat = self.pat_ident(span, ident); - ast::Param { - attrs: ThinVec::default(), - id: ast::DUMMY_NODE_ID, - pat: arg_pat, - span, - ty, - is_placeholder: false, - } - } - - // FIXME: unused `self` - pub fn fn_decl(&self, inputs: Vec, output: ast::FunctionRetTy) -> P { - P(ast::FnDecl { - inputs, - output, - }) - } - - pub fn item(&self, span: Span, name: Ident, - attrs: Vec, kind: ast::ItemKind) -> P { - // FIXME: Would be nice if our generated code didn't violate - // Rust coding conventions - P(ast::Item { - ident: name, - attrs, - id: ast::DUMMY_NODE_ID, - kind, - vis: respan(span.shrink_to_lo(), ast::VisibilityKind::Inherited), - span, - tokens: None, - }) - } - - pub fn variant(&self, span: Span, ident: Ident, tys: Vec> ) -> ast::Variant { - let fields: Vec<_> = tys.into_iter().map(|ty| { - ast::StructField { - span: ty.span, - ty, - ident: None, - vis: respan(span.shrink_to_lo(), ast::VisibilityKind::Inherited), - attrs: Vec::new(), - id: ast::DUMMY_NODE_ID, - is_placeholder: false, - } - }).collect(); - - let vdata = if fields.is_empty() { - ast::VariantData::Unit(ast::DUMMY_NODE_ID) - } else { - ast::VariantData::Tuple(fields, ast::DUMMY_NODE_ID) - }; - - ast::Variant { - attrs: Vec::new(), - data: vdata, - disr_expr: None, - id: ast::DUMMY_NODE_ID, - ident, - span, - is_placeholder: false, - } - } - - pub fn item_static(&self, - span: Span, - name: Ident, - ty: P, - mutbl: ast::Mutability, - expr: P) - -> P { - self.item(span, name, Vec::new(), ast::ItemKind::Static(ty, mutbl, expr)) - } - - pub fn item_const(&self, - span: Span, - name: Ident, - ty: P, - expr: P) - -> P { - self.item(span, name, Vec::new(), ast::ItemKind::Const(ty, expr)) - } - - pub fn attribute(&self, mi: ast::MetaItem) -> ast::Attribute { - attr::mk_attr_outer(mi) - } - - pub fn meta_word(&self, sp: Span, w: ast::Name) -> ast::MetaItem { - attr::mk_word_item(Ident::new(w, sp)) - } -} diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs deleted file mode 100644 index 44a51a01710..00000000000 --- a/src/libsyntax/ext/expand.rs +++ /dev/null @@ -1,1543 +0,0 @@ -use crate::ast::{self, AttrItem, Block, Ident, LitKind, NodeId, PatKind, Path}; -use crate::ast::{MacStmtStyle, StmtKind, ItemKind}; -use crate::attr::{self, HasAttrs}; -use crate::source_map::respan; -use crate::config::StripUnconfigured; -use crate::ext::base::*; -use crate::ext::proc_macro::{collect_derives, MarkAttrs}; -use crate::ext::hygiene::{ExpnId, SyntaxContext, ExpnData, ExpnKind}; -use crate::ext::mbe::macro_rules::annotate_err_with_kind; -use crate::ext::placeholders::{placeholder, PlaceholderExpander}; -use crate::feature_gate::{self, Features, GateIssue, is_builtin_attr, emit_feature_err}; -use crate::mut_visit::*; -use crate::parse::{DirectoryOwnership, PResult}; -use crate::parse::token; -use crate::parse::parser::Parser; -use crate::print::pprust; -use crate::ptr::P; -use crate::symbol::{sym, Symbol}; -use crate::tokenstream::{TokenStream, TokenTree}; -use crate::visit::Visitor; -use crate::util::map_in_place::MapInPlace; - -use errors::{Applicability, FatalError}; -use smallvec::{smallvec, SmallVec}; -use syntax_pos::{Span, DUMMY_SP, FileName}; - -use rustc_data_structures::fx::FxHashMap; -use rustc_data_structures::sync::Lrc; -use std::io::ErrorKind; -use std::{iter, mem, slice}; -use std::ops::DerefMut; -use std::rc::Rc; -use std::path::PathBuf; - -macro_rules! ast_fragments { - ( - $($Kind:ident($AstTy:ty) { - $kind_name:expr; - $(one fn $mut_visit_ast:ident; fn $visit_ast:ident;)? - $(many fn $flat_map_ast_elt:ident; fn $visit_ast_elt:ident;)? - fn $make_ast:ident; - })* - ) => { - /// A fragment of AST that can be produced by a single macro expansion. - /// Can also serve as an input and intermediate result for macro expansion operations. - pub enum AstFragment { - OptExpr(Option>), - $($Kind($AstTy),)* - } - - /// "Discriminant" of an AST fragment. - #[derive(Copy, Clone, PartialEq, Eq)] - pub enum AstFragmentKind { - OptExpr, - $($Kind,)* - } - - impl AstFragmentKind { - pub fn name(self) -> &'static str { - match self { - AstFragmentKind::OptExpr => "expression", - $(AstFragmentKind::$Kind => $kind_name,)* - } - } - - fn make_from<'a>(self, result: Box) -> Option { - match self { - AstFragmentKind::OptExpr => - result.make_expr().map(Some).map(AstFragment::OptExpr), - $(AstFragmentKind::$Kind => result.$make_ast().map(AstFragment::$Kind),)* - } - } - } - - impl AstFragment { - pub fn make_opt_expr(self) -> Option> { - match self { - AstFragment::OptExpr(expr) => expr, - _ => panic!("AstFragment::make_* called on the wrong kind of fragment"), - } - } - - $(pub fn $make_ast(self) -> $AstTy { - match self { - AstFragment::$Kind(ast) => ast, - _ => panic!("AstFragment::make_* called on the wrong kind of fragment"), - } - })* - - pub fn mut_visit_with(&mut self, vis: &mut F) { - match self { - AstFragment::OptExpr(opt_expr) => { - visit_clobber(opt_expr, |opt_expr| { - if let Some(expr) = opt_expr { - vis.filter_map_expr(expr) - } else { - None - } - }); - } - $($(AstFragment::$Kind(ast) => vis.$mut_visit_ast(ast),)?)* - $($(AstFragment::$Kind(ast) => - ast.flat_map_in_place(|ast| vis.$flat_map_ast_elt(ast)),)?)* - } - } - - pub fn visit_with<'a, V: Visitor<'a>>(&'a self, visitor: &mut V) { - match *self { - AstFragment::OptExpr(Some(ref expr)) => visitor.visit_expr(expr), - AstFragment::OptExpr(None) => {} - $($(AstFragment::$Kind(ref ast) => visitor.$visit_ast(ast),)?)* - $($(AstFragment::$Kind(ref ast) => for ast_elt in &ast[..] { - visitor.$visit_ast_elt(ast_elt); - })?)* - } - } - } - - impl<'a> MacResult for crate::ext::mbe::macro_rules::ParserAnyMacro<'a> { - $(fn $make_ast(self: Box>) - -> Option<$AstTy> { - Some(self.make(AstFragmentKind::$Kind).$make_ast()) - })* - } - } -} - -ast_fragments! { - Expr(P) { "expression"; one fn visit_expr; fn visit_expr; fn make_expr; } - Pat(P) { "pattern"; one fn visit_pat; fn visit_pat; fn make_pat; } - Ty(P) { "type"; one fn visit_ty; fn visit_ty; fn make_ty; } - Stmts(SmallVec<[ast::Stmt; 1]>) { - "statement"; many fn flat_map_stmt; fn visit_stmt; fn make_stmts; - } - Items(SmallVec<[P; 1]>) { - "item"; many fn flat_map_item; fn visit_item; fn make_items; - } - TraitItems(SmallVec<[ast::TraitItem; 1]>) { - "trait item"; many fn flat_map_trait_item; fn visit_trait_item; fn make_trait_items; - } - ImplItems(SmallVec<[ast::ImplItem; 1]>) { - "impl item"; many fn flat_map_impl_item; fn visit_impl_item; fn make_impl_items; - } - ForeignItems(SmallVec<[ast::ForeignItem; 1]>) { - "foreign item"; - many fn flat_map_foreign_item; - fn visit_foreign_item; - fn make_foreign_items; - } - Arms(SmallVec<[ast::Arm; 1]>) { - "match arm"; many fn flat_map_arm; fn visit_arm; fn make_arms; - } - Fields(SmallVec<[ast::Field; 1]>) { - "field expression"; many fn flat_map_field; fn visit_field; fn make_fields; - } - FieldPats(SmallVec<[ast::FieldPat; 1]>) { - "field pattern"; - many fn flat_map_field_pattern; - fn visit_field_pattern; - fn make_field_patterns; - } - GenericParams(SmallVec<[ast::GenericParam; 1]>) { - "generic parameter"; - many fn flat_map_generic_param; - fn visit_generic_param; - fn make_generic_params; - } - Params(SmallVec<[ast::Param; 1]>) { - "function parameter"; many fn flat_map_param; fn visit_param; fn make_params; - } - StructFields(SmallVec<[ast::StructField; 1]>) { - "field"; - many fn flat_map_struct_field; - fn visit_struct_field; - fn make_struct_fields; - } - Variants(SmallVec<[ast::Variant; 1]>) { - "variant"; many fn flat_map_variant; fn visit_variant; fn make_variants; - } -} - -impl AstFragmentKind { - fn dummy(self, span: Span) -> AstFragment { - self.make_from(DummyResult::any(span)).expect("couldn't create a dummy AST fragment") - } - - fn expect_from_annotatables>(self, items: I) - -> AstFragment { - let mut items = items.into_iter(); - match self { - AstFragmentKind::Arms => - AstFragment::Arms(items.map(Annotatable::expect_arm).collect()), - AstFragmentKind::Fields => - AstFragment::Fields(items.map(Annotatable::expect_field).collect()), - AstFragmentKind::FieldPats => - AstFragment::FieldPats(items.map(Annotatable::expect_field_pattern).collect()), - AstFragmentKind::GenericParams => - AstFragment::GenericParams(items.map(Annotatable::expect_generic_param).collect()), - AstFragmentKind::Params => - AstFragment::Params(items.map(Annotatable::expect_param).collect()), - AstFragmentKind::StructFields => AstFragment::StructFields( - items.map(Annotatable::expect_struct_field).collect() - ), - AstFragmentKind::Variants => - AstFragment::Variants(items.map(Annotatable::expect_variant).collect()), - AstFragmentKind::Items => - AstFragment::Items(items.map(Annotatable::expect_item).collect()), - AstFragmentKind::ImplItems => - AstFragment::ImplItems(items.map(Annotatable::expect_impl_item).collect()), - AstFragmentKind::TraitItems => - AstFragment::TraitItems(items.map(Annotatable::expect_trait_item).collect()), - AstFragmentKind::ForeignItems => - AstFragment::ForeignItems(items.map(Annotatable::expect_foreign_item).collect()), - AstFragmentKind::Stmts => - AstFragment::Stmts(items.map(Annotatable::expect_stmt).collect()), - AstFragmentKind::Expr => AstFragment::Expr( - items.next().expect("expected exactly one expression").expect_expr() - ), - AstFragmentKind::OptExpr => - AstFragment::OptExpr(items.next().map(Annotatable::expect_expr)), - AstFragmentKind::Pat | AstFragmentKind::Ty => - panic!("patterns and types aren't annotatable"), - } - } -} - -pub struct Invocation { - pub kind: InvocationKind, - pub fragment_kind: AstFragmentKind, - pub expansion_data: ExpansionData, -} - -pub enum InvocationKind { - Bang { - mac: ast::Mac, - span: Span, - }, - Attr { - attr: ast::Attribute, - item: Annotatable, - // Required for resolving derive helper attributes. - derives: Vec, - // We temporarily report errors for attribute macros placed after derives - after_derive: bool, - }, - Derive { - path: Path, - item: Annotatable, - }, - /// "Invocation" that contains all derives from an item, - /// broken into multiple `Derive` invocations when expanded. - /// FIXME: Find a way to remove it. - DeriveContainer { - derives: Vec, - item: Annotatable, - }, -} - -impl Invocation { - pub fn span(&self) -> Span { - match &self.kind { - InvocationKind::Bang { span, .. } => *span, - InvocationKind::Attr { attr, .. } => attr.span, - InvocationKind::Derive { path, .. } => path.span, - InvocationKind::DeriveContainer { item, .. } => item.span(), - } - } -} - -pub struct MacroExpander<'a, 'b> { - pub cx: &'a mut ExtCtxt<'b>, - monotonic: bool, // cf. `cx.monotonic_expander()` -} - -impl<'a, 'b> MacroExpander<'a, 'b> { - pub fn new(cx: &'a mut ExtCtxt<'b>, monotonic: bool) -> Self { - MacroExpander { cx, monotonic } - } - - pub fn expand_crate(&mut self, mut krate: ast::Crate) -> ast::Crate { - let mut module = ModuleData { - mod_path: vec![Ident::from_str(&self.cx.ecfg.crate_name)], - directory: match self.cx.source_map().span_to_unmapped_path(krate.span) { - FileName::Real(path) => path, - other => PathBuf::from(other.to_string()), - }, - }; - module.directory.pop(); - self.cx.root_path = module.directory.clone(); - self.cx.current_expansion.module = Rc::new(module); - - let orig_mod_span = krate.module.inner; - - let krate_item = AstFragment::Items(smallvec![P(ast::Item { - attrs: krate.attrs, - span: krate.span, - kind: ast::ItemKind::Mod(krate.module), - ident: Ident::invalid(), - id: ast::DUMMY_NODE_ID, - vis: respan(krate.span.shrink_to_lo(), ast::VisibilityKind::Public), - tokens: None, - })]); - - match self.fully_expand_fragment(krate_item).make_items().pop().map(P::into_inner) { - Some(ast::Item { attrs, kind: ast::ItemKind::Mod(module), .. }) => { - krate.attrs = attrs; - krate.module = module; - }, - None => { - // Resolution failed so we return an empty expansion - krate.attrs = vec![]; - krate.module = ast::Mod { - inner: orig_mod_span, - items: vec![], - inline: true, - }; - }, - _ => unreachable!(), - }; - self.cx.trace_macros_diag(); - krate - } - - // Recursively expand all macro invocations in this AST fragment. - pub fn fully_expand_fragment(&mut self, input_fragment: AstFragment) -> AstFragment { - let orig_expansion_data = self.cx.current_expansion.clone(); - self.cx.current_expansion.depth = 0; - - // Collect all macro invocations and replace them with placeholders. - let (mut fragment_with_placeholders, mut invocations) - = self.collect_invocations(input_fragment, &[]); - - // Optimization: if we resolve all imports now, - // we'll be able to immediately resolve most of imported macros. - self.resolve_imports(); - - // Resolve paths in all invocations and produce output expanded fragments for them, but - // do not insert them into our input AST fragment yet, only store in `expanded_fragments`. - // The output fragments also go through expansion recursively until no invocations are left. - // Unresolved macros produce dummy outputs as a recovery measure. - invocations.reverse(); - let mut expanded_fragments = Vec::new(); - let mut all_derive_placeholders: FxHashMap> = FxHashMap::default(); - let mut undetermined_invocations = Vec::new(); - let (mut progress, mut force) = (false, !self.monotonic); - loop { - let invoc = if let Some(invoc) = invocations.pop() { - invoc - } else { - self.resolve_imports(); - if undetermined_invocations.is_empty() { break } - invocations = mem::take(&mut undetermined_invocations); - force = !mem::replace(&mut progress, false); - continue - }; - - let eager_expansion_root = - if self.monotonic { invoc.expansion_data.id } else { orig_expansion_data.id }; - let res = match self.cx.resolver.resolve_macro_invocation( - &invoc, eager_expansion_root, force - ) { - Ok(res) => res, - Err(Indeterminate) => { - undetermined_invocations.push(invoc); - continue - } - }; - - progress = true; - let ExpansionData { depth, id: expn_id, .. } = invoc.expansion_data; - self.cx.current_expansion = invoc.expansion_data.clone(); - - // FIXME(jseyfried): Refactor out the following logic - let (expanded_fragment, new_invocations) = match res { - InvocationRes::Single(ext) => { - let fragment = self.expand_invoc(invoc, &ext.kind); - self.collect_invocations(fragment, &[]) - } - InvocationRes::DeriveContainer(exts) => { - let (derives, item) = match invoc.kind { - InvocationKind::DeriveContainer { derives, item } => (derives, item), - _ => unreachable!(), - }; - if !item.derive_allowed() { - let attr = attr::find_by_name(item.attrs(), sym::derive) - .expect("`derive` attribute should exist"); - let span = attr.span; - let mut err = self.cx.struct_span_err(span, - "`derive` may only be applied to structs, enums and unions"); - if let ast::AttrStyle::Inner = attr.style { - let trait_list = derives.iter() - .map(|t| pprust::path_to_string(t)) - .collect::>(); - let suggestion = format!("#[derive({})]", trait_list.join(", ")); - err.span_suggestion( - span, "try an outer attribute", suggestion, - // We don't 𝑘𝑛𝑜𝑤 that the following item is an ADT - Applicability::MaybeIncorrect - ); - } - err.emit(); - } - - let mut item = self.fully_configure(item); - item.visit_attrs(|attrs| attrs.retain(|a| a.path != sym::derive)); - let mut helper_attrs = Vec::new(); - let mut has_copy = false; - for ext in exts { - helper_attrs.extend(&ext.helper_attrs); - has_copy |= ext.is_derive_copy; - } - // Mark derive helpers inside this item as known and used. - // FIXME: This is a hack, derive helpers should be integrated with regular name - // resolution instead. For example, helpers introduced by a derive container - // can be in scope for all code produced by that container's expansion. - item.visit_with(&mut MarkAttrs(&helper_attrs)); - if has_copy { - self.cx.resolver.add_derives(invoc.expansion_data.id, SpecialDerives::COPY); - } - - let derive_placeholders = - all_derive_placeholders.entry(invoc.expansion_data.id).or_default(); - derive_placeholders.reserve(derives.len()); - invocations.reserve(derives.len()); - for path in derives { - let expn_id = ExpnId::fresh(None); - derive_placeholders.push(NodeId::placeholder_from_expn_id(expn_id)); - invocations.push(Invocation { - kind: InvocationKind::Derive { path, item: item.clone() }, - fragment_kind: invoc.fragment_kind, - expansion_data: ExpansionData { - id: expn_id, - ..invoc.expansion_data.clone() - }, - }); - } - let fragment = invoc.fragment_kind - .expect_from_annotatables(::std::iter::once(item)); - self.collect_invocations(fragment, derive_placeholders) - } - }; - - if expanded_fragments.len() < depth { - expanded_fragments.push(Vec::new()); - } - expanded_fragments[depth - 1].push((expn_id, expanded_fragment)); - if !self.cx.ecfg.single_step { - invocations.extend(new_invocations.into_iter().rev()); - } - } - - self.cx.current_expansion = orig_expansion_data; - - // Finally incorporate all the expanded macros into the input AST fragment. - let mut placeholder_expander = PlaceholderExpander::new(self.cx, self.monotonic); - while let Some(expanded_fragments) = expanded_fragments.pop() { - for (expn_id, expanded_fragment) in expanded_fragments.into_iter().rev() { - let derive_placeholders = - all_derive_placeholders.remove(&expn_id).unwrap_or_else(Vec::new); - placeholder_expander.add(NodeId::placeholder_from_expn_id(expn_id), - expanded_fragment, derive_placeholders); - } - } - fragment_with_placeholders.mut_visit_with(&mut placeholder_expander); - fragment_with_placeholders - } - - fn resolve_imports(&mut self) { - if self.monotonic { - self.cx.resolver.resolve_imports(); - } - } - - /// Collects all macro invocations reachable at this time in this AST fragment, and replace - /// them with "placeholders" - dummy macro invocations with specially crafted `NodeId`s. - /// Then call into resolver that builds a skeleton ("reduced graph") of the fragment and - /// prepares data for resolving paths of macro invocations. - fn collect_invocations(&mut self, mut fragment: AstFragment, extra_placeholders: &[NodeId]) - -> (AstFragment, Vec) { - // Resolve `$crate`s in the fragment for pretty-printing. - self.cx.resolver.resolve_dollar_crates(); - - let invocations = { - let mut collector = InvocationCollector { - cfg: StripUnconfigured { - sess: self.cx.parse_sess, - features: self.cx.ecfg.features, - }, - cx: self.cx, - invocations: Vec::new(), - monotonic: self.monotonic, - }; - fragment.mut_visit_with(&mut collector); - collector.invocations - }; - - // FIXME: Merge `extra_placeholders` into the `fragment` as regular placeholders. - if self.monotonic { - self.cx.resolver.visit_ast_fragment_with_placeholders( - self.cx.current_expansion.id, &fragment, extra_placeholders); - } - - (fragment, invocations) - } - - fn fully_configure(&mut self, item: Annotatable) -> Annotatable { - let mut cfg = StripUnconfigured { - sess: self.cx.parse_sess, - features: self.cx.ecfg.features, - }; - // Since the item itself has already been configured by the InvocationCollector, - // we know that fold result vector will contain exactly one element - match item { - Annotatable::Item(item) => { - Annotatable::Item(cfg.flat_map_item(item).pop().unwrap()) - } - Annotatable::TraitItem(item) => { - Annotatable::TraitItem( - item.map(|item| cfg.flat_map_trait_item(item).pop().unwrap())) - } - Annotatable::ImplItem(item) => { - Annotatable::ImplItem(item.map(|item| cfg.flat_map_impl_item(item).pop().unwrap())) - } - Annotatable::ForeignItem(item) => { - Annotatable::ForeignItem( - item.map(|item| cfg.flat_map_foreign_item(item).pop().unwrap()) - ) - } - Annotatable::Stmt(stmt) => { - Annotatable::Stmt(stmt.map(|stmt| cfg.flat_map_stmt(stmt).pop().unwrap())) - } - Annotatable::Expr(mut expr) => { - Annotatable::Expr({ cfg.visit_expr(&mut expr); expr }) - } - Annotatable::Arm(arm) => { - Annotatable::Arm(cfg.flat_map_arm(arm).pop().unwrap()) - } - Annotatable::Field(field) => { - Annotatable::Field(cfg.flat_map_field(field).pop().unwrap()) - } - Annotatable::FieldPat(fp) => { - Annotatable::FieldPat(cfg.flat_map_field_pattern(fp).pop().unwrap()) - } - Annotatable::GenericParam(param) => { - Annotatable::GenericParam(cfg.flat_map_generic_param(param).pop().unwrap()) - } - Annotatable::Param(param) => { - Annotatable::Param(cfg.flat_map_param(param).pop().unwrap()) - } - Annotatable::StructField(sf) => { - Annotatable::StructField(cfg.flat_map_struct_field(sf).pop().unwrap()) - } - Annotatable::Variant(v) => { - Annotatable::Variant(cfg.flat_map_variant(v).pop().unwrap()) - } - } - } - - fn expand_invoc(&mut self, invoc: Invocation, ext: &SyntaxExtensionKind) -> AstFragment { - if self.cx.current_expansion.depth > self.cx.ecfg.recursion_limit { - let expn_data = self.cx.current_expansion.id.expn_data(); - let suggested_limit = self.cx.ecfg.recursion_limit * 2; - let mut err = self.cx.struct_span_err(expn_data.call_site, - &format!("recursion limit reached while expanding the macro `{}`", - expn_data.kind.descr())); - err.help(&format!( - "consider adding a `#![recursion_limit=\"{}\"]` attribute to your crate", - suggested_limit)); - err.emit(); - self.cx.trace_macros_diag(); - FatalError.raise(); - } - - let (fragment_kind, span) = (invoc.fragment_kind, invoc.span()); - match invoc.kind { - InvocationKind::Bang { mac, .. } => match ext { - SyntaxExtensionKind::Bang(expander) => { - self.gate_proc_macro_expansion_kind(span, fragment_kind); - let tok_result = expander.expand(self.cx, span, mac.stream()); - self.parse_ast_fragment(tok_result, fragment_kind, &mac.path, span) - } - SyntaxExtensionKind::LegacyBang(expander) => { - let prev = self.cx.current_expansion.prior_type_ascription; - self.cx.current_expansion.prior_type_ascription = mac.prior_type_ascription; - let tok_result = expander.expand(self.cx, span, mac.stream()); - let result = if let Some(result) = fragment_kind.make_from(tok_result) { - result - } else { - let msg = format!( - "non-{kind} macro in {kind} position: {path}", - kind = fragment_kind.name(), - path = pprust::path_to_string(&mac.path), - ); - self.cx.span_err(span, &msg); - self.cx.trace_macros_diag(); - fragment_kind.dummy(span) - }; - self.cx.current_expansion.prior_type_ascription = prev; - result - } - _ => unreachable!() - } - InvocationKind::Attr { attr, mut item, .. } => match ext { - SyntaxExtensionKind::Attr(expander) => { - self.gate_proc_macro_attr_item(span, &item); - let item_tok = TokenTree::token(token::Interpolated(Lrc::new(match item { - Annotatable::Item(item) => token::NtItem(item), - Annotatable::TraitItem(item) => token::NtTraitItem(item.into_inner()), - Annotatable::ImplItem(item) => token::NtImplItem(item.into_inner()), - Annotatable::ForeignItem(item) => token::NtForeignItem(item.into_inner()), - Annotatable::Stmt(stmt) => token::NtStmt(stmt.into_inner()), - Annotatable::Expr(expr) => token::NtExpr(expr), - Annotatable::Arm(..) - | Annotatable::Field(..) - | Annotatable::FieldPat(..) - | Annotatable::GenericParam(..) - | Annotatable::Param(..) - | Annotatable::StructField(..) - | Annotatable::Variant(..) - => panic!("unexpected annotatable"), - })), DUMMY_SP).into(); - let input = self.extract_proc_macro_attr_input(attr.item.tokens, span); - let tok_result = expander.expand(self.cx, span, input, item_tok); - self.parse_ast_fragment(tok_result, fragment_kind, &attr.item.path, span) - } - SyntaxExtensionKind::LegacyAttr(expander) => { - match attr.parse_meta(self.cx.parse_sess) { - Ok(meta) => { - let item = expander.expand(self.cx, span, &meta, item); - fragment_kind.expect_from_annotatables(item) - } - Err(mut err) => { - err.emit(); - fragment_kind.dummy(span) - } - } - } - SyntaxExtensionKind::NonMacroAttr { mark_used } => { - attr::mark_known(&attr); - if *mark_used { - attr::mark_used(&attr); - } - item.visit_attrs(|attrs| attrs.push(attr)); - fragment_kind.expect_from_annotatables(iter::once(item)) - } - _ => unreachable!() - } - InvocationKind::Derive { path, item } => match ext { - SyntaxExtensionKind::Derive(expander) | - SyntaxExtensionKind::LegacyDerive(expander) => { - if !item.derive_allowed() { - return fragment_kind.dummy(span); - } - let meta = ast::MetaItem { kind: ast::MetaItemKind::Word, span, path }; - let items = expander.expand(self.cx, span, &meta, item); - fragment_kind.expect_from_annotatables(items) - } - _ => unreachable!() - } - InvocationKind::DeriveContainer { .. } => unreachable!() - } - } - - fn extract_proc_macro_attr_input(&self, tokens: TokenStream, span: Span) -> TokenStream { - let mut trees = tokens.trees(); - match trees.next() { - Some(TokenTree::Delimited(_, _, tts)) => { - if trees.next().is_none() { - return tts.into() - } - } - Some(TokenTree::Token(..)) => {} - None => return TokenStream::default(), - } - self.cx.span_err(span, "custom attribute invocations must be \ - of the form `#[foo]` or `#[foo(..)]`, the macro name must only be \ - followed by a delimiter token"); - TokenStream::default() - } - - fn gate_proc_macro_attr_item(&self, span: Span, item: &Annotatable) { - let (kind, gate) = match *item { - Annotatable::Item(ref item) => { - match item.kind { - ItemKind::Mod(_) if self.cx.ecfg.proc_macro_hygiene() => return, - ItemKind::Mod(_) => ("modules", sym::proc_macro_hygiene), - _ => return, - } - } - Annotatable::TraitItem(_) => return, - Annotatable::ImplItem(_) => return, - Annotatable::ForeignItem(_) => return, - Annotatable::Stmt(_) | - Annotatable::Expr(_) if self.cx.ecfg.proc_macro_hygiene() => return, - Annotatable::Stmt(_) => ("statements", sym::proc_macro_hygiene), - Annotatable::Expr(_) => ("expressions", sym::proc_macro_hygiene), - Annotatable::Arm(..) - | Annotatable::Field(..) - | Annotatable::FieldPat(..) - | Annotatable::GenericParam(..) - | Annotatable::Param(..) - | Annotatable::StructField(..) - | Annotatable::Variant(..) - => panic!("unexpected annotatable"), - }; - emit_feature_err( - self.cx.parse_sess, - gate, - span, - GateIssue::Language, - &format!("custom attributes cannot be applied to {}", kind), - ); - } - - fn gate_proc_macro_expansion_kind(&self, span: Span, kind: AstFragmentKind) { - let kind = match kind { - AstFragmentKind::Expr | - AstFragmentKind::OptExpr => "expressions", - AstFragmentKind::Pat => "patterns", - AstFragmentKind::Stmts => "statements", - AstFragmentKind::Ty | - AstFragmentKind::Items | - AstFragmentKind::TraitItems | - AstFragmentKind::ImplItems | - AstFragmentKind::ForeignItems => return, - AstFragmentKind::Arms - | AstFragmentKind::Fields - | AstFragmentKind::FieldPats - | AstFragmentKind::GenericParams - | AstFragmentKind::Params - | AstFragmentKind::StructFields - | AstFragmentKind::Variants - => panic!("unexpected AST fragment kind"), - }; - if self.cx.ecfg.proc_macro_hygiene() { - return - } - emit_feature_err( - self.cx.parse_sess, - sym::proc_macro_hygiene, - span, - GateIssue::Language, - &format!("procedural macros cannot be expanded to {}", kind), - ); - } - - fn parse_ast_fragment( - &mut self, - toks: TokenStream, - kind: AstFragmentKind, - path: &Path, - span: Span, - ) -> AstFragment { - let mut parser = self.cx.new_parser_from_tts(toks); - match parser.parse_ast_fragment(kind, false) { - Ok(fragment) => { - parser.ensure_complete_parse(path, kind.name(), span); - fragment - } - Err(mut err) => { - err.set_span(span); - annotate_err_with_kind(&mut err, kind, span); - err.emit(); - self.cx.trace_macros_diag(); - kind.dummy(span) - } - } - } -} - -impl<'a> Parser<'a> { - pub fn parse_ast_fragment(&mut self, kind: AstFragmentKind, macro_legacy_warnings: bool) - -> PResult<'a, AstFragment> { - Ok(match kind { - AstFragmentKind::Items => { - let mut items = SmallVec::new(); - while let Some(item) = self.parse_item()? { - items.push(item); - } - AstFragment::Items(items) - } - AstFragmentKind::TraitItems => { - let mut items = SmallVec::new(); - while self.token != token::Eof { - items.push(self.parse_trait_item(&mut false)?); - } - AstFragment::TraitItems(items) - } - AstFragmentKind::ImplItems => { - let mut items = SmallVec::new(); - while self.token != token::Eof { - items.push(self.parse_impl_item(&mut false)?); - } - AstFragment::ImplItems(items) - } - AstFragmentKind::ForeignItems => { - let mut items = SmallVec::new(); - while self.token != token::Eof { - items.push(self.parse_foreign_item(DUMMY_SP)?); - } - AstFragment::ForeignItems(items) - } - AstFragmentKind::Stmts => { - let mut stmts = SmallVec::new(); - while self.token != token::Eof && - // won't make progress on a `}` - self.token != token::CloseDelim(token::Brace) { - if let Some(stmt) = self.parse_full_stmt(macro_legacy_warnings)? { - stmts.push(stmt); - } - } - AstFragment::Stmts(stmts) - } - AstFragmentKind::Expr => AstFragment::Expr(self.parse_expr()?), - AstFragmentKind::OptExpr => { - if self.token != token::Eof { - AstFragment::OptExpr(Some(self.parse_expr()?)) - } else { - AstFragment::OptExpr(None) - } - }, - AstFragmentKind::Ty => AstFragment::Ty(self.parse_ty()?), - AstFragmentKind::Pat => AstFragment::Pat(self.parse_pat(None)?), - AstFragmentKind::Arms - | AstFragmentKind::Fields - | AstFragmentKind::FieldPats - | AstFragmentKind::GenericParams - | AstFragmentKind::Params - | AstFragmentKind::StructFields - | AstFragmentKind::Variants - => panic!("unexpected AST fragment kind"), - }) - } - - pub fn ensure_complete_parse(&mut self, macro_path: &Path, kind_name: &str, span: Span) { - if self.token != token::Eof { - let msg = format!("macro expansion ignores token `{}` and any following", - self.this_token_to_string()); - // Avoid emitting backtrace info twice. - let def_site_span = self.token.span.with_ctxt(SyntaxContext::root()); - let mut err = self.struct_span_err(def_site_span, &msg); - err.span_label(span, "caused by the macro expansion here"); - let msg = format!( - "the usage of `{}!` is likely invalid in {} context", - pprust::path_to_string(¯o_path), - kind_name, - ); - err.note(&msg); - let semi_span = self.sess.source_map().next_point(span); - - let semi_full_span = semi_span.to(self.sess.source_map().next_point(semi_span)); - match self.sess.source_map().span_to_snippet(semi_full_span) { - Ok(ref snippet) if &snippet[..] != ";" && kind_name == "expression" => { - err.span_suggestion( - semi_span, - "you might be missing a semicolon here", - ";".to_owned(), - Applicability::MaybeIncorrect, - ); - } - _ => {} - } - err.emit(); - } - } -} - -struct InvocationCollector<'a, 'b> { - cx: &'a mut ExtCtxt<'b>, - cfg: StripUnconfigured<'a>, - invocations: Vec, - monotonic: bool, -} - -impl<'a, 'b> InvocationCollector<'a, 'b> { - fn collect(&mut self, fragment_kind: AstFragmentKind, kind: InvocationKind) -> AstFragment { - // Expansion data for all the collected invocations is set upon their resolution, - // with exception of the derive container case which is not resolved and can get - // its expansion data immediately. - let expn_data = match &kind { - InvocationKind::DeriveContainer { item, .. } => Some(ExpnData { - parent: self.cx.current_expansion.id, - ..ExpnData::default( - ExpnKind::Macro(MacroKind::Attr, sym::derive), - item.span(), self.cx.parse_sess.edition, - ) - }), - _ => None, - }; - let expn_id = ExpnId::fresh(expn_data); - self.invocations.push(Invocation { - kind, - fragment_kind, - expansion_data: ExpansionData { - id: expn_id, - depth: self.cx.current_expansion.depth + 1, - ..self.cx.current_expansion.clone() - }, - }); - placeholder(fragment_kind, NodeId::placeholder_from_expn_id(expn_id)) - } - - fn collect_bang(&mut self, mac: ast::Mac, span: Span, kind: AstFragmentKind) -> AstFragment { - self.collect(kind, InvocationKind::Bang { mac, span }) - } - - fn collect_attr(&mut self, - attr: Option, - derives: Vec, - item: Annotatable, - kind: AstFragmentKind, - after_derive: bool) - -> AstFragment { - self.collect(kind, match attr { - Some(attr) => InvocationKind::Attr { attr, item, derives, after_derive }, - None => InvocationKind::DeriveContainer { derives, item }, - }) - } - - fn find_attr_invoc(&self, attrs: &mut Vec, after_derive: &mut bool) - -> Option { - let attr = attrs.iter() - .position(|a| { - if a.path == sym::derive { - *after_derive = true; - } - !attr::is_known(a) && !is_builtin_attr(a) - }) - .map(|i| attrs.remove(i)); - if let Some(attr) = &attr { - if !self.cx.ecfg.custom_inner_attributes() && - attr.style == ast::AttrStyle::Inner && attr.path != sym::test { - emit_feature_err(&self.cx.parse_sess, sym::custom_inner_attributes, - attr.span, GateIssue::Language, - "non-builtin inner attributes are unstable"); - } - } - attr - } - - /// If `item` is an attr invocation, remove and return the macro attribute and derive traits. - fn classify_item(&mut self, item: &mut T) - -> (Option, Vec, /* after_derive */ bool) - where T: HasAttrs, - { - let (mut attr, mut traits, mut after_derive) = (None, Vec::new(), false); - - item.visit_attrs(|mut attrs| { - attr = self.find_attr_invoc(&mut attrs, &mut after_derive); - traits = collect_derives(&mut self.cx, &mut attrs); - }); - - (attr, traits, after_derive) - } - - /// Alternative to `classify_item()` that ignores `#[derive]` so invocations fallthrough - /// to the unused-attributes lint (making it an error on statements and expressions - /// is a breaking change) - fn classify_nonitem(&mut self, nonitem: &mut T) - -> (Option, /* after_derive */ bool) { - let (mut attr, mut after_derive) = (None, false); - - nonitem.visit_attrs(|mut attrs| { - attr = self.find_attr_invoc(&mut attrs, &mut after_derive); - }); - - (attr, after_derive) - } - - fn configure(&mut self, node: T) -> Option { - self.cfg.configure(node) - } - - // Detect use of feature-gated or invalid attributes on macro invocations - // since they will not be detected after macro expansion. - fn check_attributes(&mut self, attrs: &[ast::Attribute]) { - let features = self.cx.ecfg.features.unwrap(); - for attr in attrs.iter() { - feature_gate::check_attribute(attr, self.cx.parse_sess, features); - - // macros are expanded before any lint passes so this warning has to be hardcoded - if attr.path == sym::derive { - self.cx.struct_span_warn(attr.span, "`#[derive]` does nothing on macro invocations") - .note("this may become a hard error in a future release") - .emit(); - } - } - } -} - -impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> { - fn visit_expr(&mut self, expr: &mut P) { - self.cfg.configure_expr(expr); - visit_clobber(expr.deref_mut(), |mut expr| { - self.cfg.configure_expr_kind(&mut expr.kind); - - // ignore derives so they remain unused - let (attr, after_derive) = self.classify_nonitem(&mut expr); - - if attr.is_some() { - // Collect the invoc regardless of whether or not attributes are permitted here - // expansion will eat the attribute so it won't error later. - attr.as_ref().map(|a| self.cfg.maybe_emit_expr_attr_err(a)); - - // AstFragmentKind::Expr requires the macro to emit an expression. - return self.collect_attr(attr, vec![], Annotatable::Expr(P(expr)), - AstFragmentKind::Expr, after_derive) - .make_expr() - .into_inner() - } - - if let ast::ExprKind::Mac(mac) = expr.kind { - self.check_attributes(&expr.attrs); - self.collect_bang(mac, expr.span, AstFragmentKind::Expr) - .make_expr() - .into_inner() - } else { - noop_visit_expr(&mut expr, self); - expr - } - }); - } - - fn flat_map_arm(&mut self, arm: ast::Arm) -> SmallVec<[ast::Arm; 1]> { - let mut arm = configure!(self, arm); - - let (attr, traits, after_derive) = self.classify_item(&mut arm); - if attr.is_some() || !traits.is_empty() { - return self.collect_attr(attr, traits, Annotatable::Arm(arm), - AstFragmentKind::Arms, after_derive) - .make_arms(); - } - - noop_flat_map_arm(arm, self) - } - - fn flat_map_field(&mut self, field: ast::Field) -> SmallVec<[ast::Field; 1]> { - let mut field = configure!(self, field); - - let (attr, traits, after_derive) = self.classify_item(&mut field); - if attr.is_some() || !traits.is_empty() { - return self.collect_attr(attr, traits, Annotatable::Field(field), - AstFragmentKind::Fields, after_derive) - .make_fields(); - } - - noop_flat_map_field(field, self) - } - - fn flat_map_field_pattern(&mut self, fp: ast::FieldPat) -> SmallVec<[ast::FieldPat; 1]> { - let mut fp = configure!(self, fp); - - let (attr, traits, after_derive) = self.classify_item(&mut fp); - if attr.is_some() || !traits.is_empty() { - return self.collect_attr(attr, traits, Annotatable::FieldPat(fp), - AstFragmentKind::FieldPats, after_derive) - .make_field_patterns(); - } - - noop_flat_map_field_pattern(fp, self) - } - - fn flat_map_param(&mut self, p: ast::Param) -> SmallVec<[ast::Param; 1]> { - let mut p = configure!(self, p); - - let (attr, traits, after_derive) = self.classify_item(&mut p); - if attr.is_some() || !traits.is_empty() { - return self.collect_attr(attr, traits, Annotatable::Param(p), - AstFragmentKind::Params, after_derive) - .make_params(); - } - - noop_flat_map_param(p, self) - } - - fn flat_map_struct_field(&mut self, sf: ast::StructField) -> SmallVec<[ast::StructField; 1]> { - let mut sf = configure!(self, sf); - - let (attr, traits, after_derive) = self.classify_item(&mut sf); - if attr.is_some() || !traits.is_empty() { - return self.collect_attr(attr, traits, Annotatable::StructField(sf), - AstFragmentKind::StructFields, after_derive) - .make_struct_fields(); - } - - noop_flat_map_struct_field(sf, self) - } - - fn flat_map_variant(&mut self, variant: ast::Variant) -> SmallVec<[ast::Variant; 1]> { - let mut variant = configure!(self, variant); - - let (attr, traits, after_derive) = self.classify_item(&mut variant); - if attr.is_some() || !traits.is_empty() { - return self.collect_attr(attr, traits, Annotatable::Variant(variant), - AstFragmentKind::Variants, after_derive) - .make_variants(); - } - - noop_flat_map_variant(variant, self) - } - - fn filter_map_expr(&mut self, expr: P) -> Option> { - let expr = configure!(self, expr); - expr.filter_map(|mut expr| { - self.cfg.configure_expr_kind(&mut expr.kind); - - // Ignore derives so they remain unused. - let (attr, after_derive) = self.classify_nonitem(&mut expr); - - if attr.is_some() { - attr.as_ref().map(|a| self.cfg.maybe_emit_expr_attr_err(a)); - - return self.collect_attr(attr, vec![], Annotatable::Expr(P(expr)), - AstFragmentKind::OptExpr, after_derive) - .make_opt_expr() - .map(|expr| expr.into_inner()) - } - - if let ast::ExprKind::Mac(mac) = expr.kind { - self.check_attributes(&expr.attrs); - self.collect_bang(mac, expr.span, AstFragmentKind::OptExpr) - .make_opt_expr() - .map(|expr| expr.into_inner()) - } else { - Some({ noop_visit_expr(&mut expr, self); expr }) - } - }) - } - - fn visit_pat(&mut self, pat: &mut P) { - self.cfg.configure_pat(pat); - match pat.kind { - PatKind::Mac(_) => {} - _ => return noop_visit_pat(pat, self), - } - - visit_clobber(pat, |mut pat| { - match mem::replace(&mut pat.kind, PatKind::Wild) { - PatKind::Mac(mac) => - self.collect_bang(mac, pat.span, AstFragmentKind::Pat).make_pat(), - _ => unreachable!(), - } - }); - } - - fn flat_map_stmt(&mut self, stmt: ast::Stmt) -> SmallVec<[ast::Stmt; 1]> { - let mut stmt = configure!(self, stmt); - - // we'll expand attributes on expressions separately - if !stmt.is_expr() { - let (attr, derives, after_derive) = if stmt.is_item() { - self.classify_item(&mut stmt) - } else { - // ignore derives on non-item statements so it falls through - // to the unused-attributes lint - let (attr, after_derive) = self.classify_nonitem(&mut stmt); - (attr, vec![], after_derive) - }; - - if attr.is_some() || !derives.is_empty() { - return self.collect_attr(attr, derives, Annotatable::Stmt(P(stmt)), - AstFragmentKind::Stmts, after_derive).make_stmts(); - } - } - - if let StmtKind::Mac(mac) = stmt.kind { - let (mac, style, attrs) = mac.into_inner(); - self.check_attributes(&attrs); - let mut placeholder = self.collect_bang(mac, stmt.span, AstFragmentKind::Stmts) - .make_stmts(); - - // If this is a macro invocation with a semicolon, then apply that - // semicolon to the final statement produced by expansion. - if style == MacStmtStyle::Semicolon { - if let Some(stmt) = placeholder.pop() { - placeholder.push(stmt.add_trailing_semicolon()); - } - } - - return placeholder; - } - - // The placeholder expander gives ids to statements, so we avoid folding the id here. - let ast::Stmt { id, kind, span } = stmt; - noop_flat_map_stmt_kind(kind, self).into_iter().map(|kind| { - ast::Stmt { id, kind, span } - }).collect() - - } - - fn visit_block(&mut self, block: &mut P) { - let old_directory_ownership = self.cx.current_expansion.directory_ownership; - self.cx.current_expansion.directory_ownership = DirectoryOwnership::UnownedViaBlock; - noop_visit_block(block, self); - self.cx.current_expansion.directory_ownership = old_directory_ownership; - } - - fn flat_map_item(&mut self, item: P) -> SmallVec<[P; 1]> { - let mut item = configure!(self, item); - - let (attr, traits, after_derive) = self.classify_item(&mut item); - if attr.is_some() || !traits.is_empty() { - return self.collect_attr(attr, traits, Annotatable::Item(item), - AstFragmentKind::Items, after_derive).make_items(); - } - - match item.kind { - ast::ItemKind::Mac(..) => { - self.check_attributes(&item.attrs); - item.and_then(|item| match item.kind { - ItemKind::Mac(mac) => self.collect( - AstFragmentKind::Items, InvocationKind::Bang { mac, span: item.span } - ).make_items(), - _ => unreachable!(), - }) - } - ast::ItemKind::Mod(ast::Mod { inner, .. }) => { - if item.ident == Ident::invalid() { - return noop_flat_map_item(item, self); - } - - let orig_directory_ownership = self.cx.current_expansion.directory_ownership; - let mut module = (*self.cx.current_expansion.module).clone(); - module.mod_path.push(item.ident); - - // Detect if this is an inline module (`mod m { ... }` as opposed to `mod m;`). - // In the non-inline case, `inner` is never the dummy span (cf. `parse_item_mod`). - // Thus, if `inner` is the dummy span, we know the module is inline. - let inline_module = item.span.contains(inner) || inner.is_dummy(); - - if inline_module { - if let Some(path) = attr::first_attr_value_str_by_name(&item.attrs, sym::path) { - self.cx.current_expansion.directory_ownership = - DirectoryOwnership::Owned { relative: None }; - module.directory.push(&*path.as_str()); - } else { - module.directory.push(&*item.ident.as_str()); - } - } else { - let path = self.cx.parse_sess.source_map().span_to_unmapped_path(inner); - let mut path = match path { - FileName::Real(path) => path, - other => PathBuf::from(other.to_string()), - }; - let directory_ownership = match path.file_name().unwrap().to_str() { - Some("mod.rs") => DirectoryOwnership::Owned { relative: None }, - Some(_) => DirectoryOwnership::Owned { - relative: Some(item.ident), - }, - None => DirectoryOwnership::UnownedViaMod(false), - }; - path.pop(); - module.directory = path; - self.cx.current_expansion.directory_ownership = directory_ownership; - } - - let orig_module = - mem::replace(&mut self.cx.current_expansion.module, Rc::new(module)); - let result = noop_flat_map_item(item, self); - self.cx.current_expansion.module = orig_module; - self.cx.current_expansion.directory_ownership = orig_directory_ownership; - result - } - - _ => noop_flat_map_item(item, self), - } - } - - fn flat_map_trait_item(&mut self, item: ast::TraitItem) -> SmallVec<[ast::TraitItem; 1]> { - let mut item = configure!(self, item); - - let (attr, traits, after_derive) = self.classify_item(&mut item); - if attr.is_some() || !traits.is_empty() { - return self.collect_attr(attr, traits, Annotatable::TraitItem(P(item)), - AstFragmentKind::TraitItems, after_derive).make_trait_items() - } - - match item.kind { - ast::TraitItemKind::Macro(mac) => { - let ast::TraitItem { attrs, span, .. } = item; - self.check_attributes(&attrs); - self.collect_bang(mac, span, AstFragmentKind::TraitItems).make_trait_items() - } - _ => noop_flat_map_trait_item(item, self), - } - } - - fn flat_map_impl_item(&mut self, item: ast::ImplItem) -> SmallVec<[ast::ImplItem; 1]> { - let mut item = configure!(self, item); - - let (attr, traits, after_derive) = self.classify_item(&mut item); - if attr.is_some() || !traits.is_empty() { - return self.collect_attr(attr, traits, Annotatable::ImplItem(P(item)), - AstFragmentKind::ImplItems, after_derive).make_impl_items(); - } - - match item.kind { - ast::ImplItemKind::Macro(mac) => { - let ast::ImplItem { attrs, span, .. } = item; - self.check_attributes(&attrs); - self.collect_bang(mac, span, AstFragmentKind::ImplItems).make_impl_items() - } - _ => noop_flat_map_impl_item(item, self), - } - } - - fn visit_ty(&mut self, ty: &mut P) { - match ty.kind { - ast::TyKind::Mac(_) => {} - _ => return noop_visit_ty(ty, self), - }; - - visit_clobber(ty, |mut ty| { - match mem::replace(&mut ty.kind, ast::TyKind::Err) { - ast::TyKind::Mac(mac) => - self.collect_bang(mac, ty.span, AstFragmentKind::Ty).make_ty(), - _ => unreachable!(), - } - }); - } - - fn visit_foreign_mod(&mut self, foreign_mod: &mut ast::ForeignMod) { - self.cfg.configure_foreign_mod(foreign_mod); - noop_visit_foreign_mod(foreign_mod, self); - } - - fn flat_map_foreign_item(&mut self, mut foreign_item: ast::ForeignItem) - -> SmallVec<[ast::ForeignItem; 1]> - { - let (attr, traits, after_derive) = self.classify_item(&mut foreign_item); - - if attr.is_some() || !traits.is_empty() { - return self.collect_attr(attr, traits, Annotatable::ForeignItem(P(foreign_item)), - AstFragmentKind::ForeignItems, after_derive) - .make_foreign_items(); - } - - if let ast::ForeignItemKind::Macro(mac) = foreign_item.kind { - self.check_attributes(&foreign_item.attrs); - return self.collect_bang(mac, foreign_item.span, AstFragmentKind::ForeignItems) - .make_foreign_items(); - } - - noop_flat_map_foreign_item(foreign_item, self) - } - - fn visit_item_kind(&mut self, item: &mut ast::ItemKind) { - match item { - ast::ItemKind::MacroDef(..) => {} - _ => { - self.cfg.configure_item_kind(item); - noop_visit_item_kind(item, self); - } - } - } - - fn flat_map_generic_param( - &mut self, - param: ast::GenericParam - ) -> SmallVec<[ast::GenericParam; 1]> - { - let mut param = configure!(self, param); - - let (attr, traits, after_derive) = self.classify_item(&mut param); - if attr.is_some() || !traits.is_empty() { - return self.collect_attr(attr, traits, Annotatable::GenericParam(param), - AstFragmentKind::GenericParams, after_derive) - .make_generic_params(); - } - - noop_flat_map_generic_param(param, self) - } - - fn visit_attribute(&mut self, at: &mut ast::Attribute) { - // turn `#[doc(include="filename")]` attributes into `#[doc(include(file="filename", - // contents="file contents")]` attributes - if !at.check_name(sym::doc) { - return noop_visit_attribute(at, self); - } - - if let Some(list) = at.meta_item_list() { - if !list.iter().any(|it| it.check_name(sym::include)) { - return noop_visit_attribute(at, self); - } - - let mut items = vec![]; - - for mut it in list { - if !it.check_name(sym::include) { - items.push({ noop_visit_meta_list_item(&mut it, self); it }); - continue; - } - - if let Some(file) = it.value_str() { - let err_count = self.cx.parse_sess.span_diagnostic.err_count(); - self.check_attributes(slice::from_ref(at)); - if self.cx.parse_sess.span_diagnostic.err_count() > err_count { - // avoid loading the file if they haven't enabled the feature - return noop_visit_attribute(at, self); - } - - let filename = self.cx.resolve_path(&*file.as_str(), it.span()); - match self.cx.source_map().load_file(&filename) { - Ok(source_file) => { - let src = source_file.src.as_ref() - .expect("freshly loaded file should have a source"); - let src_interned = Symbol::intern(src.as_str()); - - let include_info = vec![ - ast::NestedMetaItem::MetaItem( - attr::mk_name_value_item_str( - Ident::with_dummy_span(sym::file), - file, - DUMMY_SP, - ), - ), - ast::NestedMetaItem::MetaItem( - attr::mk_name_value_item_str( - Ident::with_dummy_span(sym::contents), - src_interned, - DUMMY_SP, - ), - ), - ]; - - let include_ident = Ident::with_dummy_span(sym::include); - let item = attr::mk_list_item(include_ident, include_info); - items.push(ast::NestedMetaItem::MetaItem(item)); - } - Err(e) => { - let lit = it - .meta_item() - .and_then(|item| item.name_value_literal()) - .unwrap(); - - if e.kind() == ErrorKind::InvalidData { - self.cx - .struct_span_err( - lit.span, - &format!("{} wasn't a utf-8 file", filename.display()), - ) - .span_label(lit.span, "contains invalid utf-8") - .emit(); - } else { - let mut err = self.cx.struct_span_err( - lit.span, - &format!("couldn't read {}: {}", filename.display(), e), - ); - err.span_label(lit.span, "couldn't read file"); - - err.emit(); - } - } - } - } else { - let mut err = self.cx.struct_span_err( - it.span(), - &format!("expected path to external documentation"), - ); - - // Check if the user erroneously used `doc(include(...))` syntax. - let literal = it.meta_item_list().and_then(|list| { - if list.len() == 1 { - list[0].literal().map(|literal| &literal.kind) - } else { - None - } - }); - - let (path, applicability) = match &literal { - Some(LitKind::Str(path, ..)) => { - (path.to_string(), Applicability::MachineApplicable) - } - _ => (String::from(""), Applicability::HasPlaceholders), - }; - - err.span_suggestion( - it.span(), - "provide a file path with `=`", - format!("include = \"{}\"", path), - applicability, - ); - - err.emit(); - } - } - - let meta = attr::mk_list_item(Ident::with_dummy_span(sym::doc), items); - *at = attr::Attribute { - item: AttrItem { path: meta.path, tokens: meta.kind.tokens(meta.span) }, - span: at.span, - id: at.id, - style: at.style, - is_sugared_doc: false, - }; - } else { - noop_visit_attribute(at, self) - } - } - - fn visit_id(&mut self, id: &mut ast::NodeId) { - if self.monotonic { - debug_assert_eq!(*id, ast::DUMMY_NODE_ID); - *id = self.cx.resolver.next_node_id() - } - } - - fn visit_fn_decl(&mut self, mut fn_decl: &mut P) { - self.cfg.configure_fn_decl(&mut fn_decl); - noop_visit_fn_decl(fn_decl, self); - } -} - -pub struct ExpansionConfig<'feat> { - pub crate_name: String, - pub features: Option<&'feat Features>, - pub recursion_limit: usize, - pub trace_mac: bool, - pub should_test: bool, // If false, strip `#[test]` nodes - pub single_step: bool, - pub keep_macs: bool, -} - -impl<'feat> ExpansionConfig<'feat> { - pub fn default(crate_name: String) -> ExpansionConfig<'static> { - ExpansionConfig { - crate_name, - features: None, - recursion_limit: 1024, - trace_mac: false, - should_test: false, - single_step: false, - keep_macs: false, - } - } - - fn proc_macro_hygiene(&self) -> bool { - self.features.map_or(false, |features| features.proc_macro_hygiene) - } - fn custom_inner_attributes(&self) -> bool { - self.features.map_or(false, |features| features.custom_inner_attributes) - } -} diff --git a/src/libsyntax/ext/mbe.rs b/src/libsyntax/ext/mbe.rs deleted file mode 100644 index a87da791c9b..00000000000 --- a/src/libsyntax/ext/mbe.rs +++ /dev/null @@ -1,166 +0,0 @@ -//! This module implements declarative macros: old `macro_rules` and the newer -//! `macro`. Declarative macros are also known as "macro by example", and that's -//! why we call this module `mbe`. For external documentation, prefer the -//! official terminology: "declarative macros". - -crate mod transcribe; -crate mod macro_check; -crate mod macro_parser; -crate mod macro_rules; -crate mod quoted; - -use crate::ast; -use crate::parse::token::{self, Token, TokenKind}; -use crate::tokenstream::{DelimSpan}; - -use syntax_pos::{BytePos, Span}; - -use rustc_data_structures::sync::Lrc; - -/// Contains the sub-token-trees of a "delimited" token tree, such as the contents of `(`. Note -/// that the delimiter itself might be `NoDelim`. -#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug)] -struct Delimited { - delim: token::DelimToken, - tts: Vec, -} - -impl Delimited { - /// Returns a `self::TokenTree` with a `Span` corresponding to the opening delimiter. - fn open_tt(&self, span: Span) -> TokenTree { - let open_span = if span.is_dummy() { - span - } else { - span.with_hi(span.lo() + BytePos(self.delim.len() as u32)) - }; - TokenTree::token(token::OpenDelim(self.delim), open_span) - } - - /// Returns a `self::TokenTree` with a `Span` corresponding to the closing delimiter. - fn close_tt(&self, span: Span) -> TokenTree { - let close_span = if span.is_dummy() { - span - } else { - span.with_lo(span.hi() - BytePos(self.delim.len() as u32)) - }; - TokenTree::token(token::CloseDelim(self.delim), close_span) - } -} - -#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug)] -struct SequenceRepetition { - /// The sequence of token trees - tts: Vec, - /// The optional separator - separator: Option, - /// Whether the sequence can be repeated zero (*), or one or more times (+) - kleene: KleeneToken, - /// The number of `Match`s that appear in the sequence (and subsequences) - num_captures: usize, -} - -#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug, Copy)] -struct KleeneToken { - span: Span, - op: KleeneOp, -} - -impl KleeneToken { - fn new(op: KleeneOp, span: Span) -> KleeneToken { - KleeneToken { span, op } - } -} - -/// A Kleene-style [repetition operator](http://en.wikipedia.org/wiki/Kleene_star) -/// for token sequences. -#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] -enum KleeneOp { - /// Kleene star (`*`) for zero or more repetitions - ZeroOrMore, - /// Kleene plus (`+`) for one or more repetitions - OneOrMore, - /// Kleene optional (`?`) for zero or one reptitions - ZeroOrOne, -} - -/// Similar to `tokenstream::TokenTree`, except that `$i`, `$i:ident`, and `$(...)` -/// are "first-class" token trees. Useful for parsing macros. -#[derive(Debug, Clone, PartialEq, RustcEncodable, RustcDecodable)] -enum TokenTree { - Token(Token), - Delimited(DelimSpan, Lrc), - /// A kleene-style repetition sequence - Sequence(DelimSpan, Lrc), - /// e.g., `$var` - MetaVar(Span, ast::Ident), - /// e.g., `$var:expr`. This is only used in the left hand side of MBE macros. - MetaVarDecl( - Span, - ast::Ident, /* name to bind */ - ast::Ident, /* kind of nonterminal */ - ), -} - -impl TokenTree { - /// Return the number of tokens in the tree. - fn len(&self) -> usize { - match *self { - TokenTree::Delimited(_, ref delimed) => match delimed.delim { - token::NoDelim => delimed.tts.len(), - _ => delimed.tts.len() + 2, - }, - TokenTree::Sequence(_, ref seq) => seq.tts.len(), - _ => 0, - } - } - - /// Returns `true` if the given token tree is delimited. - fn is_delimited(&self) -> bool { - match *self { - TokenTree::Delimited(..) => true, - _ => false, - } - } - - /// Returns `true` if the given token tree is a token of the given kind. - fn is_token(&self, expected_kind: &TokenKind) -> bool { - match self { - TokenTree::Token(Token { kind: actual_kind, .. }) => actual_kind == expected_kind, - _ => false, - } - } - - /// Gets the `index`-th sub-token-tree. This only makes sense for delimited trees and sequences. - fn get_tt(&self, index: usize) -> TokenTree { - match (self, index) { - (&TokenTree::Delimited(_, ref delimed), _) if delimed.delim == token::NoDelim => { - delimed.tts[index].clone() - } - (&TokenTree::Delimited(span, ref delimed), _) => { - if index == 0 { - return delimed.open_tt(span.open); - } - if index == delimed.tts.len() + 1 { - return delimed.close_tt(span.close); - } - delimed.tts[index - 1].clone() - } - (&TokenTree::Sequence(_, ref seq), _) => seq.tts[index].clone(), - _ => panic!("Cannot expand a token tree"), - } - } - - /// Retrieves the `TokenTree`'s span. - fn span(&self) -> Span { - match *self { - TokenTree::Token(Token { span, .. }) - | TokenTree::MetaVar(span, _) - | TokenTree::MetaVarDecl(span, _, _) => span, - TokenTree::Delimited(span, _) | TokenTree::Sequence(span, _) => span.entire(), - } - } - - fn token(kind: TokenKind, span: Span) -> TokenTree { - TokenTree::Token(Token::new(kind, span)) - } -} diff --git a/src/libsyntax/ext/mbe/macro_check.rs b/src/libsyntax/ext/mbe/macro_check.rs deleted file mode 100644 index aabaff4e1c3..00000000000 --- a/src/libsyntax/ext/mbe/macro_check.rs +++ /dev/null @@ -1,626 +0,0 @@ -//! Checks that meta-variables in macro definition are correctly declared and used. -//! -//! # What is checked -//! -//! ## Meta-variables must not be bound twice -//! -//! ``` -//! macro_rules! foo { ($x:tt $x:tt) => { $x }; } -//! ``` -//! -//! This check is sound (no false-negative) and complete (no false-positive). -//! -//! ## Meta-variables must not be free -//! -//! ``` -//! macro_rules! foo { () => { $x }; } -//! ``` -//! -//! This check is also done at macro instantiation but only if the branch is taken. -//! -//! ## Meta-variables must repeat at least as many times as their binder -//! -//! ``` -//! macro_rules! foo { ($($x:tt)*) => { $x }; } -//! ``` -//! -//! This check is also done at macro instantiation but only if the branch is taken. -//! -//! ## Meta-variables must repeat with the same Kleene operators as their binder -//! -//! ``` -//! macro_rules! foo { ($($x:tt)+) => { $($x)* }; } -//! ``` -//! -//! This check is not done at macro instantiation. -//! -//! # Disclaimer -//! -//! In the presence of nested macros (a macro defined in a macro), those checks may have false -//! positives and false negatives. We try to detect those cases by recognizing potential macro -//! definitions in RHSes, but nested macros may be hidden through the use of particular values of -//! meta-variables. -//! -//! ## Examples of false positive -//! -//! False positives can come from cases where we don't recognize a nested macro, because it depends -//! on particular values of meta-variables. In the following example, we think both instances of -//! `$x` are free, which is a correct statement if `$name` is anything but `macro_rules`. But when -//! `$name` is `macro_rules`, like in the instantiation below, then `$x:tt` is actually a binder of -//! the nested macro and `$x` is bound to it. -//! -//! ``` -//! macro_rules! foo { ($name:ident) => { $name! bar { ($x:tt) => { $x }; } }; } -//! foo!(macro_rules); -//! ``` -//! -//! False positives can also come from cases where we think there is a nested macro while there -//! isn't. In the following example, we think `$x` is free, which is incorrect because `bar` is not -//! a nested macro since it is not evaluated as code by `stringify!`. -//! -//! ``` -//! macro_rules! foo { () => { stringify!(macro_rules! bar { () => { $x }; }) }; } -//! ``` -//! -//! ## Examples of false negative -//! -//! False negatives can come from cases where we don't recognize a meta-variable, because it depends -//! on particular values of meta-variables. In the following examples, we don't see that if `$d` is -//! instantiated with `$` then `$d z` becomes `$z` in the nested macro definition and is thus a free -//! meta-variable. Note however, that if `foo` is instantiated, then we would check the definition -//! of `bar` and would see the issue. -//! -//! ``` -//! macro_rules! foo { ($d:tt) => { macro_rules! bar { ($y:tt) => { $d z }; } }; } -//! ``` -//! -//! # How it is checked -//! -//! There are 3 main functions: `check_binders`, `check_occurrences`, and `check_nested_macro`. They -//! all need some kind of environment. -//! -//! ## Environments -//! -//! Environments are used to pass information. -//! -//! ### From LHS to RHS -//! -//! When checking a LHS with `check_binders`, we produce (and use) an environment for binders, -//! namely `Binders`. This is a mapping from binder name to information about that binder: the span -//! of the binder for error messages and the stack of Kleene operators under which it was bound in -//! the LHS. -//! -//! This environment is used by both the LHS and RHS. The LHS uses it to detect duplicate binders. -//! The RHS uses it to detect the other errors. -//! -//! ### From outer macro to inner macro -//! -//! When checking the RHS of an outer macro and we detect a nested macro definition, we push the -//! current state, namely `MacroState`, to an environment of nested macro definitions. Each state -//! stores the LHS binders when entering the macro definition as well as the stack of Kleene -//! operators under which the inner macro is defined in the RHS. -//! -//! This environment is a stack representing the nesting of macro definitions. As such, the stack of -//! Kleene operators under which a meta-variable is repeating is the concatenation of the stacks -//! stored when entering a macro definition starting from the state in which the meta-variable is -//! bound. -use crate::ast::NodeId; -use crate::early_buffered_lints::BufferedEarlyLintId; -use crate::ext::mbe::{KleeneToken, TokenTree}; -use crate::parse::token::TokenKind; -use crate::parse::token::{DelimToken, Token}; -use crate::sess::ParseSess; -use crate::symbol::{kw, sym}; - -use rustc_data_structures::fx::FxHashMap; -use smallvec::SmallVec; -use syntax_pos::{symbol::Ident, MultiSpan, Span}; - -/// Stack represented as linked list. -/// -/// Those are used for environments because they grow incrementally and are not mutable. -enum Stack<'a, T> { - /// Empty stack. - Empty, - /// A non-empty stack. - Push { - /// The top element. - top: T, - /// The previous elements. - prev: &'a Stack<'a, T>, - }, -} - -impl<'a, T> Stack<'a, T> { - /// Returns whether a stack is empty. - fn is_empty(&self) -> bool { - match *self { - Stack::Empty => true, - _ => false, - } - } - - /// Returns a new stack with an element of top. - fn push(&'a self, top: T) -> Stack<'a, T> { - Stack::Push { top, prev: self } - } -} - -impl<'a, T> Iterator for &'a Stack<'a, T> { - type Item = &'a T; - - // Iterates from top to bottom of the stack. - fn next(&mut self) -> Option<&'a T> { - match *self { - Stack::Empty => None, - Stack::Push { ref top, ref prev } => { - *self = prev; - Some(top) - } - } - } -} - -impl From<&Stack<'_, KleeneToken>> for SmallVec<[KleeneToken; 1]> { - fn from(ops: &Stack<'_, KleeneToken>) -> SmallVec<[KleeneToken; 1]> { - let mut ops: SmallVec<[KleeneToken; 1]> = ops.cloned().collect(); - // The stack is innermost on top. We want outermost first. - ops.reverse(); - ops - } -} - -/// Information attached to a meta-variable binder in LHS. -struct BinderInfo { - /// The span of the meta-variable in LHS. - span: Span, - /// The stack of Kleene operators (outermost first). - ops: SmallVec<[KleeneToken; 1]>, -} - -/// An environment of meta-variables to their binder information. -type Binders = FxHashMap; - -/// The state at which we entered a macro definition in the RHS of another macro definition. -struct MacroState<'a> { - /// The binders of the branch where we entered the macro definition. - binders: &'a Binders, - /// The stack of Kleene operators (outermost first) where we entered the macro definition. - ops: SmallVec<[KleeneToken; 1]>, -} - -/// Checks that meta-variables are used correctly in a macro definition. -/// -/// Arguments: -/// - `sess` is used to emit diagnostics and lints -/// - `node_id` is used to emit lints -/// - `span` is used when no spans are available -/// - `lhses` and `rhses` should have the same length and represent the macro definition -pub(super) fn check_meta_variables( - sess: &ParseSess, - node_id: NodeId, - span: Span, - lhses: &[TokenTree], - rhses: &[TokenTree], -) -> bool { - if lhses.len() != rhses.len() { - sess.span_diagnostic.span_bug(span, "length mismatch between LHSes and RHSes") - } - let mut valid = true; - for (lhs, rhs) in lhses.iter().zip(rhses.iter()) { - let mut binders = Binders::default(); - check_binders(sess, node_id, lhs, &Stack::Empty, &mut binders, &Stack::Empty, &mut valid); - check_occurrences(sess, node_id, rhs, &Stack::Empty, &binders, &Stack::Empty, &mut valid); - } - valid -} - -/// Checks `lhs` as part of the LHS of a macro definition, extends `binders` with new binders, and -/// sets `valid` to false in case of errors. -/// -/// Arguments: -/// - `sess` is used to emit diagnostics and lints -/// - `node_id` is used to emit lints -/// - `lhs` is checked as part of a LHS -/// - `macros` is the stack of possible outer macros -/// - `binders` contains the binders of the LHS -/// - `ops` is the stack of Kleene operators from the LHS -/// - `valid` is set in case of errors -fn check_binders( - sess: &ParseSess, - node_id: NodeId, - lhs: &TokenTree, - macros: &Stack<'_, MacroState<'_>>, - binders: &mut Binders, - ops: &Stack<'_, KleeneToken>, - valid: &mut bool, -) { - match *lhs { - TokenTree::Token(..) => {} - // This can only happen when checking a nested macro because this LHS is then in the RHS of - // the outer macro. See ui/macros/macro-of-higher-order.rs where $y:$fragment in the - // LHS of the nested macro (and RHS of the outer macro) is parsed as MetaVar(y) Colon - // MetaVar(fragment) and not as MetaVarDecl(y, fragment). - TokenTree::MetaVar(span, name) => { - if macros.is_empty() { - sess.span_diagnostic.span_bug(span, "unexpected MetaVar in lhs"); - } - // There are 3 possibilities: - if let Some(prev_info) = binders.get(&name) { - // 1. The meta-variable is already bound in the current LHS: This is an error. - let mut span = MultiSpan::from_span(span); - span.push_span_label(prev_info.span, "previous declaration".into()); - buffer_lint(sess, span, node_id, "duplicate matcher binding"); - } else if get_binder_info(macros, binders, name).is_none() { - // 2. The meta-variable is free: This is a binder. - binders.insert(name, BinderInfo { span, ops: ops.into() }); - } else { - // 3. The meta-variable is bound: This is an occurrence. - check_occurrences(sess, node_id, lhs, macros, binders, ops, valid); - } - } - // Similarly, this can only happen when checking a toplevel macro. - TokenTree::MetaVarDecl(span, name, _kind) => { - if !macros.is_empty() { - sess.span_diagnostic.span_bug(span, "unexpected MetaVarDecl in nested lhs"); - } - if let Some(prev_info) = get_binder_info(macros, binders, name) { - // Duplicate binders at the top-level macro definition are errors. The lint is only - // for nested macro definitions. - sess.span_diagnostic - .struct_span_err(span, "duplicate matcher binding") - .span_note(prev_info.span, "previous declaration was here") - .emit(); - *valid = false; - } else { - binders.insert(name, BinderInfo { span, ops: ops.into() }); - } - } - TokenTree::Delimited(_, ref del) => { - for tt in &del.tts { - check_binders(sess, node_id, tt, macros, binders, ops, valid); - } - } - TokenTree::Sequence(_, ref seq) => { - let ops = ops.push(seq.kleene); - for tt in &seq.tts { - check_binders(sess, node_id, tt, macros, binders, &ops, valid); - } - } - } -} - -/// Returns the binder information of a meta-variable. -/// -/// Arguments: -/// - `macros` is the stack of possible outer macros -/// - `binders` contains the current binders -/// - `name` is the name of the meta-variable we are looking for -fn get_binder_info<'a>( - mut macros: &'a Stack<'a, MacroState<'a>>, - binders: &'a Binders, - name: Ident, -) -> Option<&'a BinderInfo> { - binders.get(&name).or_else(|| macros.find_map(|state| state.binders.get(&name))) -} - -/// Checks `rhs` as part of the RHS of a macro definition and sets `valid` to false in case of -/// errors. -/// -/// Arguments: -/// - `sess` is used to emit diagnostics and lints -/// - `node_id` is used to emit lints -/// - `rhs` is checked as part of a RHS -/// - `macros` is the stack of possible outer macros -/// - `binders` contains the binders of the associated LHS -/// - `ops` is the stack of Kleene operators from the RHS -/// - `valid` is set in case of errors -fn check_occurrences( - sess: &ParseSess, - node_id: NodeId, - rhs: &TokenTree, - macros: &Stack<'_, MacroState<'_>>, - binders: &Binders, - ops: &Stack<'_, KleeneToken>, - valid: &mut bool, -) { - match *rhs { - TokenTree::Token(..) => {} - TokenTree::MetaVarDecl(span, _name, _kind) => { - sess.span_diagnostic.span_bug(span, "unexpected MetaVarDecl in rhs") - } - TokenTree::MetaVar(span, name) => { - check_ops_is_prefix(sess, node_id, macros, binders, ops, span, name); - } - TokenTree::Delimited(_, ref del) => { - check_nested_occurrences(sess, node_id, &del.tts, macros, binders, ops, valid); - } - TokenTree::Sequence(_, ref seq) => { - let ops = ops.push(seq.kleene); - check_nested_occurrences(sess, node_id, &seq.tts, macros, binders, &ops, valid); - } - } -} - -/// Represents the processed prefix of a nested macro. -#[derive(Clone, Copy, PartialEq, Eq)] -enum NestedMacroState { - /// Nothing that matches a nested macro definition was processed yet. - Empty, - /// The token `macro_rules` was processed. - MacroRules, - /// The tokens `macro_rules!` were processed. - MacroRulesNot, - /// The tokens `macro_rules!` followed by a name were processed. The name may be either directly - /// an identifier or a meta-variable (that hopefully would be instantiated by an identifier). - MacroRulesNotName, - /// The keyword `macro` was processed. - Macro, - /// The keyword `macro` followed by a name was processed. - MacroName, - /// The keyword `macro` followed by a name and a token delimited by parentheses was processed. - MacroNameParen, -} - -/// Checks `tts` as part of the RHS of a macro definition, tries to recognize nested macro -/// definitions, and sets `valid` to false in case of errors. -/// -/// Arguments: -/// - `sess` is used to emit diagnostics and lints -/// - `node_id` is used to emit lints -/// - `tts` is checked as part of a RHS and may contain macro definitions -/// - `macros` is the stack of possible outer macros -/// - `binders` contains the binders of the associated LHS -/// - `ops` is the stack of Kleene operators from the RHS -/// - `valid` is set in case of errors -fn check_nested_occurrences( - sess: &ParseSess, - node_id: NodeId, - tts: &[TokenTree], - macros: &Stack<'_, MacroState<'_>>, - binders: &Binders, - ops: &Stack<'_, KleeneToken>, - valid: &mut bool, -) { - let mut state = NestedMacroState::Empty; - let nested_macros = macros.push(MacroState { binders, ops: ops.into() }); - let mut nested_binders = Binders::default(); - for tt in tts { - match (state, tt) { - ( - NestedMacroState::Empty, - &TokenTree::Token(Token { kind: TokenKind::Ident(name, false), .. }), - ) => { - if name == sym::macro_rules { - state = NestedMacroState::MacroRules; - } else if name == kw::Macro { - state = NestedMacroState::Macro; - } - } - ( - NestedMacroState::MacroRules, - &TokenTree::Token(Token { kind: TokenKind::Not, .. }), - ) => { - state = NestedMacroState::MacroRulesNot; - } - ( - NestedMacroState::MacroRulesNot, - &TokenTree::Token(Token { kind: TokenKind::Ident(..), .. }), - ) => { - state = NestedMacroState::MacroRulesNotName; - } - (NestedMacroState::MacroRulesNot, &TokenTree::MetaVar(..)) => { - state = NestedMacroState::MacroRulesNotName; - // We check that the meta-variable is correctly used. - check_occurrences(sess, node_id, tt, macros, binders, ops, valid); - } - (NestedMacroState::MacroRulesNotName, &TokenTree::Delimited(_, ref del)) - | (NestedMacroState::MacroName, &TokenTree::Delimited(_, ref del)) - if del.delim == DelimToken::Brace => - { - let legacy = state == NestedMacroState::MacroRulesNotName; - state = NestedMacroState::Empty; - let rest = - check_nested_macro(sess, node_id, legacy, &del.tts, &nested_macros, valid); - // If we did not check the whole macro definition, then check the rest as if outside - // the macro definition. - check_nested_occurrences( - sess, - node_id, - &del.tts[rest..], - macros, - binders, - ops, - valid, - ); - } - ( - NestedMacroState::Macro, - &TokenTree::Token(Token { kind: TokenKind::Ident(..), .. }), - ) => { - state = NestedMacroState::MacroName; - } - (NestedMacroState::Macro, &TokenTree::MetaVar(..)) => { - state = NestedMacroState::MacroName; - // We check that the meta-variable is correctly used. - check_occurrences(sess, node_id, tt, macros, binders, ops, valid); - } - (NestedMacroState::MacroName, &TokenTree::Delimited(_, ref del)) - if del.delim == DelimToken::Paren => - { - state = NestedMacroState::MacroNameParen; - nested_binders = Binders::default(); - check_binders( - sess, - node_id, - tt, - &nested_macros, - &mut nested_binders, - &Stack::Empty, - valid, - ); - } - (NestedMacroState::MacroNameParen, &TokenTree::Delimited(_, ref del)) - if del.delim == DelimToken::Brace => - { - state = NestedMacroState::Empty; - check_occurrences( - sess, - node_id, - tt, - &nested_macros, - &nested_binders, - &Stack::Empty, - valid, - ); - } - (_, ref tt) => { - state = NestedMacroState::Empty; - check_occurrences(sess, node_id, tt, macros, binders, ops, valid); - } - } - } -} - -/// Checks the body of nested macro, returns where the check stopped, and sets `valid` to false in -/// case of errors. -/// -/// The token trees are checked as long as they look like a list of (LHS) => {RHS} token trees. This -/// check is a best-effort to detect a macro definition. It returns the position in `tts` where we -/// stopped checking because we detected we were not in a macro definition anymore. -/// -/// Arguments: -/// - `sess` is used to emit diagnostics and lints -/// - `node_id` is used to emit lints -/// - `legacy` specifies whether the macro is legacy -/// - `tts` is checked as a list of (LHS) => {RHS} -/// - `macros` is the stack of outer macros -/// - `valid` is set in case of errors -fn check_nested_macro( - sess: &ParseSess, - node_id: NodeId, - legacy: bool, - tts: &[TokenTree], - macros: &Stack<'_, MacroState<'_>>, - valid: &mut bool, -) -> usize { - let n = tts.len(); - let mut i = 0; - let separator = if legacy { TokenKind::Semi } else { TokenKind::Comma }; - loop { - // We expect 3 token trees: `(LHS) => {RHS}`. The separator is checked after. - if i + 2 >= n - || !tts[i].is_delimited() - || !tts[i + 1].is_token(&TokenKind::FatArrow) - || !tts[i + 2].is_delimited() - { - break; - } - let lhs = &tts[i]; - let rhs = &tts[i + 2]; - let mut binders = Binders::default(); - check_binders(sess, node_id, lhs, macros, &mut binders, &Stack::Empty, valid); - check_occurrences(sess, node_id, rhs, macros, &binders, &Stack::Empty, valid); - // Since the last semicolon is optional for legacy macros and decl_macro are not terminated, - // we increment our checked position by how many token trees we already checked (the 3 - // above) before checking for the separator. - i += 3; - if i == n || !tts[i].is_token(&separator) { - break; - } - // We increment our checked position for the semicolon. - i += 1; - } - i -} - -/// Checks that a meta-variable occurrence is valid. -/// -/// Arguments: -/// - `sess` is used to emit diagnostics and lints -/// - `node_id` is used to emit lints -/// - `macros` is the stack of possible outer macros -/// - `binders` contains the binders of the associated LHS -/// - `ops` is the stack of Kleene operators from the RHS -/// - `span` is the span of the meta-variable to check -/// - `name` is the name of the meta-variable to check -fn check_ops_is_prefix( - sess: &ParseSess, - node_id: NodeId, - macros: &Stack<'_, MacroState<'_>>, - binders: &Binders, - ops: &Stack<'_, KleeneToken>, - span: Span, - name: Ident, -) { - let macros = macros.push(MacroState { binders, ops: ops.into() }); - // Accumulates the stacks the operators of each state until (and including when) the - // meta-variable is found. The innermost stack is first. - let mut acc: SmallVec<[&SmallVec<[KleeneToken; 1]>; 1]> = SmallVec::new(); - for state in ¯os { - acc.push(&state.ops); - if let Some(binder) = state.binders.get(&name) { - // This variable concatenates the stack of operators from the RHS of the LHS where the - // meta-variable was defined to where it is used (in possibly nested macros). The - // outermost operator is first. - let mut occurrence_ops: SmallVec<[KleeneToken; 2]> = SmallVec::new(); - // We need to iterate from the end to start with outermost stack. - for ops in acc.iter().rev() { - occurrence_ops.extend_from_slice(ops); - } - ops_is_prefix(sess, node_id, span, name, &binder.ops, &occurrence_ops); - return; - } - } - buffer_lint(sess, span.into(), node_id, &format!("unknown macro variable `{}`", name)); -} - -/// Returns whether `binder_ops` is a prefix of `occurrence_ops`. -/// -/// The stack of Kleene operators of a meta-variable occurrence just needs to have the stack of -/// Kleene operators of its binder as a prefix. -/// -/// Consider $i in the following example: -/// -/// ( $( $i:ident = $($j:ident),+ );* ) => { $($( $i += $j; )+)* } -/// -/// It occurs under the Kleene stack ["*", "+"] and is bound under ["*"] only. -/// -/// Arguments: -/// - `sess` is used to emit diagnostics and lints -/// - `node_id` is used to emit lints -/// - `span` is the span of the meta-variable being check -/// - `name` is the name of the meta-variable being check -/// - `binder_ops` is the stack of Kleene operators for the binder -/// - `occurrence_ops` is the stack of Kleene operators for the occurrence -fn ops_is_prefix( - sess: &ParseSess, - node_id: NodeId, - span: Span, - name: Ident, - binder_ops: &[KleeneToken], - occurrence_ops: &[KleeneToken], -) { - for (i, binder) in binder_ops.iter().enumerate() { - if i >= occurrence_ops.len() { - let mut span = MultiSpan::from_span(span); - span.push_span_label(binder.span, "expected repetition".into()); - let message = &format!("variable '{}' is still repeating at this depth", name); - buffer_lint(sess, span, node_id, message); - return; - } - let occurrence = &occurrence_ops[i]; - if occurrence.op != binder.op { - let mut span = MultiSpan::from_span(span); - span.push_span_label(binder.span, "expected repetition".into()); - span.push_span_label(occurrence.span, "conflicting repetition".into()); - let message = "meta-variable repeats with different Kleene operator"; - buffer_lint(sess, span, node_id, message); - return; - } - } -} - -fn buffer_lint(sess: &ParseSess, span: MultiSpan, node_id: NodeId, message: &str) { - sess.buffer_lint(BufferedEarlyLintId::MetaVariableMisuse, span, node_id, message); -} diff --git a/src/libsyntax/ext/mbe/macro_parser.rs b/src/libsyntax/ext/mbe/macro_parser.rs deleted file mode 100644 index ff382c316ff..00000000000 --- a/src/libsyntax/ext/mbe/macro_parser.rs +++ /dev/null @@ -1,943 +0,0 @@ -//! This is an NFA-based parser, which calls out to the main rust parser for named non-terminals -//! (which it commits to fully when it hits one in a grammar). There's a set of current NFA threads -//! and a set of next ones. Instead of NTs, we have a special case for Kleene star. The big-O, in -//! pathological cases, is worse than traditional use of NFA or Earley parsing, but it's an easier -//! fit for Macro-by-Example-style rules. -//! -//! (In order to prevent the pathological case, we'd need to lazily construct the resulting -//! `NamedMatch`es at the very end. It'd be a pain, and require more memory to keep around old -//! items, but it would also save overhead) -//! -//! We don't say this parser uses the Earley algorithm, because it's unnecessarily inaccurate. -//! The macro parser restricts itself to the features of finite state automata. Earley parsers -//! can be described as an extension of NFAs with completion rules, prediction rules, and recursion. -//! -//! Quick intro to how the parser works: -//! -//! A 'position' is a dot in the middle of a matcher, usually represented as a -//! dot. For example `· a $( a )* a b` is a position, as is `a $( · a )* a b`. -//! -//! The parser walks through the input a character at a time, maintaining a list -//! of threads consistent with the current position in the input string: `cur_items`. -//! -//! As it processes them, it fills up `eof_items` with threads that would be valid if -//! the macro invocation is now over, `bb_items` with threads that are waiting on -//! a Rust non-terminal like `$e:expr`, and `next_items` with threads that are waiting -//! on a particular token. Most of the logic concerns moving the · through the -//! repetitions indicated by Kleene stars. The rules for moving the · without -//! consuming any input are called epsilon transitions. It only advances or calls -//! out to the real Rust parser when no `cur_items` threads remain. -//! -//! Example: -//! -//! ```text, ignore -//! Start parsing a a a a b against [· a $( a )* a b]. -//! -//! Remaining input: a a a a b -//! next: [· a $( a )* a b] -//! -//! - - - Advance over an a. - - - -//! -//! Remaining input: a a a b -//! cur: [a · $( a )* a b] -//! Descend/Skip (first item). -//! next: [a $( · a )* a b] [a $( a )* · a b]. -//! -//! - - - Advance over an a. - - - -//! -//! Remaining input: a a b -//! cur: [a $( a · )* a b] [a $( a )* a · b] -//! Follow epsilon transition: Finish/Repeat (first item) -//! next: [a $( a )* · a b] [a $( · a )* a b] [a $( a )* a · b] -//! -//! - - - Advance over an a. - - - (this looks exactly like the last step) -//! -//! Remaining input: a b -//! cur: [a $( a · )* a b] [a $( a )* a · b] -//! Follow epsilon transition: Finish/Repeat (first item) -//! next: [a $( a )* · a b] [a $( · a )* a b] [a $( a )* a · b] -//! -//! - - - Advance over an a. - - - (this looks exactly like the last step) -//! -//! Remaining input: b -//! cur: [a $( a · )* a b] [a $( a )* a · b] -//! Follow epsilon transition: Finish/Repeat (first item) -//! next: [a $( a )* · a b] [a $( · a )* a b] [a $( a )* a · b] -//! -//! - - - Advance over a b. - - - -//! -//! Remaining input: '' -//! eof: [a $( a )* a b ·] -//! ``` - -crate use NamedMatch::*; -crate use ParseResult::*; -use TokenTreeOrTokenTreeSlice::*; - -use crate::ast::{Ident, Name}; -use crate::ext::mbe::{self, TokenTree}; -use crate::parse::{Directory, PResult}; -use crate::parse::parser::{Parser, PathStyle}; -use crate::parse::token::{self, DocComment, Nonterminal, Token}; -use crate::print::pprust; -use crate::sess::ParseSess; -use crate::symbol::{kw, sym, Symbol}; -use crate::tokenstream::{DelimSpan, TokenStream}; - -use errors::FatalError; -use smallvec::{smallvec, SmallVec}; -use syntax_pos::Span; - -use rustc_data_structures::fx::FxHashMap; -use rustc_data_structures::sync::Lrc; -use std::collections::hash_map::Entry::{Occupied, Vacant}; -use std::mem; -use std::ops::{Deref, DerefMut}; - -// To avoid costly uniqueness checks, we require that `MatchSeq` always has a nonempty body. - -/// Either a sequence of token trees or a single one. This is used as the representation of the -/// sequence of tokens that make up a matcher. -#[derive(Clone)] -enum TokenTreeOrTokenTreeSlice<'tt> { - Tt(TokenTree), - TtSeq(&'tt [TokenTree]), -} - -impl<'tt> TokenTreeOrTokenTreeSlice<'tt> { - /// Returns the number of constituent top-level token trees of `self` (top-level in that it - /// will not recursively descend into subtrees). - fn len(&self) -> usize { - match *self { - TtSeq(ref v) => v.len(), - Tt(ref tt) => tt.len(), - } - } - - /// The `index`-th token tree of `self`. - fn get_tt(&self, index: usize) -> TokenTree { - match *self { - TtSeq(ref v) => v[index].clone(), - Tt(ref tt) => tt.get_tt(index), - } - } -} - -/// An unzipping of `TokenTree`s... see the `stack` field of `MatcherPos`. -/// -/// This is used by `inner_parse_loop` to keep track of delimited submatchers that we have -/// descended into. -#[derive(Clone)] -struct MatcherTtFrame<'tt> { - /// The "parent" matcher that we are descending into. - elts: TokenTreeOrTokenTreeSlice<'tt>, - /// The position of the "dot" in `elts` at the time we descended. - idx: usize, -} - -type NamedMatchVec = SmallVec<[NamedMatch; 4]>; - -/// Represents a single "position" (aka "matcher position", aka "item"), as -/// described in the module documentation. -/// -/// Here: -/// -/// - `'root` represents the lifetime of the stack slot that holds the root -/// `MatcherPos`. As described in `MatcherPosHandle`, the root `MatcherPos` -/// structure is stored on the stack, but subsequent instances are put into -/// the heap. -/// - `'tt` represents the lifetime of the token trees that this matcher -/// position refers to. -/// -/// It is important to distinguish these two lifetimes because we have a -/// `SmallVec>` below, and the destructor of -/// that is considered to possibly access the data from its elements (it lacks -/// a `#[may_dangle]` attribute). As a result, the compiler needs to know that -/// all the elements in that `SmallVec` strictly outlive the root stack slot -/// lifetime. By separating `'tt` from `'root`, we can show that. -#[derive(Clone)] -struct MatcherPos<'root, 'tt> { - /// The token or sequence of tokens that make up the matcher - top_elts: TokenTreeOrTokenTreeSlice<'tt>, - - /// The position of the "dot" in this matcher - idx: usize, - - /// The first span of source that the beginning of this matcher corresponds to. In other - /// words, the token in the source whose span is `sp_open` is matched against the first token of - /// the matcher. - sp_open: Span, - - /// For each named metavar in the matcher, we keep track of token trees matched against the - /// metavar by the black box parser. In particular, there may be more than one match per - /// metavar if we are in a repetition (each repetition matches each of the variables). - /// Moreover, matchers and repetitions can be nested; the `matches` field is shared (hence the - /// `Rc`) among all "nested" matchers. `match_lo`, `match_cur`, and `match_hi` keep track of - /// the current position of the `self` matcher position in the shared `matches` list. - /// - /// Also, note that while we are descending into a sequence, matchers are given their own - /// `matches` vector. Only once we reach the end of a full repetition of the sequence do we add - /// all bound matches from the submatcher into the shared top-level `matches` vector. If `sep` - /// and `up` are `Some`, then `matches` is _not_ the shared top-level list. Instead, if one - /// wants the shared `matches`, one should use `up.matches`. - matches: Box<[Lrc]>, - /// The position in `matches` corresponding to the first metavar in this matcher's sequence of - /// token trees. In other words, the first metavar in the first token of `top_elts` corresponds - /// to `matches[match_lo]`. - match_lo: usize, - /// The position in `matches` corresponding to the metavar we are currently trying to match - /// against the source token stream. `match_lo <= match_cur <= match_hi`. - match_cur: usize, - /// Similar to `match_lo` except `match_hi` is the position in `matches` of the _last_ metavar - /// in this matcher. - match_hi: usize, - - // The following fields are used if we are matching a repetition. If we aren't, they should be - // `None`. - - /// The KleeneOp of this sequence if we are in a repetition. - seq_op: Option, - - /// The separator if we are in a repetition. - sep: Option, - - /// The "parent" matcher position if we are in a repetition. That is, the matcher position just - /// before we enter the sequence. - up: Option>, - - /// Specifically used to "unzip" token trees. By "unzip", we mean to unwrap the delimiters from - /// a delimited token tree (e.g., something wrapped in `(` `)`) or to get the contents of a doc - /// comment... - /// - /// When matching against matchers with nested delimited submatchers (e.g., `pat ( pat ( .. ) - /// pat ) pat`), we need to keep track of the matchers we are descending into. This stack does - /// that where the bottom of the stack is the outermost matcher. - /// Also, throughout the comments, this "descent" is often referred to as "unzipping"... - stack: SmallVec<[MatcherTtFrame<'tt>; 1]>, -} - -impl<'root, 'tt> MatcherPos<'root, 'tt> { - /// Adds `m` as a named match for the `idx`-th metavar. - fn push_match(&mut self, idx: usize, m: NamedMatch) { - let matches = Lrc::make_mut(&mut self.matches[idx]); - matches.push(m); - } -} - -// Lots of MatcherPos instances are created at runtime. Allocating them on the -// heap is slow. Furthermore, using SmallVec to allocate them all -// on the stack is also slow, because MatcherPos is quite a large type and -// instances get moved around a lot between vectors, which requires lots of -// slow memcpy calls. -// -// Therefore, the initial MatcherPos is always allocated on the stack, -// subsequent ones (of which there aren't that many) are allocated on the heap, -// and this type is used to encapsulate both cases. -enum MatcherPosHandle<'root, 'tt> { - Ref(&'root mut MatcherPos<'root, 'tt>), - Box(Box>), -} - -impl<'root, 'tt> Clone for MatcherPosHandle<'root, 'tt> { - // This always produces a new Box. - fn clone(&self) -> Self { - MatcherPosHandle::Box(match *self { - MatcherPosHandle::Ref(ref r) => Box::new((**r).clone()), - MatcherPosHandle::Box(ref b) => b.clone(), - }) - } -} - -impl<'root, 'tt> Deref for MatcherPosHandle<'root, 'tt> { - type Target = MatcherPos<'root, 'tt>; - fn deref(&self) -> &Self::Target { - match *self { - MatcherPosHandle::Ref(ref r) => r, - MatcherPosHandle::Box(ref b) => b, - } - } -} - -impl<'root, 'tt> DerefMut for MatcherPosHandle<'root, 'tt> { - fn deref_mut(&mut self) -> &mut MatcherPos<'root, 'tt> { - match *self { - MatcherPosHandle::Ref(ref mut r) => r, - MatcherPosHandle::Box(ref mut b) => b, - } - } -} - -/// Represents the possible results of an attempted parse. -crate enum ParseResult { - /// Parsed successfully. - Success(T), - /// Arm failed to match. If the second parameter is `token::Eof`, it indicates an unexpected - /// end of macro invocation. Otherwise, it indicates that no rules expected the given token. - Failure(Token, &'static str), - /// Fatal error (malformed macro?). Abort compilation. - Error(syntax_pos::Span, String), -} - -/// A `ParseResult` where the `Success` variant contains a mapping of `Ident`s to `NamedMatch`es. -/// This represents the mapping of metavars to the token trees they bind to. -crate type NamedParseResult = ParseResult>; - -/// Count how many metavars are named in the given matcher `ms`. -pub(super) fn count_names(ms: &[TokenTree]) -> usize { - ms.iter().fold(0, |count, elt| { - count + match *elt { - TokenTree::Sequence(_, ref seq) => seq.num_captures, - TokenTree::Delimited(_, ref delim) => count_names(&delim.tts), - TokenTree::MetaVar(..) => 0, - TokenTree::MetaVarDecl(..) => 1, - TokenTree::Token(..) => 0, - } - }) -} - -/// `len` `Vec`s (initially shared and empty) that will store matches of metavars. -fn create_matches(len: usize) -> Box<[Lrc]> { - if len == 0 { - vec![] - } else { - let empty_matches = Lrc::new(SmallVec::new()); - vec![empty_matches; len] - }.into_boxed_slice() -} - -/// Generates the top-level matcher position in which the "dot" is before the first token of the -/// matcher `ms` and we are going to start matching at the span `open` in the source. -fn initial_matcher_pos<'root, 'tt>(ms: &'tt [TokenTree], open: Span) -> MatcherPos<'root, 'tt> { - let match_idx_hi = count_names(ms); - let matches = create_matches(match_idx_hi); - MatcherPos { - // Start with the top level matcher given to us - top_elts: TtSeq(ms), // "elts" is an abbr. for "elements" - // The "dot" is before the first token of the matcher - idx: 0, - // We start matching at the span `open` in the source code - sp_open: open, - - // Initialize `matches` to a bunch of empty `Vec`s -- one for each metavar in `top_elts`. - // `match_lo` for `top_elts` is 0 and `match_hi` is `matches.len()`. `match_cur` is 0 since - // we haven't actually matched anything yet. - matches, - match_lo: 0, - match_cur: 0, - match_hi: match_idx_hi, - - // Haven't descended into any delimiters, so empty stack - stack: smallvec![], - - // Haven't descended into any sequences, so both of these are `None`. - seq_op: None, - sep: None, - up: None, - } -} - -/// `NamedMatch` is a pattern-match result for a single `token::MATCH_NONTERMINAL`: -/// so it is associated with a single ident in a parse, and all -/// `MatchedNonterminal`s in the `NamedMatch` have the same non-terminal type -/// (expr, item, etc). Each leaf in a single `NamedMatch` corresponds to a -/// single `token::MATCH_NONTERMINAL` in the `TokenTree` that produced it. -/// -/// The in-memory structure of a particular `NamedMatch` represents the match -/// that occurred when a particular subset of a matcher was applied to a -/// particular token tree. -/// -/// The width of each `MatchedSeq` in the `NamedMatch`, and the identity of -/// the `MatchedNonterminal`s, will depend on the token tree it was applied -/// to: each `MatchedSeq` corresponds to a single `TTSeq` in the originating -/// token tree. The depth of the `NamedMatch` structure will therefore depend -/// only on the nesting depth of `ast::TTSeq`s in the originating -/// token tree it was derived from. -#[derive(Debug, Clone)] -crate enum NamedMatch { - MatchedSeq(Lrc, DelimSpan), - MatchedNonterminal(Lrc), -} - -/// Takes a sequence of token trees `ms` representing a matcher which successfully matched input -/// and an iterator of items that matched input and produces a `NamedParseResult`. -fn nameize>( - sess: &ParseSess, - ms: &[TokenTree], - mut res: I, -) -> NamedParseResult { - // Recursively descend into each type of matcher (e.g., sequences, delimited, metavars) and make - // sure that each metavar has _exactly one_ binding. If a metavar does not have exactly one - // binding, then there is an error. If it does, then we insert the binding into the - // `NamedParseResult`. - fn n_rec>( - sess: &ParseSess, - m: &TokenTree, - res: &mut I, - ret_val: &mut FxHashMap, - ) -> Result<(), (syntax_pos::Span, String)> { - match *m { - TokenTree::Sequence(_, ref seq) => for next_m in &seq.tts { - n_rec(sess, next_m, res.by_ref(), ret_val)? - }, - TokenTree::Delimited(_, ref delim) => for next_m in &delim.tts { - n_rec(sess, next_m, res.by_ref(), ret_val)?; - }, - TokenTree::MetaVarDecl(span, _, id) if id.name == kw::Invalid => { - if sess.missing_fragment_specifiers.borrow_mut().remove(&span) { - return Err((span, "missing fragment specifier".to_string())); - } - } - TokenTree::MetaVarDecl(sp, bind_name, _) => { - match ret_val.entry(bind_name) { - Vacant(spot) => { - spot.insert(res.next().unwrap()); - } - Occupied(..) => { - return Err((sp, format!("duplicated bind name: {}", bind_name))) - } - } - } - TokenTree::MetaVar(..) | TokenTree::Token(..) => (), - } - - Ok(()) - } - - let mut ret_val = FxHashMap::default(); - for m in ms { - match n_rec(sess, m, res.by_ref(), &mut ret_val) { - Ok(_) => {} - Err((sp, msg)) => return Error(sp, msg), - } - } - - Success(ret_val) -} - -/// Performs a token equality check, ignoring syntax context (that is, an unhygienic comparison) -fn token_name_eq(t1: &Token, t2: &Token) -> bool { - if let (Some((ident1, is_raw1)), Some((ident2, is_raw2))) = (t1.ident(), t2.ident()) { - ident1.name == ident2.name && is_raw1 == is_raw2 - } else if let (Some(ident1), Some(ident2)) = (t1.lifetime(), t2.lifetime()) { - ident1.name == ident2.name - } else { - t1.kind == t2.kind - } -} - -/// Process the matcher positions of `cur_items` until it is empty. In the process, this will -/// produce more items in `next_items`, `eof_items`, and `bb_items`. -/// -/// For more info about the how this happens, see the module-level doc comments and the inline -/// comments of this function. -/// -/// # Parameters -/// -/// - `sess`: the parsing session into which errors are emitted. -/// - `cur_items`: the set of current items to be processed. This should be empty by the end of a -/// successful execution of this function. -/// - `next_items`: the set of newly generated items. These are used to replenish `cur_items` in -/// the function `parse`. -/// - `eof_items`: the set of items that would be valid if this was the EOF. -/// - `bb_items`: the set of items that are waiting for the black-box parser. -/// - `token`: the current token of the parser. -/// - `span`: the `Span` in the source code corresponding to the token trees we are trying to match -/// against the matcher positions in `cur_items`. -/// -/// # Returns -/// -/// A `ParseResult`. Note that matches are kept track of through the items generated. -fn inner_parse_loop<'root, 'tt>( - sess: &ParseSess, - cur_items: &mut SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>, - next_items: &mut Vec>, - eof_items: &mut SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>, - bb_items: &mut SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>, - token: &Token, -) -> ParseResult<()> { - // Pop items from `cur_items` until it is empty. - while let Some(mut item) = cur_items.pop() { - // When unzipped trees end, remove them. This corresponds to backtracking out of a - // delimited submatcher into which we already descended. In backtracking out again, we need - // to advance the "dot" past the delimiters in the outer matcher. - while item.idx >= item.top_elts.len() { - match item.stack.pop() { - Some(MatcherTtFrame { elts, idx }) => { - item.top_elts = elts; - item.idx = idx + 1; - } - None => break, - } - } - - // Get the current position of the "dot" (`idx`) in `item` and the number of token trees in - // the matcher (`len`). - let idx = item.idx; - let len = item.top_elts.len(); - - // If `idx >= len`, then we are at or past the end of the matcher of `item`. - if idx >= len { - // We are repeating iff there is a parent. If the matcher is inside of a repetition, - // then we could be at the end of a sequence or at the beginning of the next - // repetition. - if item.up.is_some() { - // At this point, regardless of whether there is a separator, we should add all - // matches from the complete repetition of the sequence to the shared, top-level - // `matches` list (actually, `up.matches`, which could itself not be the top-level, - // but anyway...). Moreover, we add another item to `cur_items` in which the "dot" - // is at the end of the `up` matcher. This ensures that the "dot" in the `up` - // matcher is also advanced sufficiently. - // - // NOTE: removing the condition `idx == len` allows trailing separators. - if idx == len { - // Get the `up` matcher - let mut new_pos = item.up.clone().unwrap(); - - // Add matches from this repetition to the `matches` of `up` - for idx in item.match_lo..item.match_hi { - let sub = item.matches[idx].clone(); - let span = DelimSpan::from_pair(item.sp_open, token.span); - new_pos.push_match(idx, MatchedSeq(sub, span)); - } - - // Move the "dot" past the repetition in `up` - new_pos.match_cur = item.match_hi; - new_pos.idx += 1; - cur_items.push(new_pos); - } - - // Check if we need a separator. - if idx == len && item.sep.is_some() { - // We have a separator, and it is the current token. We can advance past the - // separator token. - if item.sep - .as_ref() - .map(|sep| token_name_eq(token, sep)) - .unwrap_or(false) - { - item.idx += 1; - next_items.push(item); - } - } - // We don't need a separator. Move the "dot" back to the beginning of the matcher - // and try to match again UNLESS we are only allowed to have _one_ repetition. - else if item.seq_op != Some(mbe::KleeneOp::ZeroOrOne) { - item.match_cur = item.match_lo; - item.idx = 0; - cur_items.push(item); - } - } - // If we are not in a repetition, then being at the end of a matcher means that we have - // reached the potential end of the input. - else { - eof_items.push(item); - } - } - // We are in the middle of a matcher. - else { - // Look at what token in the matcher we are trying to match the current token (`token`) - // against. Depending on that, we may generate new items. - match item.top_elts.get_tt(idx) { - // Need to descend into a sequence - TokenTree::Sequence(sp, seq) => { - // Examine the case where there are 0 matches of this sequence. We are - // implicitly disallowing OneOrMore from having 0 matches here. Thus, that will - // result in a "no rules expected token" error by virtue of this matcher not - // working. - if seq.kleene.op == mbe::KleeneOp::ZeroOrMore - || seq.kleene.op == mbe::KleeneOp::ZeroOrOne - { - let mut new_item = item.clone(); - new_item.match_cur += seq.num_captures; - new_item.idx += 1; - for idx in item.match_cur..item.match_cur + seq.num_captures { - new_item.push_match(idx, MatchedSeq(Lrc::new(smallvec![]), sp)); - } - cur_items.push(new_item); - } - - let matches = create_matches(item.matches.len()); - cur_items.push(MatcherPosHandle::Box(Box::new(MatcherPos { - stack: smallvec![], - sep: seq.separator.clone(), - seq_op: Some(seq.kleene.op), - idx: 0, - matches, - match_lo: item.match_cur, - match_cur: item.match_cur, - match_hi: item.match_cur + seq.num_captures, - up: Some(item), - sp_open: sp.open, - top_elts: Tt(TokenTree::Sequence(sp, seq)), - }))); - } - - // We need to match a metavar (but the identifier is invalid)... this is an error - TokenTree::MetaVarDecl(span, _, id) if id.name == kw::Invalid => { - if sess.missing_fragment_specifiers.borrow_mut().remove(&span) { - return Error(span, "missing fragment specifier".to_string()); - } - } - - // We need to match a metavar with a valid ident... call out to the black-box - // parser by adding an item to `bb_items`. - TokenTree::MetaVarDecl(_, _, id) => { - // Built-in nonterminals never start with these tokens, - // so we can eliminate them from consideration. - if may_begin_with(token, id.name) { - bb_items.push(item); - } - } - - // We need to descend into a delimited submatcher or a doc comment. To do this, we - // push the current matcher onto a stack and push a new item containing the - // submatcher onto `cur_items`. - // - // At the beginning of the loop, if we reach the end of the delimited submatcher, - // we pop the stack to backtrack out of the descent. - seq @ TokenTree::Delimited(..) | - seq @ TokenTree::Token(Token { kind: DocComment(..), .. }) => { - let lower_elts = mem::replace(&mut item.top_elts, Tt(seq)); - let idx = item.idx; - item.stack.push(MatcherTtFrame { - elts: lower_elts, - idx, - }); - item.idx = 0; - cur_items.push(item); - } - - // We just matched a normal token. We can just advance the parser. - TokenTree::Token(t) if token_name_eq(&t, token) => { - item.idx += 1; - next_items.push(item); - } - - // There was another token that was not `token`... This means we can't add any - // rules. NOTE that this is not necessarily an error unless _all_ items in - // `cur_items` end up doing this. There may still be some other matchers that do - // end up working out. - TokenTree::Token(..) | TokenTree::MetaVar(..) => {} - } - } - } - - // Yay a successful parse (so far)! - Success(()) -} - -/// Use the given sequence of token trees (`ms`) as a matcher. Match the given token stream `tts` -/// against it and return the match. -/// -/// # Parameters -/// -/// - `sess`: The session into which errors are emitted -/// - `tts`: The tokenstream we are matching against the pattern `ms` -/// - `ms`: A sequence of token trees representing a pattern against which we are matching -/// - `directory`: Information about the file locations (needed for the black-box parser) -/// - `recurse_into_modules`: Whether or not to recurse into modules (needed for the black-box -/// parser) -pub(super) fn parse( - sess: &ParseSess, - tts: TokenStream, - ms: &[TokenTree], - directory: Option>, - recurse_into_modules: bool, -) -> NamedParseResult { - // Create a parser that can be used for the "black box" parts. - let mut parser = Parser::new( - sess, - tts, - directory, - recurse_into_modules, - true, - crate::MACRO_ARGUMENTS, - ); - - // A queue of possible matcher positions. We initialize it with the matcher position in which - // the "dot" is before the first token of the first token tree in `ms`. `inner_parse_loop` then - // processes all of these possible matcher positions and produces possible next positions into - // `next_items`. After some post-processing, the contents of `next_items` replenish `cur_items` - // and we start over again. - // - // This MatcherPos instance is allocated on the stack. All others -- and - // there are frequently *no* others! -- are allocated on the heap. - let mut initial = initial_matcher_pos(ms, parser.token.span); - let mut cur_items = smallvec![MatcherPosHandle::Ref(&mut initial)]; - let mut next_items = Vec::new(); - - loop { - // Matcher positions black-box parsed by parser.rs (`parser`) - let mut bb_items = SmallVec::new(); - - // Matcher positions that would be valid if the macro invocation was over now - let mut eof_items = SmallVec::new(); - assert!(next_items.is_empty()); - - // Process `cur_items` until either we have finished the input or we need to get some - // parsing from the black-box parser done. The result is that `next_items` will contain a - // bunch of possible next matcher positions in `next_items`. - match inner_parse_loop( - sess, - &mut cur_items, - &mut next_items, - &mut eof_items, - &mut bb_items, - &parser.token, - ) { - Success(_) => {} - Failure(token, msg) => return Failure(token, msg), - Error(sp, msg) => return Error(sp, msg), - } - - // inner parse loop handled all cur_items, so it's empty - assert!(cur_items.is_empty()); - - // We need to do some post processing after the `inner_parser_loop`. - // - // Error messages here could be improved with links to original rules. - - // If we reached the EOF, check that there is EXACTLY ONE possible matcher. Otherwise, - // either the parse is ambiguous (which should never happen) or there is a syntax error. - if parser.token == token::Eof { - if eof_items.len() == 1 { - let matches = eof_items[0] - .matches - .iter_mut() - .map(|dv| Lrc::make_mut(dv).pop().unwrap()); - return nameize(sess, ms, matches); - } else if eof_items.len() > 1 { - return Error( - parser.token.span, - "ambiguity: multiple successful parses".to_string(), - ); - } else { - return Failure( - Token::new(token::Eof, if parser.token.span.is_dummy() { - parser.token.span - } else { - sess.source_map().next_point(parser.token.span) - }), - "missing tokens in macro arguments", - ); - } - } - // Performance hack: eof_items may share matchers via Rc with other things that we want - // to modify. Dropping eof_items now may drop these refcounts to 1, preventing an - // unnecessary implicit clone later in Rc::make_mut. - drop(eof_items); - - // Another possibility is that we need to call out to parse some rust nonterminal - // (black-box) parser. However, if there is not EXACTLY ONE of these, something is wrong. - if (!bb_items.is_empty() && !next_items.is_empty()) || bb_items.len() > 1 { - let nts = bb_items - .iter() - .map(|item| match item.top_elts.get_tt(item.idx) { - TokenTree::MetaVarDecl(_, bind, name) => format!("{} ('{}')", name, bind), - _ => panic!(), - }) - .collect::>() - .join(" or "); - - return Error( - parser.token.span, - format!( - "local ambiguity: multiple parsing options: {}", - match next_items.len() { - 0 => format!("built-in NTs {}.", nts), - 1 => format!("built-in NTs {} or 1 other option.", nts), - n => format!("built-in NTs {} or {} other options.", nts, n), - } - ), - ); - } - // If there are no possible next positions AND we aren't waiting for the black-box parser, - // then there is a syntax error. - else if bb_items.is_empty() && next_items.is_empty() { - return Failure( - parser.token.take(), - "no rules expected this token in macro call", - ); - } - // Dump all possible `next_items` into `cur_items` for the next iteration. - else if !next_items.is_empty() { - // Now process the next token - cur_items.extend(next_items.drain(..)); - parser.bump(); - } - // Finally, we have the case where we need to call the black-box parser to get some - // nonterminal. - else { - assert_eq!(bb_items.len(), 1); - - let mut item = bb_items.pop().unwrap(); - if let TokenTree::MetaVarDecl(span, _, ident) = item.top_elts.get_tt(item.idx) { - let match_cur = item.match_cur; - item.push_match( - match_cur, - MatchedNonterminal(Lrc::new(parse_nt(&mut parser, span, ident.name))), - ); - item.idx += 1; - item.match_cur += 1; - } else { - unreachable!() - } - cur_items.push(item); - } - - assert!(!cur_items.is_empty()); - } -} - -/// The token is an identifier, but not `_`. -/// We prohibit passing `_` to macros expecting `ident` for now. -fn get_macro_name(token: &Token) -> Option<(Name, bool)> { - match token.kind { - token::Ident(name, is_raw) if name != kw::Underscore => Some((name, is_raw)), - _ => None, - } -} - -/// Checks whether a non-terminal may begin with a particular token. -/// -/// Returning `false` is a *stability guarantee* that such a matcher will *never* begin with that -/// token. Be conservative (return true) if not sure. -fn may_begin_with(token: &Token, name: Name) -> bool { - /// Checks whether the non-terminal may contain a single (non-keyword) identifier. - fn may_be_ident(nt: &token::Nonterminal) -> bool { - match *nt { - token::NtItem(_) | token::NtBlock(_) | token::NtVis(_) => false, - _ => true, - } - } - - match name { - sym::expr => token.can_begin_expr() - // This exception is here for backwards compatibility. - && !token.is_keyword(kw::Let), - sym::ty => token.can_begin_type(), - sym::ident => get_macro_name(token).is_some(), - sym::literal => token.can_begin_literal_or_bool(), - sym::vis => match token.kind { - // The follow-set of :vis + "priv" keyword + interpolated - token::Comma | token::Ident(..) | token::Interpolated(_) => true, - _ => token.can_begin_type(), - }, - sym::block => match token.kind { - token::OpenDelim(token::Brace) => true, - token::Interpolated(ref nt) => match **nt { - token::NtItem(_) - | token::NtPat(_) - | token::NtTy(_) - | token::NtIdent(..) - | token::NtMeta(_) - | token::NtPath(_) - | token::NtVis(_) => false, // none of these may start with '{'. - _ => true, - }, - _ => false, - }, - sym::path | sym::meta => match token.kind { - token::ModSep | token::Ident(..) => true, - token::Interpolated(ref nt) => match **nt { - token::NtPath(_) | token::NtMeta(_) => true, - _ => may_be_ident(&nt), - }, - _ => false, - }, - sym::pat => match token.kind { - token::Ident(..) | // box, ref, mut, and other identifiers (can stricten) - token::OpenDelim(token::Paren) | // tuple pattern - token::OpenDelim(token::Bracket) | // slice pattern - token::BinOp(token::And) | // reference - token::BinOp(token::Minus) | // negative literal - token::AndAnd | // double reference - token::Literal(..) | // literal - token::DotDot | // range pattern (future compat) - token::DotDotDot | // range pattern (future compat) - token::ModSep | // path - token::Lt | // path (UFCS constant) - token::BinOp(token::Shl) => true, // path (double UFCS) - token::Interpolated(ref nt) => may_be_ident(nt), - _ => false, - }, - sym::lifetime => match token.kind { - token::Lifetime(_) => true, - token::Interpolated(ref nt) => match **nt { - token::NtLifetime(_) | token::NtTT(_) => true, - _ => false, - }, - _ => false, - }, - _ => match token.kind { - token::CloseDelim(_) => false, - _ => true, - }, - } -} - -/// A call to the "black-box" parser to parse some Rust non-terminal. -/// -/// # Parameters -/// -/// - `p`: the "black-box" parser to use -/// - `sp`: the `Span` we want to parse -/// - `name`: the name of the metavar _matcher_ we want to match (e.g., `tt`, `ident`, `block`, -/// etc...) -/// -/// # Returns -/// -/// The parsed non-terminal. -fn parse_nt(p: &mut Parser<'_>, sp: Span, name: Symbol) -> Nonterminal { - if name == sym::tt { - return token::NtTT(p.parse_token_tree()); - } - // check at the beginning and the parser checks after each bump - p.process_potential_macro_variable(); - match parse_nt_inner(p, sp, name) { - Ok(nt) => nt, - Err(mut err) => { - err.emit(); - FatalError.raise(); - } - } -} - -fn parse_nt_inner<'a>(p: &mut Parser<'a>, sp: Span, name: Symbol) -> PResult<'a, Nonterminal> { - Ok(match name { - sym::item => match p.parse_item()? { - Some(i) => token::NtItem(i), - None => return Err(p.fatal("expected an item keyword")), - }, - sym::block => token::NtBlock(p.parse_block()?), - sym::stmt => match p.parse_stmt()? { - Some(s) => token::NtStmt(s), - None => return Err(p.fatal("expected a statement")), - }, - sym::pat => token::NtPat(p.parse_pat(None)?), - sym::expr => token::NtExpr(p.parse_expr()?), - sym::literal => token::NtLiteral(p.parse_literal_maybe_minus()?), - sym::ty => token::NtTy(p.parse_ty()?), - // this could be handled like a token, since it is one - sym::ident => if let Some((name, is_raw)) = get_macro_name(&p.token) { - let span = p.token.span; - p.bump(); - token::NtIdent(Ident::new(name, span), is_raw) - } else { - let token_str = pprust::token_to_string(&p.token); - return Err(p.fatal(&format!("expected ident, found {}", &token_str))); - } - sym::path => token::NtPath(p.parse_path(PathStyle::Type)?), - sym::meta => token::NtMeta(p.parse_attr_item()?), - sym::vis => token::NtVis(p.parse_visibility(true)?), - sym::lifetime => if p.check_lifetime() { - token::NtLifetime(p.expect_lifetime().ident) - } else { - let token_str = pprust::token_to_string(&p.token); - return Err(p.fatal(&format!("expected a lifetime, found `{}`", &token_str))); - } - // this is not supposed to happen, since it has been checked - // when compiling the macro. - _ => p.span_bug(sp, "invalid fragment specifier"), - }) -} diff --git a/src/libsyntax/ext/mbe/macro_rules.rs b/src/libsyntax/ext/mbe/macro_rules.rs deleted file mode 100644 index b4223298422..00000000000 --- a/src/libsyntax/ext/mbe/macro_rules.rs +++ /dev/null @@ -1,1194 +0,0 @@ -use crate::ast; -use crate::attr::{self, TransparencyError}; -use crate::edition::Edition; -use crate::ext::base::{DummyResult, ExtCtxt, MacResult, TTMacroExpander}; -use crate::ext::base::{SyntaxExtension, SyntaxExtensionKind}; -use crate::ext::expand::{AstFragment, AstFragmentKind}; -use crate::ext::mbe; -use crate::ext::mbe::macro_check; -use crate::ext::mbe::macro_parser::parse; -use crate::ext::mbe::macro_parser::{Error, Failure, Success}; -use crate::ext::mbe::macro_parser::{MatchedNonterminal, MatchedSeq, NamedParseResult}; -use crate::ext::mbe::transcribe::transcribe; -use crate::feature_gate::Features; -use crate::parse::parser::Parser; -use crate::parse::token::TokenKind::*; -use crate::parse::token::{self, NtTT, Token}; -use crate::parse::Directory; -use crate::print::pprust; -use crate::sess::ParseSess; -use crate::symbol::{kw, sym, Symbol}; -use crate::tokenstream::{DelimSpan, TokenStream, TokenTree}; - -use errors::{DiagnosticBuilder, FatalError}; -use log::debug; -use syntax_pos::hygiene::Transparency; -use syntax_pos::Span; - -use rustc_data_structures::fx::FxHashMap; -use std::borrow::Cow; -use std::collections::hash_map::Entry; -use std::slice; - -use errors::Applicability; -use rustc_data_structures::sync::Lrc; - -const VALID_FRAGMENT_NAMES_MSG: &str = "valid fragment specifiers are \ - `ident`, `block`, `stmt`, `expr`, `pat`, `ty`, `lifetime`, \ - `literal`, `path`, `meta`, `tt`, `item` and `vis`"; - -crate struct ParserAnyMacro<'a> { - parser: Parser<'a>, - - /// Span of the expansion site of the macro this parser is for - site_span: Span, - /// The ident of the macro we're parsing - macro_ident: ast::Ident, - arm_span: Span, -} - -crate fn annotate_err_with_kind( - err: &mut DiagnosticBuilder<'_>, - kind: AstFragmentKind, - span: Span, -) { - match kind { - AstFragmentKind::Ty => { - err.span_label(span, "this macro call doesn't expand to a type"); - } - AstFragmentKind::Pat => { - err.span_label(span, "this macro call doesn't expand to a pattern"); - } - _ => {} - }; -} - -impl<'a> ParserAnyMacro<'a> { - crate fn make(mut self: Box>, kind: AstFragmentKind) -> AstFragment { - let ParserAnyMacro { site_span, macro_ident, ref mut parser, arm_span } = *self; - let fragment = panictry!(parser.parse_ast_fragment(kind, true).map_err(|mut e| { - if parser.token == token::Eof && e.message().ends_with(", found ``") { - if !e.span.is_dummy() { - // early end of macro arm (#52866) - e.replace_span_with(parser.sess.source_map().next_point(parser.token.span)); - } - let msg = &e.message[0]; - e.message[0] = ( - format!( - "macro expansion ends with an incomplete expression: {}", - msg.0.replace(", found ``", ""), - ), - msg.1, - ); - } - if e.span.is_dummy() { - // Get around lack of span in error (#30128) - e.replace_span_with(site_span); - if parser.sess.source_map().span_to_filename(arm_span).is_real() { - e.span_label(arm_span, "in this macro arm"); - } - } else if !parser.sess.source_map().span_to_filename(parser.token.span).is_real() { - e.span_label(site_span, "in this macro invocation"); - } - match kind { - AstFragmentKind::Pat if macro_ident.name == sym::vec => { - let mut suggestion = None; - if let Ok(code) = parser.sess.source_map().span_to_snippet(site_span) { - if let Some(bang) = code.find('!') { - suggestion = Some(code[bang + 1..].to_string()); - } - } - if let Some(suggestion) = suggestion { - e.span_suggestion( - site_span, - "use a slice pattern here instead", - suggestion, - Applicability::MachineApplicable, - ); - } else { - e.span_label( - site_span, - "use a slice pattern here instead", - ); - } - e.help("for more information, see https://doc.rust-lang.org/edition-guide/\ - rust-2018/slice-patterns.html"); - } - _ => annotate_err_with_kind(&mut e, kind, site_span), - }; - e - })); - - // We allow semicolons at the end of expressions -- e.g., the semicolon in - // `macro_rules! m { () => { panic!(); } }` isn't parsed by `.parse_expr()`, - // but `m!()` is allowed in expression positions (cf. issue #34706). - if kind == AstFragmentKind::Expr && parser.token == token::Semi { - parser.bump(); - } - - // Make sure we don't have any tokens left to parse so we don't silently drop anything. - let path = ast::Path::from_ident(macro_ident.with_span_pos(site_span)); - parser.ensure_complete_parse(&path, kind.name(), site_span); - fragment - } -} - -struct MacroRulesMacroExpander { - name: ast::Ident, - span: Span, - transparency: Transparency, - lhses: Vec, - rhses: Vec, - valid: bool, -} - -impl TTMacroExpander for MacroRulesMacroExpander { - fn expand<'cx>( - &self, - cx: &'cx mut ExtCtxt<'_>, - sp: Span, - input: TokenStream, - ) -> Box { - if !self.valid { - return DummyResult::any(sp); - } - generic_extension( - cx, sp, self.span, self.name, self.transparency, input, &self.lhses, &self.rhses - ) - } -} - -fn trace_macros_note(cx: &mut ExtCtxt<'_>, sp: Span, message: String) { - let sp = sp.macro_backtrace().last().map(|trace| trace.call_site).unwrap_or(sp); - cx.expansions.entry(sp).or_default().push(message); -} - -/// Given `lhses` and `rhses`, this is the new macro we create -fn generic_extension<'cx>( - cx: &'cx mut ExtCtxt<'_>, - sp: Span, - def_span: Span, - name: ast::Ident, - transparency: Transparency, - arg: TokenStream, - lhses: &[mbe::TokenTree], - rhses: &[mbe::TokenTree], -) -> Box { - if cx.trace_macros() { - let msg = format!("expanding `{}! {{ {} }}`", name, pprust::tts_to_string(arg.clone())); - trace_macros_note(cx, sp, msg); - } - - // Which arm's failure should we report? (the one furthest along) - let mut best_failure: Option<(Token, &str)> = None; - - for (i, lhs) in lhses.iter().enumerate() { - // try each arm's matchers - let lhs_tt = match *lhs { - mbe::TokenTree::Delimited(_, ref delim) => &delim.tts[..], - _ => cx.span_bug(sp, "malformed macro lhs"), - }; - - match TokenTree::parse(cx, lhs_tt, arg.clone()) { - Success(named_matches) => { - let rhs = match rhses[i] { - // ignore delimiters - mbe::TokenTree::Delimited(_, ref delimed) => delimed.tts.clone(), - _ => cx.span_bug(sp, "malformed macro rhs"), - }; - let arm_span = rhses[i].span(); - - let rhs_spans = rhs.iter().map(|t| t.span()).collect::>(); - // rhs has holes ( `$id` and `$(...)` that need filled) - let mut tts = transcribe(cx, &named_matches, rhs, transparency); - - // Replace all the tokens for the corresponding positions in the macro, to maintain - // proper positions in error reporting, while maintaining the macro_backtrace. - if rhs_spans.len() == tts.len() { - tts = tts.map_enumerated(|i, mut tt| { - let mut sp = rhs_spans[i]; - sp = sp.with_ctxt(tt.span().ctxt()); - tt.set_span(sp); - tt - }); - } - - if cx.trace_macros() { - let msg = format!("to `{}`", pprust::tts_to_string(tts.clone())); - trace_macros_note(cx, sp, msg); - } - - let directory = Directory { - path: Cow::from(cx.current_expansion.module.directory.as_path()), - ownership: cx.current_expansion.directory_ownership, - }; - let mut p = Parser::new(cx.parse_sess(), tts, Some(directory), true, false, None); - p.root_module_name = - cx.current_expansion.module.mod_path.last().map(|id| id.as_str().to_string()); - p.last_type_ascription = cx.current_expansion.prior_type_ascription; - - p.process_potential_macro_variable(); - // Let the context choose how to interpret the result. - // Weird, but useful for X-macros. - return Box::new(ParserAnyMacro { - parser: p, - - // Pass along the original expansion site and the name of the macro - // so we can print a useful error message if the parse of the expanded - // macro leaves unparsed tokens. - site_span: sp, - macro_ident: name, - arm_span, - }); - } - Failure(token, msg) => match best_failure { - Some((ref best_token, _)) if best_token.span.lo() >= token.span.lo() => {} - _ => best_failure = Some((token, msg)), - }, - Error(err_sp, ref msg) => cx.span_fatal(err_sp.substitute_dummy(sp), &msg[..]), - } - } - - let (token, label) = best_failure.expect("ran no matchers"); - let span = token.span.substitute_dummy(sp); - let mut err = cx.struct_span_err(span, &parse_failure_msg(&token)); - err.span_label(span, label); - if !def_span.is_dummy() && cx.source_map().span_to_filename(def_span).is_real() { - err.span_label(cx.source_map().def_span(def_span), "when calling this macro"); - } - - // Check whether there's a missing comma in this macro call, like `println!("{}" a);` - if let Some((arg, comma_span)) = arg.add_comma() { - for lhs in lhses { - // try each arm's matchers - let lhs_tt = match *lhs { - mbe::TokenTree::Delimited(_, ref delim) => &delim.tts[..], - _ => continue, - }; - match TokenTree::parse(cx, lhs_tt, arg.clone()) { - Success(_) => { - if comma_span.is_dummy() { - err.note("you might be missing a comma"); - } else { - err.span_suggestion_short( - comma_span, - "missing comma here", - ", ".to_string(), - Applicability::MachineApplicable, - ); - } - } - _ => {} - } - } - } - err.emit(); - cx.trace_macros_diag(); - DummyResult::any(sp) -} - -// Note that macro-by-example's input is also matched against a token tree: -// $( $lhs:tt => $rhs:tt );+ -// -// Holy self-referential! - -/// Converts a macro item into a syntax extension. -pub fn compile_declarative_macro( - sess: &ParseSess, - features: &Features, - def: &ast::Item, - edition: Edition, -) -> SyntaxExtension { - let diag = &sess.span_diagnostic; - let lhs_nm = ast::Ident::new(sym::lhs, def.span); - let rhs_nm = ast::Ident::new(sym::rhs, def.span); - let tt_spec = ast::Ident::new(sym::tt, def.span); - - // Parse the macro_rules! invocation - let body = match def.kind { - ast::ItemKind::MacroDef(ref body) => body, - _ => unreachable!(), - }; - - // The pattern that macro_rules matches. - // The grammar for macro_rules! is: - // $( $lhs:tt => $rhs:tt );+ - // ...quasiquoting this would be nice. - // These spans won't matter, anyways - let argument_gram = vec![ - mbe::TokenTree::Sequence( - DelimSpan::dummy(), - Lrc::new(mbe::SequenceRepetition { - tts: vec![ - mbe::TokenTree::MetaVarDecl(def.span, lhs_nm, tt_spec), - mbe::TokenTree::token(token::FatArrow, def.span), - mbe::TokenTree::MetaVarDecl(def.span, rhs_nm, tt_spec), - ], - separator: Some(Token::new( - if body.legacy { token::Semi } else { token::Comma }, - def.span, - )), - kleene: mbe::KleeneToken::new(mbe::KleeneOp::OneOrMore, def.span), - num_captures: 2, - }), - ), - // to phase into semicolon-termination instead of semicolon-separation - mbe::TokenTree::Sequence( - DelimSpan::dummy(), - Lrc::new(mbe::SequenceRepetition { - tts: vec![mbe::TokenTree::token( - if body.legacy { token::Semi } else { token::Comma }, - def.span, - )], - separator: None, - kleene: mbe::KleeneToken::new(mbe::KleeneOp::ZeroOrMore, def.span), - num_captures: 0, - }), - ), - ]; - - let argument_map = match parse(sess, body.stream(), &argument_gram, None, true) { - Success(m) => m, - Failure(token, msg) => { - let s = parse_failure_msg(&token); - let sp = token.span.substitute_dummy(def.span); - let mut err = sess.span_diagnostic.struct_span_fatal(sp, &s); - err.span_label(sp, msg); - err.emit(); - FatalError.raise(); - } - Error(sp, s) => { - sess.span_diagnostic.span_fatal(sp.substitute_dummy(def.span), &s).raise(); - } - }; - - let mut valid = true; - - // Extract the arguments: - let lhses = match argument_map[&lhs_nm] { - MatchedSeq(ref s, _) => s - .iter() - .map(|m| { - if let MatchedNonterminal(ref nt) = *m { - if let NtTT(ref tt) = **nt { - let tt = mbe::quoted::parse( - tt.clone().into(), - true, - sess, - ) - .pop() - .unwrap(); - valid &= check_lhs_nt_follows(sess, features, &def.attrs, &tt); - return tt; - } - } - sess.span_diagnostic.span_bug(def.span, "wrong-structured lhs") - }) - .collect::>(), - _ => sess.span_diagnostic.span_bug(def.span, "wrong-structured lhs"), - }; - - let rhses = match argument_map[&rhs_nm] { - MatchedSeq(ref s, _) => s - .iter() - .map(|m| { - if let MatchedNonterminal(ref nt) = *m { - if let NtTT(ref tt) = **nt { - return mbe::quoted::parse( - tt.clone().into(), - false, - sess, - ) - .pop() - .unwrap(); - } - } - sess.span_diagnostic.span_bug(def.span, "wrong-structured lhs") - }) - .collect::>(), - _ => sess.span_diagnostic.span_bug(def.span, "wrong-structured rhs"), - }; - - for rhs in &rhses { - valid &= check_rhs(sess, rhs); - } - - // don't abort iteration early, so that errors for multiple lhses can be reported - for lhs in &lhses { - valid &= check_lhs_no_empty_seq(sess, slice::from_ref(lhs)); - } - - // We use CRATE_NODE_ID instead of `def.id` otherwise we may emit buffered lints for a node id - // that is not lint-checked and trigger the "failed to process buffered lint here" bug. - valid &= macro_check::check_meta_variables(sess, ast::CRATE_NODE_ID, def.span, &lhses, &rhses); - - let (transparency, transparency_error) = attr::find_transparency(&def.attrs, body.legacy); - match transparency_error { - Some(TransparencyError::UnknownTransparency(value, span)) => - diag.span_err(span, &format!("unknown macro transparency: `{}`", value)), - Some(TransparencyError::MultipleTransparencyAttrs(old_span, new_span)) => - diag.span_err(vec![old_span, new_span], "multiple macro transparency attributes"), - None => {} - } - - let expander: Box<_> = Box::new(MacroRulesMacroExpander { - name: def.ident, span: def.span, transparency, lhses, rhses, valid - }); - - SyntaxExtension::new( - sess, - SyntaxExtensionKind::LegacyBang(expander), - def.span, - Vec::new(), - edition, - def.ident.name, - &def.attrs, - ) -} - -fn check_lhs_nt_follows( - sess: &ParseSess, - features: &Features, - attrs: &[ast::Attribute], - lhs: &mbe::TokenTree, -) -> bool { - // lhs is going to be like TokenTree::Delimited(...), where the - // entire lhs is those tts. Or, it can be a "bare sequence", not wrapped in parens. - if let mbe::TokenTree::Delimited(_, ref tts) = *lhs { - check_matcher(sess, features, attrs, &tts.tts) - } else { - let msg = "invalid macro matcher; matchers must be contained in balanced delimiters"; - sess.span_diagnostic.span_err(lhs.span(), msg); - false - } - // we don't abort on errors on rejection, the driver will do that for us - // after parsing/expansion. we can report every error in every macro this way. -} - -/// Checks that the lhs contains no repetition which could match an empty token -/// tree, because then the matcher would hang indefinitely. -fn check_lhs_no_empty_seq(sess: &ParseSess, tts: &[mbe::TokenTree]) -> bool { - use mbe::TokenTree; - for tt in tts { - match *tt { - TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => (), - TokenTree::Delimited(_, ref del) => { - if !check_lhs_no_empty_seq(sess, &del.tts) { - return false; - } - } - TokenTree::Sequence(span, ref seq) => { - if seq.separator.is_none() - && seq.tts.iter().all(|seq_tt| match *seq_tt { - TokenTree::MetaVarDecl(_, _, id) => id.name == sym::vis, - TokenTree::Sequence(_, ref sub_seq) => { - sub_seq.kleene.op == mbe::KleeneOp::ZeroOrMore - || sub_seq.kleene.op == mbe::KleeneOp::ZeroOrOne - } - _ => false, - }) - { - let sp = span.entire(); - sess.span_diagnostic.span_err(sp, "repetition matches empty token tree"); - return false; - } - if !check_lhs_no_empty_seq(sess, &seq.tts) { - return false; - } - } - } - } - - true -} - -fn check_rhs(sess: &ParseSess, rhs: &mbe::TokenTree) -> bool { - match *rhs { - mbe::TokenTree::Delimited(..) => return true, - _ => sess.span_diagnostic.span_err(rhs.span(), "macro rhs must be delimited"), - } - false -} - -fn check_matcher( - sess: &ParseSess, - features: &Features, - attrs: &[ast::Attribute], - matcher: &[mbe::TokenTree], -) -> bool { - let first_sets = FirstSets::new(matcher); - let empty_suffix = TokenSet::empty(); - let err = sess.span_diagnostic.err_count(); - check_matcher_core(sess, features, attrs, &first_sets, matcher, &empty_suffix); - err == sess.span_diagnostic.err_count() -} - -// `The FirstSets` for a matcher is a mapping from subsequences in the -// matcher to the FIRST set for that subsequence. -// -// This mapping is partially precomputed via a backwards scan over the -// token trees of the matcher, which provides a mapping from each -// repetition sequence to its *first* set. -// -// (Hypothetically, sequences should be uniquely identifiable via their -// spans, though perhaps that is false, e.g., for macro-generated macros -// that do not try to inject artificial span information. My plan is -// to try to catch such cases ahead of time and not include them in -// the precomputed mapping.) -struct FirstSets { - // this maps each TokenTree::Sequence `$(tt ...) SEP OP` that is uniquely identified by its - // span in the original matcher to the First set for the inner sequence `tt ...`. - // - // If two sequences have the same span in a matcher, then map that - // span to None (invalidating the mapping here and forcing the code to - // use a slow path). - first: FxHashMap>, -} - -impl FirstSets { - fn new(tts: &[mbe::TokenTree]) -> FirstSets { - use mbe::TokenTree; - - let mut sets = FirstSets { first: FxHashMap::default() }; - build_recur(&mut sets, tts); - return sets; - - // walks backward over `tts`, returning the FIRST for `tts` - // and updating `sets` at the same time for all sequence - // substructure we find within `tts`. - fn build_recur(sets: &mut FirstSets, tts: &[TokenTree]) -> TokenSet { - let mut first = TokenSet::empty(); - for tt in tts.iter().rev() { - match *tt { - TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => { - first.replace_with(tt.clone()); - } - TokenTree::Delimited(span, ref delimited) => { - build_recur(sets, &delimited.tts[..]); - first.replace_with(delimited.open_tt(span.open)); - } - TokenTree::Sequence(sp, ref seq_rep) => { - let subfirst = build_recur(sets, &seq_rep.tts[..]); - - match sets.first.entry(sp.entire()) { - Entry::Vacant(vac) => { - vac.insert(Some(subfirst.clone())); - } - Entry::Occupied(mut occ) => { - // if there is already an entry, then a span must have collided. - // This should not happen with typical macro_rules macros, - // but syntax extensions need not maintain distinct spans, - // so distinct syntax trees can be assigned the same span. - // In such a case, the map cannot be trusted; so mark this - // entry as unusable. - occ.insert(None); - } - } - - // If the sequence contents can be empty, then the first - // token could be the separator token itself. - - if let (Some(sep), true) = (&seq_rep.separator, subfirst.maybe_empty) { - first.add_one_maybe(TokenTree::Token(sep.clone())); - } - - // Reverse scan: Sequence comes before `first`. - if subfirst.maybe_empty - || seq_rep.kleene.op == mbe::KleeneOp::ZeroOrMore - || seq_rep.kleene.op == mbe::KleeneOp::ZeroOrOne - { - // If sequence is potentially empty, then - // union them (preserving first emptiness). - first.add_all(&TokenSet { maybe_empty: true, ..subfirst }); - } else { - // Otherwise, sequence guaranteed - // non-empty; replace first. - first = subfirst; - } - } - } - } - - first - } - } - - // walks forward over `tts` until all potential FIRST tokens are - // identified. - fn first(&self, tts: &[mbe::TokenTree]) -> TokenSet { - use mbe::TokenTree; - - let mut first = TokenSet::empty(); - for tt in tts.iter() { - assert!(first.maybe_empty); - match *tt { - TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => { - first.add_one(tt.clone()); - return first; - } - TokenTree::Delimited(span, ref delimited) => { - first.add_one(delimited.open_tt(span.open)); - return first; - } - TokenTree::Sequence(sp, ref seq_rep) => { - let subfirst_owned; - let subfirst = match self.first.get(&sp.entire()) { - Some(&Some(ref subfirst)) => subfirst, - Some(&None) => { - subfirst_owned = self.first(&seq_rep.tts[..]); - &subfirst_owned - } - None => { - panic!("We missed a sequence during FirstSets construction"); - } - }; - - // If the sequence contents can be empty, then the first - // token could be the separator token itself. - if let (Some(sep), true) = (&seq_rep.separator, subfirst.maybe_empty) { - first.add_one_maybe(TokenTree::Token(sep.clone())); - } - - assert!(first.maybe_empty); - first.add_all(subfirst); - if subfirst.maybe_empty - || seq_rep.kleene.op == mbe::KleeneOp::ZeroOrMore - || seq_rep.kleene.op == mbe::KleeneOp::ZeroOrOne - { - // Continue scanning for more first - // tokens, but also make sure we - // restore empty-tracking state. - first.maybe_empty = true; - continue; - } else { - return first; - } - } - } - } - - // we only exit the loop if `tts` was empty or if every - // element of `tts` matches the empty sequence. - assert!(first.maybe_empty); - first - } -} - -// A set of `mbe::TokenTree`s, which may include `TokenTree::Match`s -// (for macro-by-example syntactic variables). It also carries the -// `maybe_empty` flag; that is true if and only if the matcher can -// match an empty token sequence. -// -// The First set is computed on submatchers like `$($a:expr b),* $(c)* d`, -// which has corresponding FIRST = {$a:expr, c, d}. -// Likewise, `$($a:expr b),* $(c)+ d` has FIRST = {$a:expr, c}. -// -// (Notably, we must allow for *-op to occur zero times.) -#[derive(Clone, Debug)] -struct TokenSet { - tokens: Vec, - maybe_empty: bool, -} - -impl TokenSet { - // Returns a set for the empty sequence. - fn empty() -> Self { - TokenSet { tokens: Vec::new(), maybe_empty: true } - } - - // Returns the set `{ tok }` for the single-token (and thus - // non-empty) sequence [tok]. - fn singleton(tok: mbe::TokenTree) -> Self { - TokenSet { tokens: vec![tok], maybe_empty: false } - } - - // Changes self to be the set `{ tok }`. - // Since `tok` is always present, marks self as non-empty. - fn replace_with(&mut self, tok: mbe::TokenTree) { - self.tokens.clear(); - self.tokens.push(tok); - self.maybe_empty = false; - } - - // Changes self to be the empty set `{}`; meant for use when - // the particular token does not matter, but we want to - // record that it occurs. - fn replace_with_irrelevant(&mut self) { - self.tokens.clear(); - self.maybe_empty = false; - } - - // Adds `tok` to the set for `self`, marking sequence as non-empy. - fn add_one(&mut self, tok: mbe::TokenTree) { - if !self.tokens.contains(&tok) { - self.tokens.push(tok); - } - self.maybe_empty = false; - } - - // Adds `tok` to the set for `self`. (Leaves `maybe_empty` flag alone.) - fn add_one_maybe(&mut self, tok: mbe::TokenTree) { - if !self.tokens.contains(&tok) { - self.tokens.push(tok); - } - } - - // Adds all elements of `other` to this. - // - // (Since this is a set, we filter out duplicates.) - // - // If `other` is potentially empty, then preserves the previous - // setting of the empty flag of `self`. If `other` is guaranteed - // non-empty, then `self` is marked non-empty. - fn add_all(&mut self, other: &Self) { - for tok in &other.tokens { - if !self.tokens.contains(tok) { - self.tokens.push(tok.clone()); - } - } - if !other.maybe_empty { - self.maybe_empty = false; - } - } -} - -// Checks that `matcher` is internally consistent and that it -// can legally be followed by a token `N`, for all `N` in `follow`. -// (If `follow` is empty, then it imposes no constraint on -// the `matcher`.) -// -// Returns the set of NT tokens that could possibly come last in -// `matcher`. (If `matcher` matches the empty sequence, then -// `maybe_empty` will be set to true.) -// -// Requires that `first_sets` is pre-computed for `matcher`; -// see `FirstSets::new`. -fn check_matcher_core( - sess: &ParseSess, - features: &Features, - attrs: &[ast::Attribute], - first_sets: &FirstSets, - matcher: &[mbe::TokenTree], - follow: &TokenSet, -) -> TokenSet { - use mbe::TokenTree; - - let mut last = TokenSet::empty(); - - // 2. For each token and suffix [T, SUFFIX] in M: - // ensure that T can be followed by SUFFIX, and if SUFFIX may be empty, - // then ensure T can also be followed by any element of FOLLOW. - 'each_token: for i in 0..matcher.len() { - let token = &matcher[i]; - let suffix = &matcher[i + 1..]; - - let build_suffix_first = || { - let mut s = first_sets.first(suffix); - if s.maybe_empty { - s.add_all(follow); - } - s - }; - - // (we build `suffix_first` on demand below; you can tell - // which cases are supposed to fall through by looking for the - // initialization of this variable.) - let suffix_first; - - // First, update `last` so that it corresponds to the set - // of NT tokens that might end the sequence `... token`. - match *token { - TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => { - let can_be_followed_by_any; - if let Err(bad_frag) = has_legal_fragment_specifier(sess, features, attrs, token) { - let msg = format!("invalid fragment specifier `{}`", bad_frag); - sess.span_diagnostic - .struct_span_err(token.span(), &msg) - .help(VALID_FRAGMENT_NAMES_MSG) - .emit(); - // (This eliminates false positives and duplicates - // from error messages.) - can_be_followed_by_any = true; - } else { - can_be_followed_by_any = token_can_be_followed_by_any(token); - } - - if can_be_followed_by_any { - // don't need to track tokens that work with any, - last.replace_with_irrelevant(); - // ... and don't need to check tokens that can be - // followed by anything against SUFFIX. - continue 'each_token; - } else { - last.replace_with(token.clone()); - suffix_first = build_suffix_first(); - } - } - TokenTree::Delimited(span, ref d) => { - let my_suffix = TokenSet::singleton(d.close_tt(span.close)); - check_matcher_core(sess, features, attrs, first_sets, &d.tts, &my_suffix); - // don't track non NT tokens - last.replace_with_irrelevant(); - - // also, we don't need to check delimited sequences - // against SUFFIX - continue 'each_token; - } - TokenTree::Sequence(_, ref seq_rep) => { - suffix_first = build_suffix_first(); - // The trick here: when we check the interior, we want - // to include the separator (if any) as a potential - // (but not guaranteed) element of FOLLOW. So in that - // case, we make a temp copy of suffix and stuff - // delimiter in there. - // - // FIXME: Should I first scan suffix_first to see if - // delimiter is already in it before I go through the - // work of cloning it? But then again, this way I may - // get a "tighter" span? - let mut new; - let my_suffix = if let Some(sep) = &seq_rep.separator { - new = suffix_first.clone(); - new.add_one_maybe(TokenTree::Token(sep.clone())); - &new - } else { - &suffix_first - }; - - // At this point, `suffix_first` is built, and - // `my_suffix` is some TokenSet that we can use - // for checking the interior of `seq_rep`. - let next = - check_matcher_core(sess, features, attrs, first_sets, &seq_rep.tts, my_suffix); - if next.maybe_empty { - last.add_all(&next); - } else { - last = next; - } - - // the recursive call to check_matcher_core already ran the 'each_last - // check below, so we can just keep going forward here. - continue 'each_token; - } - } - - // (`suffix_first` guaranteed initialized once reaching here.) - - // Now `last` holds the complete set of NT tokens that could - // end the sequence before SUFFIX. Check that every one works with `suffix`. - 'each_last: for token in &last.tokens { - if let TokenTree::MetaVarDecl(_, name, frag_spec) = *token { - for next_token in &suffix_first.tokens { - match is_in_follow(next_token, frag_spec.name) { - IsInFollow::Invalid(msg, help) => { - sess.span_diagnostic - .struct_span_err(next_token.span(), &msg) - .help(help) - .emit(); - // don't bother reporting every source of - // conflict for a particular element of `last`. - continue 'each_last; - } - IsInFollow::Yes => {} - IsInFollow::No(possible) => { - let may_be = if last.tokens.len() == 1 && suffix_first.tokens.len() == 1 - { - "is" - } else { - "may be" - }; - - let sp = next_token.span(); - let mut err = sess.span_diagnostic.struct_span_err( - sp, - &format!( - "`${name}:{frag}` {may_be} followed by `{next}`, which \ - is not allowed for `{frag}` fragments", - name = name, - frag = frag_spec, - next = quoted_tt_to_string(next_token), - may_be = may_be - ), - ); - err.span_label( - sp, - format!("not allowed after `{}` fragments", frag_spec), - ); - let msg = "allowed there are: "; - match possible { - &[] => {} - &[t] => { - err.note(&format!( - "only {} is allowed after `{}` fragments", - t, frag_spec, - )); - } - ts => { - err.note(&format!( - "{}{} or {}", - msg, - ts[..ts.len() - 1] - .iter() - .map(|s| *s) - .collect::>() - .join(", "), - ts[ts.len() - 1], - )); - } - } - err.emit(); - } - } - } - } - } - } - last -} - -fn token_can_be_followed_by_any(tok: &mbe::TokenTree) -> bool { - if let mbe::TokenTree::MetaVarDecl(_, _, frag_spec) = *tok { - frag_can_be_followed_by_any(frag_spec.name) - } else { - // (Non NT's can always be followed by anthing in matchers.) - true - } -} - -/// Returns `true` if a fragment of type `frag` can be followed by any sort of -/// token. We use this (among other things) as a useful approximation -/// for when `frag` can be followed by a repetition like `$(...)*` or -/// `$(...)+`. In general, these can be a bit tricky to reason about, -/// so we adopt a conservative position that says that any fragment -/// specifier which consumes at most one token tree can be followed by -/// a fragment specifier (indeed, these fragments can be followed by -/// ANYTHING without fear of future compatibility hazards). -fn frag_can_be_followed_by_any(frag: Symbol) -> bool { - match frag { - sym::item | // always terminated by `}` or `;` - sym::block | // exactly one token tree - sym::ident | // exactly one token tree - sym::literal | // exactly one token tree - sym::meta | // exactly one token tree - sym::lifetime | // exactly one token tree - sym::tt => // exactly one token tree - true, - - _ => - false, - } -} - -enum IsInFollow { - Yes, - No(&'static [&'static str]), - Invalid(String, &'static str), -} - -/// Returns `true` if `frag` can legally be followed by the token `tok`. For -/// fragments that can consume an unbounded number of tokens, `tok` -/// must be within a well-defined follow set. This is intended to -/// guarantee future compatibility: for example, without this rule, if -/// we expanded `expr` to include a new binary operator, we might -/// break macros that were relying on that binary operator as a -/// separator. -// when changing this do not forget to update doc/book/macros.md! -fn is_in_follow(tok: &mbe::TokenTree, frag: Symbol) -> IsInFollow { - use mbe::TokenTree; - - if let TokenTree::Token(Token { kind: token::CloseDelim(_), .. }) = *tok { - // closing a token tree can never be matched by any fragment; - // iow, we always require that `(` and `)` match, etc. - IsInFollow::Yes - } else { - match frag { - sym::item => { - // since items *must* be followed by either a `;` or a `}`, we can - // accept anything after them - IsInFollow::Yes - } - sym::block => { - // anything can follow block, the braces provide an easy boundary to - // maintain - IsInFollow::Yes - } - sym::stmt | sym::expr => { - const TOKENS: &[&str] = &["`=>`", "`,`", "`;`"]; - match tok { - TokenTree::Token(token) => match token.kind { - FatArrow | Comma | Semi => IsInFollow::Yes, - _ => IsInFollow::No(TOKENS), - }, - _ => IsInFollow::No(TOKENS), - } - } - sym::pat => { - const TOKENS: &[&str] = &["`=>`", "`,`", "`=`", "`|`", "`if`", "`in`"]; - match tok { - TokenTree::Token(token) => match token.kind { - FatArrow | Comma | Eq | BinOp(token::Or) => IsInFollow::Yes, - Ident(name, false) if name == kw::If || name == kw::In => IsInFollow::Yes, - _ => IsInFollow::No(TOKENS), - }, - _ => IsInFollow::No(TOKENS), - } - } - sym::path | sym::ty => { - const TOKENS: &[&str] = &[ - "`{`", "`[`", "`=>`", "`,`", "`>`", "`=`", "`:`", "`;`", "`|`", "`as`", - "`where`", - ]; - match tok { - TokenTree::Token(token) => match token.kind { - OpenDelim(token::DelimToken::Brace) - | OpenDelim(token::DelimToken::Bracket) - | Comma - | FatArrow - | Colon - | Eq - | Gt - | BinOp(token::Shr) - | Semi - | BinOp(token::Or) => IsInFollow::Yes, - Ident(name, false) if name == kw::As || name == kw::Where => { - IsInFollow::Yes - } - _ => IsInFollow::No(TOKENS), - }, - TokenTree::MetaVarDecl(_, _, frag) if frag.name == sym::block => { - IsInFollow::Yes - } - _ => IsInFollow::No(TOKENS), - } - } - sym::ident | sym::lifetime => { - // being a single token, idents and lifetimes are harmless - IsInFollow::Yes - } - sym::literal => { - // literals may be of a single token, or two tokens (negative numbers) - IsInFollow::Yes - } - sym::meta | sym::tt => { - // being either a single token or a delimited sequence, tt is - // harmless - IsInFollow::Yes - } - sym::vis => { - // Explicitly disallow `priv`, on the off chance it comes back. - const TOKENS: &[&str] = &["`,`", "an ident", "a type"]; - match tok { - TokenTree::Token(token) => match token.kind { - Comma => IsInFollow::Yes, - Ident(name, is_raw) if is_raw || name != kw::Priv => IsInFollow::Yes, - _ => { - if token.can_begin_type() { - IsInFollow::Yes - } else { - IsInFollow::No(TOKENS) - } - } - }, - TokenTree::MetaVarDecl(_, _, frag) - if frag.name == sym::ident - || frag.name == sym::ty - || frag.name == sym::path => - { - IsInFollow::Yes - } - _ => IsInFollow::No(TOKENS), - } - } - kw::Invalid => IsInFollow::Yes, - _ => IsInFollow::Invalid( - format!("invalid fragment specifier `{}`", frag), - VALID_FRAGMENT_NAMES_MSG, - ), - } - } -} - -fn has_legal_fragment_specifier( - sess: &ParseSess, - features: &Features, - attrs: &[ast::Attribute], - tok: &mbe::TokenTree, -) -> Result<(), String> { - debug!("has_legal_fragment_specifier({:?})", tok); - if let mbe::TokenTree::MetaVarDecl(_, _, ref frag_spec) = *tok { - let frag_span = tok.span(); - if !is_legal_fragment_specifier(sess, features, attrs, frag_spec.name, frag_span) { - return Err(frag_spec.to_string()); - } - } - Ok(()) -} - -fn is_legal_fragment_specifier( - _sess: &ParseSess, - _features: &Features, - _attrs: &[ast::Attribute], - frag_name: Symbol, - _frag_span: Span, -) -> bool { - /* - * If new fragment specifiers are invented in nightly, `_sess`, - * `_features`, `_attrs`, and `_frag_span` will be useful here - * for checking against feature gates. See past versions of - * this function. - */ - match frag_name { - sym::item - | sym::block - | sym::stmt - | sym::expr - | sym::pat - | sym::lifetime - | sym::path - | sym::ty - | sym::ident - | sym::meta - | sym::tt - | sym::vis - | sym::literal - | kw::Invalid => true, - _ => false, - } -} - -fn quoted_tt_to_string(tt: &mbe::TokenTree) -> String { - match *tt { - mbe::TokenTree::Token(ref token) => crate::print::pprust::token_to_string(&token), - mbe::TokenTree::MetaVar(_, name) => format!("${}", name), - mbe::TokenTree::MetaVarDecl(_, name, kind) => format!("${}:{}", name, kind), - _ => panic!( - "unexpected mbe::TokenTree::{{Sequence or Delimited}} \ - in follow set checker" - ), - } -} - -impl TokenTree { - /// Use this token tree as a matcher to parse given tts. - fn parse(cx: &ExtCtxt<'_>, mtch: &[mbe::TokenTree], tts: TokenStream) - -> NamedParseResult { - // `None` is because we're not interpolating - let directory = Directory { - path: Cow::from(cx.current_expansion.module.directory.as_path()), - ownership: cx.current_expansion.directory_ownership, - }; - parse(cx.parse_sess(), tts, mtch, Some(directory), true) - } -} - -/// Generates an appropriate parsing failure message. For EOF, this is "unexpected end...". For -/// other tokens, this is "unexpected token...". -fn parse_failure_msg(tok: &Token) -> String { - match tok.kind { - token::Eof => "unexpected end of macro invocation".to_string(), - _ => format!( - "no rules expected the token `{}`", - pprust::token_to_string(tok), - ), - } -} diff --git a/src/libsyntax/ext/mbe/quoted.rs b/src/libsyntax/ext/mbe/quoted.rs deleted file mode 100644 index 3cec4bc60e7..00000000000 --- a/src/libsyntax/ext/mbe/quoted.rs +++ /dev/null @@ -1,263 +0,0 @@ -use crate::ast; -use crate::ext::mbe::macro_parser; -use crate::ext::mbe::{TokenTree, KleeneOp, KleeneToken, SequenceRepetition, Delimited}; -use crate::parse::token::{self, Token}; -use crate::print::pprust; -use crate::sess::ParseSess; -use crate::symbol::kw; -use crate::tokenstream; - -use syntax_pos::Span; - -use rustc_data_structures::sync::Lrc; - -/// Takes a `tokenstream::TokenStream` and returns a `Vec`. Specifically, this -/// takes a generic `TokenStream`, such as is used in the rest of the compiler, and returns a -/// collection of `TokenTree` for use in parsing a macro. -/// -/// # Parameters -/// -/// - `input`: a token stream to read from, the contents of which we are parsing. -/// - `expect_matchers`: `parse` can be used to parse either the "patterns" or the "body" of a -/// macro. Both take roughly the same form _except_ that in a pattern, metavars are declared with -/// their "matcher" type. For example `$var:expr` or `$id:ident`. In this example, `expr` and -/// `ident` are "matchers". They are not present in the body of a macro rule -- just in the -/// pattern, so we pass a parameter to indicate whether to expect them or not. -/// - `sess`: the parsing session. Any errors will be emitted to this session. -/// - `features`, `attrs`: language feature flags and attributes so that we know whether to use -/// unstable features or not. -/// - `edition`: which edition are we in. -/// - `macro_node_id`: the NodeId of the macro we are parsing. -/// -/// # Returns -/// -/// A collection of `self::TokenTree`. There may also be some errors emitted to `sess`. -pub(super) fn parse( - input: tokenstream::TokenStream, - expect_matchers: bool, - sess: &ParseSess, -) -> Vec { - // Will contain the final collection of `self::TokenTree` - let mut result = Vec::new(); - - // For each token tree in `input`, parse the token into a `self::TokenTree`, consuming - // additional trees if need be. - let mut trees = input.trees(); - while let Some(tree) = trees.next() { - // Given the parsed tree, if there is a metavar and we are expecting matchers, actually - // parse out the matcher (i.e., in `$id:ident` this would parse the `:` and `ident`). - let tree = parse_tree( - tree, - &mut trees, - expect_matchers, - sess, - ); - match tree { - TokenTree::MetaVar(start_sp, ident) if expect_matchers => { - let span = match trees.next() { - Some(tokenstream::TokenTree::Token(Token { kind: token::Colon, span })) => { - match trees.next() { - Some(tokenstream::TokenTree::Token(token)) => match token.ident() { - Some((kind, _)) => { - let span = token.span.with_lo(start_sp.lo()); - result.push(TokenTree::MetaVarDecl(span, ident, kind)); - continue; - } - _ => token.span, - }, - tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span), - } - } - tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(start_sp), - }; - sess.missing_fragment_specifiers.borrow_mut().insert(span); - result.push(TokenTree::MetaVarDecl(span, ident, ast::Ident::invalid())); - } - - // Not a metavar or no matchers allowed, so just return the tree - _ => result.push(tree), - } - } - result -} - -/// Takes a `tokenstream::TokenTree` and returns a `self::TokenTree`. Specifically, this takes a -/// generic `TokenTree`, such as is used in the rest of the compiler, and returns a `TokenTree` -/// for use in parsing a macro. -/// -/// Converting the given tree may involve reading more tokens. -/// -/// # Parameters -/// -/// - `tree`: the tree we wish to convert. -/// - `trees`: an iterator over trees. We may need to read more tokens from it in order to finish -/// converting `tree` -/// - `expect_matchers`: same as for `parse` (see above). -/// - `sess`: the parsing session. Any errors will be emitted to this session. -/// - `features`, `attrs`: language feature flags and attributes so that we know whether to use -/// unstable features or not. -fn parse_tree( - tree: tokenstream::TokenTree, - trees: &mut impl Iterator, - expect_matchers: bool, - sess: &ParseSess, -) -> TokenTree { - // Depending on what `tree` is, we could be parsing different parts of a macro - match tree { - // `tree` is a `$` token. Look at the next token in `trees` - tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }) => match trees.next() { - // `tree` is followed by a delimited set of token trees. This indicates the beginning - // of a repetition sequence in the macro (e.g. `$(pat)*`). - Some(tokenstream::TokenTree::Delimited(span, delim, tts)) => { - // Must have `(` not `{` or `[` - if delim != token::Paren { - let tok = pprust::token_kind_to_string(&token::OpenDelim(delim)); - let msg = format!("expected `(`, found `{}`", tok); - sess.span_diagnostic.span_err(span.entire(), &msg); - } - // Parse the contents of the sequence itself - let sequence = parse( - tts.into(), - expect_matchers, - sess, - ); - // Get the Kleene operator and optional separator - let (separator, kleene) = parse_sep_and_kleene_op(trees, span.entire(), sess); - // Count the number of captured "names" (i.e., named metavars) - let name_captures = macro_parser::count_names(&sequence); - TokenTree::Sequence( - span, - Lrc::new(SequenceRepetition { - tts: sequence, - separator, - kleene, - num_captures: name_captures, - }), - ) - } - - // `tree` is followed by an `ident`. This could be `$meta_var` or the `$crate` special - // metavariable that names the crate of the invocation. - Some(tokenstream::TokenTree::Token(token)) if token.is_ident() => { - let (ident, is_raw) = token.ident().unwrap(); - let span = ident.span.with_lo(span.lo()); - if ident.name == kw::Crate && !is_raw { - TokenTree::token(token::Ident(kw::DollarCrate, is_raw), span) - } else { - TokenTree::MetaVar(span, ident) - } - } - - // `tree` is followed by a random token. This is an error. - Some(tokenstream::TokenTree::Token(token)) => { - let msg = - format!("expected identifier, found `{}`", pprust::token_to_string(&token),); - sess.span_diagnostic.span_err(token.span, &msg); - TokenTree::MetaVar(token.span, ast::Ident::invalid()) - } - - // There are no more tokens. Just return the `$` we already have. - None => TokenTree::token(token::Dollar, span), - }, - - // `tree` is an arbitrary token. Keep it. - tokenstream::TokenTree::Token(token) => TokenTree::Token(token), - - // `tree` is the beginning of a delimited set of tokens (e.g., `(` or `{`). We need to - // descend into the delimited set and further parse it. - tokenstream::TokenTree::Delimited(span, delim, tts) => TokenTree::Delimited( - span, - Lrc::new(Delimited { - delim, - tts: parse( - tts.into(), - expect_matchers, - sess, - ), - }), - ), - } -} - -/// Takes a token and returns `Some(KleeneOp)` if the token is `+` `*` or `?`. Otherwise, return -/// `None`. -fn kleene_op(token: &Token) -> Option { - match token.kind { - token::BinOp(token::Star) => Some(KleeneOp::ZeroOrMore), - token::BinOp(token::Plus) => Some(KleeneOp::OneOrMore), - token::Question => Some(KleeneOp::ZeroOrOne), - _ => None, - } -} - -/// Parse the next token tree of the input looking for a KleeneOp. Returns -/// -/// - Ok(Ok((op, span))) if the next token tree is a KleeneOp -/// - Ok(Err(tok, span)) if the next token tree is a token but not a KleeneOp -/// - Err(span) if the next token tree is not a token -fn parse_kleene_op( - input: &mut impl Iterator, - span: Span, -) -> Result, Span> { - match input.next() { - Some(tokenstream::TokenTree::Token(token)) => match kleene_op(&token) { - Some(op) => Ok(Ok((op, token.span))), - None => Ok(Err(token)), - }, - tree => Err(tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span)), - } -} - -/// Attempt to parse a single Kleene star, possibly with a separator. -/// -/// For example, in a pattern such as `$(a),*`, `a` is the pattern to be repeated, `,` is the -/// separator, and `*` is the Kleene operator. This function is specifically concerned with parsing -/// the last two tokens of such a pattern: namely, the optional separator and the Kleene operator -/// itself. Note that here we are parsing the _macro_ itself, rather than trying to match some -/// stream of tokens in an invocation of a macro. -/// -/// This function will take some input iterator `input` corresponding to `span` and a parsing -/// session `sess`. If the next one (or possibly two) tokens in `input` correspond to a Kleene -/// operator and separator, then a tuple with `(separator, KleeneOp)` is returned. Otherwise, an -/// error with the appropriate span is emitted to `sess` and a dummy value is returned. -fn parse_sep_and_kleene_op( - input: &mut impl Iterator, - span: Span, - sess: &ParseSess, -) -> (Option, KleeneToken) { - // We basically look at two token trees here, denoted as #1 and #2 below - let span = match parse_kleene_op(input, span) { - // #1 is a `?`, `+`, or `*` KleeneOp - Ok(Ok((op, span))) => return (None, KleeneToken::new(op, span)), - - // #1 is a separator followed by #2, a KleeneOp - Ok(Err(token)) => match parse_kleene_op(input, token.span) { - // #2 is the `?` Kleene op, which does not take a separator (error) - Ok(Ok((KleeneOp::ZeroOrOne, span))) => { - // Error! - sess.span_diagnostic.span_err( - token.span, - "the `?` macro repetition operator does not take a separator", - ); - - // Return a dummy - return (None, KleeneToken::new(KleeneOp::ZeroOrMore, span)); - } - - // #2 is a KleeneOp :D - Ok(Ok((op, span))) => return (Some(token), KleeneToken::new(op, span)), - - // #2 is a random token or not a token at all :( - Ok(Err(Token { span, .. })) | Err(span) => span, - }, - - // #1 is not a token - Err(span) => span, - }; - - // If we ever get to this point, we have experienced an "unexpected token" error - sess.span_diagnostic.span_err(span, "expected one of: `*`, `+`, or `?`"); - - // Return a dummy - (None, KleeneToken::new(KleeneOp::ZeroOrMore, span)) -} diff --git a/src/libsyntax/ext/mbe/transcribe.rs b/src/libsyntax/ext/mbe/transcribe.rs deleted file mode 100644 index da930436d81..00000000000 --- a/src/libsyntax/ext/mbe/transcribe.rs +++ /dev/null @@ -1,398 +0,0 @@ -use crate::ast::{Ident, Mac}; -use crate::ext::base::ExtCtxt; -use crate::ext::mbe; -use crate::ext::mbe::macro_parser::{MatchedNonterminal, MatchedSeq, NamedMatch}; -use crate::mut_visit::{self, MutVisitor}; -use crate::parse::token::{self, NtTT, Token}; -use crate::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndJoint}; - -use smallvec::{smallvec, SmallVec}; - -use errors::pluralise; -use rustc_data_structures::fx::FxHashMap; -use rustc_data_structures::sync::Lrc; -use syntax_pos::hygiene::{ExpnId, Transparency}; -use syntax_pos::Span; - -use std::mem; - -// A Marker adds the given mark to the syntax context. -struct Marker(ExpnId, Transparency); - -impl MutVisitor for Marker { - fn visit_span(&mut self, span: &mut Span) { - *span = span.apply_mark(self.0, self.1) - } - - fn visit_mac(&mut self, mac: &mut Mac) { - mut_visit::noop_visit_mac(mac, self) - } -} - -impl Marker { - fn visit_delim_span(&mut self, dspan: &mut DelimSpan) { - self.visit_span(&mut dspan.open); - self.visit_span(&mut dspan.close); - } -} - -/// An iterator over the token trees in a delimited token tree (`{ ... }`) or a sequence (`$(...)`). -enum Frame { - Delimited { forest: Lrc, idx: usize, span: DelimSpan }, - Sequence { forest: Lrc, idx: usize, sep: Option }, -} - -impl Frame { - /// Construct a new frame around the delimited set of tokens. - fn new(tts: Vec) -> Frame { - let forest = Lrc::new(mbe::Delimited { delim: token::NoDelim, tts }); - Frame::Delimited { forest, idx: 0, span: DelimSpan::dummy() } - } -} - -impl Iterator for Frame { - type Item = mbe::TokenTree; - - fn next(&mut self) -> Option { - match *self { - Frame::Delimited { ref forest, ref mut idx, .. } => { - *idx += 1; - forest.tts.get(*idx - 1).cloned() - } - Frame::Sequence { ref forest, ref mut idx, .. } => { - *idx += 1; - forest.tts.get(*idx - 1).cloned() - } - } - } -} - -/// This can do Macro-By-Example transcription. -/// - `interp` is a map of meta-variables to the tokens (non-terminals) they matched in the -/// invocation. We are assuming we already know there is a match. -/// - `src` is the RHS of the MBE, that is, the "example" we are filling in. -/// -/// For example, -/// -/// ```rust -/// macro_rules! foo { -/// ($id:ident) => { println!("{}", stringify!($id)); } -/// } -/// -/// foo!(bar); -/// ``` -/// -/// `interp` would contain `$id => bar` and `src` would contain `println!("{}", stringify!($id));`. -/// -/// `transcribe` would return a `TokenStream` containing `println!("{}", stringify!(bar));`. -/// -/// Along the way, we do some additional error checking. -pub(super) fn transcribe( - cx: &ExtCtxt<'_>, - interp: &FxHashMap, - src: Vec, - transparency: Transparency, -) -> TokenStream { - // Nothing for us to transcribe... - if src.is_empty() { - return TokenStream::default(); - } - - // We descend into the RHS (`src`), expanding things as we go. This stack contains the things - // we have yet to expand/are still expanding. We start the stack off with the whole RHS. - let mut stack: SmallVec<[Frame; 1]> = smallvec![Frame::new(src)]; - - // As we descend in the RHS, we will need to be able to match nested sequences of matchers. - // `repeats` keeps track of where we are in matching at each level, with the last element being - // the most deeply nested sequence. This is used as a stack. - let mut repeats = Vec::new(); - - // `result` contains resulting token stream from the TokenTree we just finished processing. At - // the end, this will contain the full result of transcription, but at arbitrary points during - // `transcribe`, `result` will contain subsets of the final result. - // - // Specifically, as we descend into each TokenTree, we will push the existing results onto the - // `result_stack` and clear `results`. We will then produce the results of transcribing the - // TokenTree into `results`. Then, as we unwind back out of the `TokenTree`, we will pop the - // `result_stack` and append `results` too it to produce the new `results` up to that point. - // - // Thus, if we try to pop the `result_stack` and it is empty, we have reached the top-level - // again, and we are done transcribing. - let mut result: Vec = Vec::new(); - let mut result_stack = Vec::new(); - let mut marker = Marker(cx.current_expansion.id, transparency); - - loop { - // Look at the last frame on the stack. - let tree = if let Some(tree) = stack.last_mut().unwrap().next() { - // If it still has a TokenTree we have not looked at yet, use that tree. - tree - } - // The else-case never produces a value for `tree` (it `continue`s or `return`s). - else { - // Otherwise, if we have just reached the end of a sequence and we can keep repeating, - // go back to the beginning of the sequence. - if let Frame::Sequence { idx, sep, .. } = stack.last_mut().unwrap() { - let (repeat_idx, repeat_len) = repeats.last_mut().unwrap(); - *repeat_idx += 1; - if repeat_idx < repeat_len { - *idx = 0; - if let Some(sep) = sep { - result.push(TokenTree::Token(sep.clone()).into()); - } - continue; - } - } - - // We are done with the top of the stack. Pop it. Depending on what it was, we do - // different things. Note that the outermost item must be the delimited, wrapped RHS - // that was passed in originally to `transcribe`. - match stack.pop().unwrap() { - // Done with a sequence. Pop from repeats. - Frame::Sequence { .. } => { - repeats.pop(); - } - - // We are done processing a Delimited. If this is the top-level delimited, we are - // done. Otherwise, we unwind the result_stack to append what we have produced to - // any previous results. - Frame::Delimited { forest, span, .. } => { - if result_stack.is_empty() { - // No results left to compute! We are back at the top-level. - return TokenStream::new(result); - } - - // Step back into the parent Delimited. - let tree = - TokenTree::Delimited(span, forest.delim, TokenStream::new(result).into()); - result = result_stack.pop().unwrap(); - result.push(tree.into()); - } - } - continue; - }; - - // At this point, we know we are in the middle of a TokenTree (the last one on `stack`). - // `tree` contains the next `TokenTree` to be processed. - match tree { - // We are descending into a sequence. We first make sure that the matchers in the RHS - // and the matches in `interp` have the same shape. Otherwise, either the caller or the - // macro writer has made a mistake. - seq @ mbe::TokenTree::Sequence(..) => { - match lockstep_iter_size(&seq, interp, &repeats) { - LockstepIterSize::Unconstrained => { - cx.span_fatal( - seq.span(), /* blame macro writer */ - "attempted to repeat an expression containing no syntax variables \ - matched as repeating at this depth", - ); - } - - LockstepIterSize::Contradiction(ref msg) => { - // FIXME: this really ought to be caught at macro definition time... It - // happens when two meta-variables are used in the same repetition in a - // sequence, but they come from different sequence matchers and repeat - // different amounts. - cx.span_fatal(seq.span(), &msg[..]); - } - - LockstepIterSize::Constraint(len, _) => { - // We do this to avoid an extra clone above. We know that this is a - // sequence already. - let (sp, seq) = if let mbe::TokenTree::Sequence(sp, seq) = seq { - (sp, seq) - } else { - unreachable!() - }; - - // Is the repetition empty? - if len == 0 { - if seq.kleene.op == mbe::KleeneOp::OneOrMore { - // FIXME: this really ought to be caught at macro definition - // time... It happens when the Kleene operator in the matcher and - // the body for the same meta-variable do not match. - cx.span_fatal(sp.entire(), "this must repeat at least once"); - } - } else { - // 0 is the initial counter (we have done 0 repretitions so far). `len` - // is the total number of reptitions we should generate. - repeats.push((0, len)); - - // The first time we encounter the sequence we push it to the stack. It - // then gets reused (see the beginning of the loop) until we are done - // repeating. - stack.push(Frame::Sequence { - idx: 0, - sep: seq.separator.clone(), - forest: seq, - }); - } - } - } - } - - // Replace the meta-var with the matched token tree from the invocation. - mbe::TokenTree::MetaVar(mut sp, mut ident) => { - // Find the matched nonterminal from the macro invocation, and use it to replace - // the meta-var. - if let Some(cur_matched) = lookup_cur_matched(ident, interp, &repeats) { - if let MatchedNonterminal(ref nt) = cur_matched { - // FIXME #2887: why do we apply a mark when matching a token tree meta-var - // (e.g. `$x:tt`), but not when we are matching any other type of token - // tree? - if let NtTT(ref tt) = **nt { - result.push(tt.clone().into()); - } else { - marker.visit_span(&mut sp); - let token = TokenTree::token(token::Interpolated(nt.clone()), sp); - result.push(token.into()); - } - } else { - // We were unable to descend far enough. This is an error. - cx.span_fatal( - sp, /* blame the macro writer */ - &format!("variable '{}' is still repeating at this depth", ident), - ); - } - } else { - // If we aren't able to match the meta-var, we push it back into the result but - // with modified syntax context. (I believe this supports nested macros). - marker.visit_span(&mut sp); - marker.visit_ident(&mut ident); - result.push(TokenTree::token(token::Dollar, sp).into()); - result.push(TokenTree::Token(Token::from_ast_ident(ident)).into()); - } - } - - // If we are entering a new delimiter, we push its contents to the `stack` to be - // processed, and we push all of the currently produced results to the `result_stack`. - // We will produce all of the results of the inside of the `Delimited` and then we will - // jump back out of the Delimited, pop the result_stack and add the new results back to - // the previous results (from outside the Delimited). - mbe::TokenTree::Delimited(mut span, delimited) => { - marker.visit_delim_span(&mut span); - stack.push(Frame::Delimited { forest: delimited, idx: 0, span }); - result_stack.push(mem::take(&mut result)); - } - - // Nothing much to do here. Just push the token to the result, being careful to - // preserve syntax context. - mbe::TokenTree::Token(token) => { - let mut tt = TokenTree::Token(token); - marker.visit_tt(&mut tt); - result.push(tt.into()); - } - - // There should be no meta-var declarations in the invocation of a macro. - mbe::TokenTree::MetaVarDecl(..) => panic!("unexpected `TokenTree::MetaVarDecl"), - } - } -} - -/// Lookup the meta-var named `ident` and return the matched token tree from the invocation using -/// the set of matches `interpolations`. -/// -/// See the definition of `repeats` in the `transcribe` function. `repeats` is used to descend -/// into the right place in nested matchers. If we attempt to descend too far, the macro writer has -/// made a mistake, and we return `None`. -fn lookup_cur_matched<'a>( - ident: Ident, - interpolations: &'a FxHashMap, - repeats: &[(usize, usize)], -) -> Option<&'a NamedMatch> { - interpolations.get(&ident).map(|matched| { - let mut matched = matched; - for &(idx, _) in repeats { - match matched { - MatchedNonterminal(_) => break, - MatchedSeq(ref ads, _) => matched = ads.get(idx).unwrap(), - } - } - - matched - }) -} - -/// An accumulator over a TokenTree to be used with `fold`. During transcription, we need to make -/// sure that the size of each sequence and all of its nested sequences are the same as the sizes -/// of all the matched (nested) sequences in the macro invocation. If they don't match, somebody -/// has made a mistake (either the macro writer or caller). -#[derive(Clone)] -enum LockstepIterSize { - /// No constraints on length of matcher. This is true for any TokenTree variants except a - /// `MetaVar` with an actual `MatchedSeq` (as opposed to a `MatchedNonterminal`). - Unconstrained, - - /// A `MetaVar` with an actual `MatchedSeq`. The length of the match and the name of the - /// meta-var are returned. - Constraint(usize, Ident), - - /// Two `Constraint`s on the same sequence had different lengths. This is an error. - Contradiction(String), -} - -impl LockstepIterSize { - /// Find incompatibilities in matcher/invocation sizes. - /// - `Unconstrained` is compatible with everything. - /// - `Contradiction` is incompatible with everything. - /// - `Constraint(len)` is only compatible with other constraints of the same length. - fn with(self, other: LockstepIterSize) -> LockstepIterSize { - match self { - LockstepIterSize::Unconstrained => other, - LockstepIterSize::Contradiction(_) => self, - LockstepIterSize::Constraint(l_len, ref l_id) => match other { - LockstepIterSize::Unconstrained => self, - LockstepIterSize::Contradiction(_) => other, - LockstepIterSize::Constraint(r_len, _) if l_len == r_len => self, - LockstepIterSize::Constraint(r_len, r_id) => { - let msg = format!( - "meta-variable `{}` repeats {} time{}, but `{}` repeats {} time{}", - l_id, - l_len, - pluralise!(l_len), - r_id, - r_len, - pluralise!(r_len), - ); - LockstepIterSize::Contradiction(msg) - } - }, - } - } -} - -/// Given a `tree`, make sure that all sequences have the same length as the matches for the -/// appropriate meta-vars in `interpolations`. -/// -/// Note that if `repeats` does not match the exact correct depth of a meta-var, -/// `lookup_cur_matched` will return `None`, which is why this still works even in the presnece of -/// multiple nested matcher sequences. -fn lockstep_iter_size( - tree: &mbe::TokenTree, - interpolations: &FxHashMap, - repeats: &[(usize, usize)], -) -> LockstepIterSize { - use mbe::TokenTree; - match *tree { - TokenTree::Delimited(_, ref delimed) => { - delimed.tts.iter().fold(LockstepIterSize::Unconstrained, |size, tt| { - size.with(lockstep_iter_size(tt, interpolations, repeats)) - }) - } - TokenTree::Sequence(_, ref seq) => { - seq.tts.iter().fold(LockstepIterSize::Unconstrained, |size, tt| { - size.with(lockstep_iter_size(tt, interpolations, repeats)) - }) - } - TokenTree::MetaVar(_, name) | TokenTree::MetaVarDecl(_, name, _) => { - match lookup_cur_matched(name, interpolations, repeats) { - Some(matched) => match matched { - MatchedNonterminal(_) => LockstepIterSize::Unconstrained, - MatchedSeq(ref ads, _) => LockstepIterSize::Constraint(ads.len(), name), - }, - _ => LockstepIterSize::Unconstrained, - } - } - TokenTree::Token(..) => LockstepIterSize::Unconstrained, - } -} diff --git a/src/libsyntax/ext/placeholders.rs b/src/libsyntax/ext/placeholders.rs deleted file mode 100644 index 4fae25bbde6..00000000000 --- a/src/libsyntax/ext/placeholders.rs +++ /dev/null @@ -1,349 +0,0 @@ -use crate::ast::{self, NodeId}; -use crate::source_map::{DUMMY_SP, dummy_spanned}; -use crate::ext::base::ExtCtxt; -use crate::ext::expand::{AstFragment, AstFragmentKind}; -use crate::tokenstream::TokenStream; -use crate::mut_visit::*; -use crate::ptr::P; -use crate::ThinVec; - -use smallvec::{smallvec, SmallVec}; - -use rustc_data_structures::fx::FxHashMap; - -pub fn placeholder(kind: AstFragmentKind, id: ast::NodeId) -> AstFragment { - fn mac_placeholder() -> ast::Mac { - ast::Mac { - path: ast::Path { span: DUMMY_SP, segments: Vec::new() }, - tts: TokenStream::default().into(), - delim: ast::MacDelimiter::Brace, - span: DUMMY_SP, - prior_type_ascription: None, - } - } - - let ident = ast::Ident::invalid(); - let attrs = Vec::new(); - let generics = ast::Generics::default(); - let vis = dummy_spanned(ast::VisibilityKind::Inherited); - let span = DUMMY_SP; - let expr_placeholder = || P(ast::Expr { - id, span, - attrs: ThinVec::new(), - kind: ast::ExprKind::Mac(mac_placeholder()), - }); - let ty = || P(ast::Ty { - id, - kind: ast::TyKind::Mac(mac_placeholder()), - span, - }); - let pat = || P(ast::Pat { - id, - kind: ast::PatKind::Mac(mac_placeholder()), - span, - }); - - match kind { - AstFragmentKind::Expr => AstFragment::Expr(expr_placeholder()), - AstFragmentKind::OptExpr => AstFragment::OptExpr(Some(expr_placeholder())), - AstFragmentKind::Items => AstFragment::Items(smallvec![P(ast::Item { - id, span, ident, vis, attrs, - kind: ast::ItemKind::Mac(mac_placeholder()), - tokens: None, - })]), - AstFragmentKind::TraitItems => AstFragment::TraitItems(smallvec![ast::TraitItem { - id, span, ident, attrs, generics, - kind: ast::TraitItemKind::Macro(mac_placeholder()), - tokens: None, - }]), - AstFragmentKind::ImplItems => AstFragment::ImplItems(smallvec![ast::ImplItem { - id, span, ident, vis, attrs, generics, - kind: ast::ImplItemKind::Macro(mac_placeholder()), - defaultness: ast::Defaultness::Final, - tokens: None, - }]), - AstFragmentKind::ForeignItems => - AstFragment::ForeignItems(smallvec![ast::ForeignItem { - id, span, ident, vis, attrs, - kind: ast::ForeignItemKind::Macro(mac_placeholder()), - }]), - AstFragmentKind::Pat => AstFragment::Pat(P(ast::Pat { - id, span, kind: ast::PatKind::Mac(mac_placeholder()), - })), - AstFragmentKind::Ty => AstFragment::Ty(P(ast::Ty { - id, span, kind: ast::TyKind::Mac(mac_placeholder()), - })), - AstFragmentKind::Stmts => AstFragment::Stmts(smallvec![{ - let mac = P((mac_placeholder(), ast::MacStmtStyle::Braces, ThinVec::new())); - ast::Stmt { id, span, kind: ast::StmtKind::Mac(mac) } - }]), - AstFragmentKind::Arms => AstFragment::Arms(smallvec![ - ast::Arm { - attrs: Default::default(), - body: expr_placeholder(), - guard: None, - id, - pat: pat(), - span, - is_placeholder: true, - } - ]), - AstFragmentKind::Fields => AstFragment::Fields(smallvec![ - ast::Field { - attrs: Default::default(), - expr: expr_placeholder(), - id, - ident, - is_shorthand: false, - span, - is_placeholder: true, - } - ]), - AstFragmentKind::FieldPats => AstFragment::FieldPats(smallvec![ - ast::FieldPat { - attrs: Default::default(), - id, - ident, - is_shorthand: false, - pat: pat(), - span, - is_placeholder: true, - } - ]), - AstFragmentKind::GenericParams => AstFragment::GenericParams(smallvec![{ - ast::GenericParam { - attrs: Default::default(), - bounds: Default::default(), - id, - ident, - is_placeholder: true, - kind: ast::GenericParamKind::Lifetime, - } - }]), - AstFragmentKind::Params => AstFragment::Params(smallvec![ - ast::Param { - attrs: Default::default(), - id, - pat: pat(), - span, - ty: ty(), - is_placeholder: true, - } - ]), - AstFragmentKind::StructFields => AstFragment::StructFields(smallvec![ - ast::StructField { - attrs: Default::default(), - id, - ident: None, - span, - ty: ty(), - vis, - is_placeholder: true, - } - ]), - AstFragmentKind::Variants => AstFragment::Variants(smallvec![ - ast::Variant { - attrs: Default::default(), - data: ast::VariantData::Struct(Default::default(), false), - disr_expr: None, - id, - ident, - span, - is_placeholder: true, - } - ]) - } -} - -pub struct PlaceholderExpander<'a, 'b> { - expanded_fragments: FxHashMap, - cx: &'a mut ExtCtxt<'b>, - monotonic: bool, -} - -impl<'a, 'b> PlaceholderExpander<'a, 'b> { - pub fn new(cx: &'a mut ExtCtxt<'b>, monotonic: bool) -> Self { - PlaceholderExpander { - cx, - expanded_fragments: FxHashMap::default(), - monotonic, - } - } - - pub fn add(&mut self, id: ast::NodeId, mut fragment: AstFragment, placeholders: Vec) { - fragment.mut_visit_with(self); - if let AstFragment::Items(mut items) = fragment { - for placeholder in placeholders { - match self.remove(placeholder) { - AstFragment::Items(derived_items) => items.extend(derived_items), - _ => unreachable!(), - } - } - fragment = AstFragment::Items(items); - } - self.expanded_fragments.insert(id, fragment); - } - - fn remove(&mut self, id: ast::NodeId) -> AstFragment { - self.expanded_fragments.remove(&id).unwrap() - } -} - -impl<'a, 'b> MutVisitor for PlaceholderExpander<'a, 'b> { - fn flat_map_arm(&mut self, arm: ast::Arm) -> SmallVec<[ast::Arm; 1]> { - if arm.is_placeholder { - self.remove(arm.id).make_arms() - } else { - noop_flat_map_arm(arm, self) - } - } - - fn flat_map_field(&mut self, field: ast::Field) -> SmallVec<[ast::Field; 1]> { - if field.is_placeholder { - self.remove(field.id).make_fields() - } else { - noop_flat_map_field(field, self) - } - } - - fn flat_map_field_pattern(&mut self, fp: ast::FieldPat) -> SmallVec<[ast::FieldPat; 1]> { - if fp.is_placeholder { - self.remove(fp.id).make_field_patterns() - } else { - noop_flat_map_field_pattern(fp, self) - } - } - - fn flat_map_generic_param( - &mut self, - param: ast::GenericParam - ) -> SmallVec<[ast::GenericParam; 1]> - { - if param.is_placeholder { - self.remove(param.id).make_generic_params() - } else { - noop_flat_map_generic_param(param, self) - } - } - - fn flat_map_param(&mut self, p: ast::Param) -> SmallVec<[ast::Param; 1]> { - if p.is_placeholder { - self.remove(p.id).make_params() - } else { - noop_flat_map_param(p, self) - } - } - - fn flat_map_struct_field(&mut self, sf: ast::StructField) -> SmallVec<[ast::StructField; 1]> { - if sf.is_placeholder { - self.remove(sf.id).make_struct_fields() - } else { - noop_flat_map_struct_field(sf, self) - } - } - - fn flat_map_variant(&mut self, variant: ast::Variant) -> SmallVec<[ast::Variant; 1]> { - if variant.is_placeholder { - self.remove(variant.id).make_variants() - } else { - noop_flat_map_variant(variant, self) - } - } - - fn flat_map_item(&mut self, item: P) -> SmallVec<[P; 1]> { - match item.kind { - ast::ItemKind::Mac(_) => return self.remove(item.id).make_items(), - ast::ItemKind::MacroDef(_) => return smallvec![item], - _ => {} - } - - noop_flat_map_item(item, self) - } - - fn flat_map_trait_item(&mut self, item: ast::TraitItem) -> SmallVec<[ast::TraitItem; 1]> { - match item.kind { - ast::TraitItemKind::Macro(_) => self.remove(item.id).make_trait_items(), - _ => noop_flat_map_trait_item(item, self), - } - } - - fn flat_map_impl_item(&mut self, item: ast::ImplItem) -> SmallVec<[ast::ImplItem; 1]> { - match item.kind { - ast::ImplItemKind::Macro(_) => self.remove(item.id).make_impl_items(), - _ => noop_flat_map_impl_item(item, self), - } - } - - fn flat_map_foreign_item(&mut self, item: ast::ForeignItem) -> SmallVec<[ast::ForeignItem; 1]> { - match item.kind { - ast::ForeignItemKind::Macro(_) => self.remove(item.id).make_foreign_items(), - _ => noop_flat_map_foreign_item(item, self), - } - } - - fn visit_expr(&mut self, expr: &mut P) { - match expr.kind { - ast::ExprKind::Mac(_) => *expr = self.remove(expr.id).make_expr(), - _ => noop_visit_expr(expr, self), - } - } - - fn filter_map_expr(&mut self, expr: P) -> Option> { - match expr.kind { - ast::ExprKind::Mac(_) => self.remove(expr.id).make_opt_expr(), - _ => noop_filter_map_expr(expr, self), - } - } - - fn flat_map_stmt(&mut self, stmt: ast::Stmt) -> SmallVec<[ast::Stmt; 1]> { - let (style, mut stmts) = match stmt.kind { - ast::StmtKind::Mac(mac) => (mac.1, self.remove(stmt.id).make_stmts()), - _ => return noop_flat_map_stmt(stmt, self), - }; - - if style == ast::MacStmtStyle::Semicolon { - if let Some(stmt) = stmts.pop() { - stmts.push(stmt.add_trailing_semicolon()); - } - } - - stmts - } - - fn visit_pat(&mut self, pat: &mut P) { - match pat.kind { - ast::PatKind::Mac(_) => *pat = self.remove(pat.id).make_pat(), - _ => noop_visit_pat(pat, self), - } - } - - fn visit_ty(&mut self, ty: &mut P) { - match ty.kind { - ast::TyKind::Mac(_) => *ty = self.remove(ty.id).make_ty(), - _ => noop_visit_ty(ty, self), - } - } - - fn visit_block(&mut self, block: &mut P) { - noop_visit_block(block, self); - - for stmt in block.stmts.iter_mut() { - if self.monotonic { - assert_eq!(stmt.id, ast::DUMMY_NODE_ID); - stmt.id = self.cx.resolver.next_node_id(); - } - } - } - - fn visit_mod(&mut self, module: &mut ast::Mod) { - noop_visit_mod(module, self); - module.items.retain(|item| match item.kind { - ast::ItemKind::Mac(_) if !self.cx.ecfg.keep_macs => false, // remove macro definitions - _ => true, - }); - } - - fn visit_mac(&mut self, _mac: &mut ast::Mac) { - // Do nothing. - } -} diff --git a/src/libsyntax/ext/proc_macro.rs b/src/libsyntax/ext/proc_macro.rs deleted file mode 100644 index 72062d2ffbf..00000000000 --- a/src/libsyntax/ext/proc_macro.rs +++ /dev/null @@ -1,219 +0,0 @@ -use crate::ast::{self, ItemKind, Attribute, Mac}; -use crate::attr::{mark_used, mark_known}; -use crate::errors::{Applicability, FatalError}; -use crate::ext::base::{self, *}; -use crate::ext::proc_macro_server; -use crate::parse::{self, token}; -use crate::symbol::sym; -use crate::tokenstream::{self, TokenStream}; -use crate::visit::Visitor; - -use rustc_data_structures::sync::Lrc; -use syntax_pos::{Span, DUMMY_SP}; - -const EXEC_STRATEGY: proc_macro::bridge::server::SameThread = - proc_macro::bridge::server::SameThread; - -pub struct BangProcMacro { - pub client: proc_macro::bridge::client::Client< - fn(proc_macro::TokenStream) -> proc_macro::TokenStream, - >, -} - -impl base::ProcMacro for BangProcMacro { - fn expand<'cx>(&self, - ecx: &'cx mut ExtCtxt<'_>, - span: Span, - input: TokenStream) - -> TokenStream { - let server = proc_macro_server::Rustc::new(ecx); - match self.client.run(&EXEC_STRATEGY, server, input) { - Ok(stream) => stream, - Err(e) => { - let msg = "proc macro panicked"; - let mut err = ecx.struct_span_fatal(span, msg); - if let Some(s) = e.as_str() { - err.help(&format!("message: {}", s)); - } - - err.emit(); - FatalError.raise(); - } - } - } -} - -pub struct AttrProcMacro { - pub client: proc_macro::bridge::client::Client< - fn(proc_macro::TokenStream, proc_macro::TokenStream) -> proc_macro::TokenStream, - >, -} - -impl base::AttrProcMacro for AttrProcMacro { - fn expand<'cx>(&self, - ecx: &'cx mut ExtCtxt<'_>, - span: Span, - annotation: TokenStream, - annotated: TokenStream) - -> TokenStream { - let server = proc_macro_server::Rustc::new(ecx); - match self.client.run(&EXEC_STRATEGY, server, annotation, annotated) { - Ok(stream) => stream, - Err(e) => { - let msg = "custom attribute panicked"; - let mut err = ecx.struct_span_fatal(span, msg); - if let Some(s) = e.as_str() { - err.help(&format!("message: {}", s)); - } - - err.emit(); - FatalError.raise(); - } - } - } -} - -pub struct ProcMacroDerive { - pub client: proc_macro::bridge::client::Client< - fn(proc_macro::TokenStream) -> proc_macro::TokenStream, - >, -} - -impl MultiItemModifier for ProcMacroDerive { - fn expand(&self, - ecx: &mut ExtCtxt<'_>, - span: Span, - _meta_item: &ast::MetaItem, - item: Annotatable) - -> Vec { - let item = match item { - Annotatable::Arm(..) | - Annotatable::Field(..) | - Annotatable::FieldPat(..) | - Annotatable::GenericParam(..) | - Annotatable::Param(..) | - Annotatable::StructField(..) | - Annotatable::Variant(..) - => panic!("unexpected annotatable"), - Annotatable::Item(item) => item, - Annotatable::ImplItem(_) | - Annotatable::TraitItem(_) | - Annotatable::ForeignItem(_) | - Annotatable::Stmt(_) | - Annotatable::Expr(_) => { - ecx.span_err(span, "proc-macro derives may only be \ - applied to a struct, enum, or union"); - return Vec::new() - } - }; - match item.kind { - ItemKind::Struct(..) | - ItemKind::Enum(..) | - ItemKind::Union(..) => {}, - _ => { - ecx.span_err(span, "proc-macro derives may only be \ - applied to a struct, enum, or union"); - return Vec::new() - } - } - - let token = token::Interpolated(Lrc::new(token::NtItem(item))); - let input = tokenstream::TokenTree::token(token, DUMMY_SP).into(); - - let server = proc_macro_server::Rustc::new(ecx); - let stream = match self.client.run(&EXEC_STRATEGY, server, input) { - Ok(stream) => stream, - Err(e) => { - let msg = "proc-macro derive panicked"; - let mut err = ecx.struct_span_fatal(span, msg); - if let Some(s) = e.as_str() { - err.help(&format!("message: {}", s)); - } - - err.emit(); - FatalError.raise(); - } - }; - - let error_count_before = ecx.parse_sess.span_diagnostic.err_count(); - let msg = "proc-macro derive produced unparseable tokens"; - - let mut parser = parse::stream_to_parser(ecx.parse_sess, stream, Some("proc-macro derive")); - let mut items = vec![]; - - loop { - match parser.parse_item() { - Ok(None) => break, - Ok(Some(item)) => { - items.push(Annotatable::Item(item)) - } - Err(mut err) => { - // FIXME: handle this better - err.cancel(); - ecx.struct_span_fatal(span, msg).emit(); - FatalError.raise(); - } - } - } - - - // fail if there have been errors emitted - if ecx.parse_sess.span_diagnostic.err_count() > error_count_before { - ecx.struct_span_fatal(span, msg).emit(); - FatalError.raise(); - } - - items - } -} - -crate struct MarkAttrs<'a>(crate &'a [ast::Name]); - -impl<'a> Visitor<'a> for MarkAttrs<'a> { - fn visit_attribute(&mut self, attr: &Attribute) { - if let Some(ident) = attr.ident() { - if self.0.contains(&ident.name) { - mark_used(attr); - mark_known(attr); - } - } - } - - fn visit_mac(&mut self, _mac: &Mac) {} -} - -pub fn is_proc_macro_attr(attr: &Attribute) -> bool { - [sym::proc_macro, sym::proc_macro_attribute, sym::proc_macro_derive] - .iter().any(|kind| attr.check_name(*kind)) -} - -crate fn collect_derives(cx: &mut ExtCtxt<'_>, attrs: &mut Vec) -> Vec { - let mut result = Vec::new(); - attrs.retain(|attr| { - if attr.path != sym::derive { - return true; - } - if !attr.is_meta_item_list() { - cx.struct_span_err(attr.span, "malformed `derive` attribute input") - .span_suggestion( - attr.span, - "missing traits to be derived", - "#[derive(Trait1, Trait2, ...)]".to_owned(), - Applicability::HasPlaceholders, - ).emit(); - return false; - } - - match attr.parse_derive_paths(cx.parse_sess) { - Ok(traits) => { - result.extend(traits); - true - } - Err(mut e) => { - e.emit(); - false - } - } - }); - result -} diff --git a/src/libsyntax/ext/proc_macro_server.rs b/src/libsyntax/ext/proc_macro_server.rs deleted file mode 100644 index e0e1a1c468c..00000000000 --- a/src/libsyntax/ext/proc_macro_server.rs +++ /dev/null @@ -1,712 +0,0 @@ -use crate::ast; -use crate::ext::base::ExtCtxt; -use crate::parse::{self, token}; -use crate::parse::lexer::comments; -use crate::print::pprust; -use crate::sess::ParseSess; -use crate::tokenstream::{self, DelimSpan, IsJoint::*, TokenStream, TreeAndJoint}; - -use errors::Diagnostic; -use rustc_data_structures::sync::Lrc; -use syntax_pos::{BytePos, FileName, MultiSpan, Pos, SourceFile, Span}; -use syntax_pos::symbol::{kw, sym, Symbol}; - -use proc_macro::{Delimiter, Level, LineColumn, Spacing}; -use proc_macro::bridge::{server, TokenTree}; -use std::{ascii, panic}; -use std::ops::Bound; - -trait FromInternal { - fn from_internal(x: T) -> Self; -} - -trait ToInternal { - fn to_internal(self) -> T; -} - -impl FromInternal for Delimiter { - fn from_internal(delim: token::DelimToken) -> Delimiter { - match delim { - token::Paren => Delimiter::Parenthesis, - token::Brace => Delimiter::Brace, - token::Bracket => Delimiter::Bracket, - token::NoDelim => Delimiter::None, - } - } -} - -impl ToInternal for Delimiter { - fn to_internal(self) -> token::DelimToken { - match self { - Delimiter::Parenthesis => token::Paren, - Delimiter::Brace => token::Brace, - Delimiter::Bracket => token::Bracket, - Delimiter::None => token::NoDelim, - } - } -} - -impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec)> - for TokenTree -{ - fn from_internal(((tree, is_joint), sess, stack): (TreeAndJoint, &ParseSess, &mut Vec)) - -> Self { - use crate::parse::token::*; - - let joint = is_joint == Joint; - let Token { kind, span } = match tree { - tokenstream::TokenTree::Delimited(span, delim, tts) => { - let delimiter = Delimiter::from_internal(delim); - return TokenTree::Group(Group { - delimiter, - stream: tts.into(), - span, - }); - } - tokenstream::TokenTree::Token(token) => token, - }; - - macro_rules! tt { - ($ty:ident { $($field:ident $(: $value:expr)*),+ $(,)? }) => ( - TokenTree::$ty(self::$ty { - $($field $(: $value)*,)+ - span, - }) - ); - ($ty:ident::$method:ident($($value:expr),*)) => ( - TokenTree::$ty(self::$ty::$method($($value,)* span)) - ); - } - macro_rules! op { - ($a:expr) => { - tt!(Punct::new($a, joint)) - }; - ($a:expr, $b:expr) => {{ - stack.push(tt!(Punct::new($b, joint))); - tt!(Punct::new($a, true)) - }}; - ($a:expr, $b:expr, $c:expr) => {{ - stack.push(tt!(Punct::new($c, joint))); - stack.push(tt!(Punct::new($b, true))); - tt!(Punct::new($a, true)) - }}; - } - - match kind { - Eq => op!('='), - Lt => op!('<'), - Le => op!('<', '='), - EqEq => op!('=', '='), - Ne => op!('!', '='), - Ge => op!('>', '='), - Gt => op!('>'), - AndAnd => op!('&', '&'), - OrOr => op!('|', '|'), - Not => op!('!'), - Tilde => op!('~'), - BinOp(Plus) => op!('+'), - BinOp(Minus) => op!('-'), - BinOp(Star) => op!('*'), - BinOp(Slash) => op!('/'), - BinOp(Percent) => op!('%'), - BinOp(Caret) => op!('^'), - BinOp(And) => op!('&'), - BinOp(Or) => op!('|'), - BinOp(Shl) => op!('<', '<'), - BinOp(Shr) => op!('>', '>'), - BinOpEq(Plus) => op!('+', '='), - BinOpEq(Minus) => op!('-', '='), - BinOpEq(Star) => op!('*', '='), - BinOpEq(Slash) => op!('/', '='), - BinOpEq(Percent) => op!('%', '='), - BinOpEq(Caret) => op!('^', '='), - BinOpEq(And) => op!('&', '='), - BinOpEq(Or) => op!('|', '='), - BinOpEq(Shl) => op!('<', '<', '='), - BinOpEq(Shr) => op!('>', '>', '='), - At => op!('@'), - Dot => op!('.'), - DotDot => op!('.', '.'), - DotDotDot => op!('.', '.', '.'), - DotDotEq => op!('.', '.', '='), - Comma => op!(','), - Semi => op!(';'), - Colon => op!(':'), - ModSep => op!(':', ':'), - RArrow => op!('-', '>'), - LArrow => op!('<', '-'), - FatArrow => op!('=', '>'), - Pound => op!('#'), - Dollar => op!('$'), - Question => op!('?'), - SingleQuote => op!('\''), - - Ident(name, false) if name == kw::DollarCrate => tt!(Ident::dollar_crate()), - Ident(name, is_raw) => tt!(Ident::new(name, is_raw)), - Lifetime(name) => { - let ident = ast::Ident::new(name, span).without_first_quote(); - stack.push(tt!(Ident::new(ident.name, false))); - tt!(Punct::new('\'', true)) - } - Literal(lit) => tt!(Literal { lit }), - DocComment(c) => { - let style = comments::doc_comment_style(&c.as_str()); - let stripped = comments::strip_doc_comment_decoration(&c.as_str()); - let mut escaped = String::new(); - for ch in stripped.chars() { - escaped.extend(ch.escape_debug()); - } - let stream = vec![ - Ident(sym::doc, false), - Eq, - TokenKind::lit(token::Str, Symbol::intern(&escaped), None), - ] - .into_iter() - .map(|kind| tokenstream::TokenTree::token(kind, span)) - .collect(); - stack.push(TokenTree::Group(Group { - delimiter: Delimiter::Bracket, - stream, - span: DelimSpan::from_single(span), - })); - if style == ast::AttrStyle::Inner { - stack.push(tt!(Punct::new('!', false))); - } - tt!(Punct::new('#', false)) - } - - Interpolated(nt) => { - let stream = parse::nt_to_tokenstream(&nt, sess, span); - TokenTree::Group(Group { - delimiter: Delimiter::None, - stream, - span: DelimSpan::from_single(span), - }) - } - - OpenDelim(..) | CloseDelim(..) => unreachable!(), - Whitespace | Comment | Shebang(..) | Unknown(..) | Eof => unreachable!(), - } - } -} - -impl ToInternal for TokenTree { - fn to_internal(self) -> TokenStream { - use crate::parse::token::*; - - let (ch, joint, span) = match self { - TokenTree::Punct(Punct { ch, joint, span }) => (ch, joint, span), - TokenTree::Group(Group { - delimiter, - stream, - span, - }) => { - return tokenstream::TokenTree::Delimited( - span, - delimiter.to_internal(), - stream.into(), - ) - .into(); - } - TokenTree::Ident(self::Ident { sym, is_raw, span }) => { - return tokenstream::TokenTree::token(Ident(sym, is_raw), span).into(); - } - TokenTree::Literal(self::Literal { - lit: token::Lit { kind: token::Integer, symbol, suffix }, - span, - }) if symbol.as_str().starts_with("-") => { - let minus = BinOp(BinOpToken::Minus); - let symbol = Symbol::intern(&symbol.as_str()[1..]); - let integer = TokenKind::lit(token::Integer, symbol, suffix); - let a = tokenstream::TokenTree::token(minus, span); - let b = tokenstream::TokenTree::token(integer, span); - return vec![a, b].into_iter().collect(); - } - TokenTree::Literal(self::Literal { - lit: token::Lit { kind: token::Float, symbol, suffix }, - span, - }) if symbol.as_str().starts_with("-") => { - let minus = BinOp(BinOpToken::Minus); - let symbol = Symbol::intern(&symbol.as_str()[1..]); - let float = TokenKind::lit(token::Float, symbol, suffix); - let a = tokenstream::TokenTree::token(minus, span); - let b = tokenstream::TokenTree::token(float, span); - return vec![a, b].into_iter().collect(); - } - TokenTree::Literal(self::Literal { lit, span }) => { - return tokenstream::TokenTree::token(Literal(lit), span).into() - } - }; - - let kind = match ch { - '=' => Eq, - '<' => Lt, - '>' => Gt, - '!' => Not, - '~' => Tilde, - '+' => BinOp(Plus), - '-' => BinOp(Minus), - '*' => BinOp(Star), - '/' => BinOp(Slash), - '%' => BinOp(Percent), - '^' => BinOp(Caret), - '&' => BinOp(And), - '|' => BinOp(Or), - '@' => At, - '.' => Dot, - ',' => Comma, - ';' => Semi, - ':' => Colon, - '#' => Pound, - '$' => Dollar, - '?' => Question, - '\'' => SingleQuote, - _ => unreachable!(), - }; - - let tree = tokenstream::TokenTree::token(kind, span); - TokenStream::new(vec![(tree, if joint { Joint } else { NonJoint })]) - } -} - -impl ToInternal for Level { - fn to_internal(self) -> errors::Level { - match self { - Level::Error => errors::Level::Error, - Level::Warning => errors::Level::Warning, - Level::Note => errors::Level::Note, - Level::Help => errors::Level::Help, - _ => unreachable!("unknown proc_macro::Level variant: {:?}", self), - } - } -} - -#[derive(Clone)] -pub struct TokenStreamIter { - cursor: tokenstream::Cursor, - stack: Vec>, -} - -#[derive(Clone)] -pub struct Group { - delimiter: Delimiter, - stream: TokenStream, - span: DelimSpan, -} - -#[derive(Copy, Clone, PartialEq, Eq, Hash)] -pub struct Punct { - ch: char, - // NB. not using `Spacing` here because it doesn't implement `Hash`. - joint: bool, - span: Span, -} - -impl Punct { - fn new(ch: char, joint: bool, span: Span) -> Punct { - const LEGAL_CHARS: &[char] = &['=', '<', '>', '!', '~', '+', '-', '*', '/', '%', '^', - '&', '|', '@', '.', ',', ';', ':', '#', '$', '?', '\'']; - if !LEGAL_CHARS.contains(&ch) { - panic!("unsupported character `{:?}`", ch) - } - Punct { ch, joint, span } - } -} - -#[derive(Copy, Clone, PartialEq, Eq, Hash)] -pub struct Ident { - sym: Symbol, - is_raw: bool, - span: Span, -} - -impl Ident { - fn is_valid(string: &str) -> bool { - let mut chars = string.chars(); - if let Some(start) = chars.next() { - rustc_lexer::is_id_start(start) && chars.all(rustc_lexer::is_id_continue) - } else { - false - } - } - fn new(sym: Symbol, is_raw: bool, span: Span) -> Ident { - let string = sym.as_str(); - if !Self::is_valid(&string) { - panic!("`{:?}` is not a valid identifier", string) - } - if is_raw && !sym.can_be_raw() { - panic!("`{}` cannot be a raw identifier", string); - } - Ident { sym, is_raw, span } - } - fn dollar_crate(span: Span) -> Ident { - // `$crate` is accepted as an ident only if it comes from the compiler. - Ident { sym: kw::DollarCrate, is_raw: false, span } - } -} - -// FIXME(eddyb) `Literal` should not expose internal `Debug` impls. -#[derive(Clone, Debug)] -pub struct Literal { - lit: token::Lit, - span: Span, -} - -pub(crate) struct Rustc<'a> { - sess: &'a ParseSess, - def_site: Span, - call_site: Span, - mixed_site: Span, -} - -impl<'a> Rustc<'a> { - pub fn new(cx: &'a ExtCtxt<'_>) -> Self { - let expn_data = cx.current_expansion.id.expn_data(); - Rustc { - sess: cx.parse_sess, - def_site: cx.with_def_site_ctxt(expn_data.def_site), - call_site: cx.with_call_site_ctxt(expn_data.call_site), - mixed_site: cx.with_mixed_site_ctxt(expn_data.call_site), - } - } - - fn lit(&mut self, kind: token::LitKind, symbol: Symbol, suffix: Option) -> Literal { - Literal { - lit: token::Lit::new(kind, symbol, suffix), - span: server::Span::call_site(self), - } - } -} - -impl server::Types for Rustc<'_> { - type TokenStream = TokenStream; - type TokenStreamBuilder = tokenstream::TokenStreamBuilder; - type TokenStreamIter = TokenStreamIter; - type Group = Group; - type Punct = Punct; - type Ident = Ident; - type Literal = Literal; - type SourceFile = Lrc; - type MultiSpan = Vec; - type Diagnostic = Diagnostic; - type Span = Span; -} - -impl server::TokenStream for Rustc<'_> { - fn new(&mut self) -> Self::TokenStream { - TokenStream::default() - } - fn is_empty(&mut self, stream: &Self::TokenStream) -> bool { - stream.is_empty() - } - fn from_str(&mut self, src: &str) -> Self::TokenStream { - parse::parse_stream_from_source_str( - FileName::proc_macro_source_code(src), - src.to_string(), - self.sess, - Some(self.call_site), - ) - } - fn to_string(&mut self, stream: &Self::TokenStream) -> String { - pprust::tts_to_string(stream.clone()) - } - fn from_token_tree( - &mut self, - tree: TokenTree, - ) -> Self::TokenStream { - tree.to_internal() - } - fn into_iter(&mut self, stream: Self::TokenStream) -> Self::TokenStreamIter { - TokenStreamIter { - cursor: stream.trees(), - stack: vec![], - } - } -} - -impl server::TokenStreamBuilder for Rustc<'_> { - fn new(&mut self) -> Self::TokenStreamBuilder { - tokenstream::TokenStreamBuilder::new() - } - fn push(&mut self, builder: &mut Self::TokenStreamBuilder, stream: Self::TokenStream) { - builder.push(stream); - } - fn build(&mut self, builder: Self::TokenStreamBuilder) -> Self::TokenStream { - builder.build() - } -} - -impl server::TokenStreamIter for Rustc<'_> { - fn next( - &mut self, - iter: &mut Self::TokenStreamIter, - ) -> Option> { - loop { - let tree = iter.stack.pop().or_else(|| { - let next = iter.cursor.next_with_joint()?; - Some(TokenTree::from_internal((next, self.sess, &mut iter.stack))) - })?; - // HACK: The condition "dummy span + group with empty delimiter" represents an AST - // fragment approximately converted into a token stream. This may happen, for - // example, with inputs to proc macro attributes, including derives. Such "groups" - // need to flattened during iteration over stream's token trees. - // Eventually this needs to be removed in favor of keeping original token trees - // and not doing the roundtrip through AST. - if let TokenTree::Group(ref group) = tree { - if group.delimiter == Delimiter::None && group.span.entire().is_dummy() { - iter.cursor.append(group.stream.clone()); - continue; - } - } - return Some(tree); - } - } -} - -impl server::Group for Rustc<'_> { - fn new(&mut self, delimiter: Delimiter, stream: Self::TokenStream) -> Self::Group { - Group { - delimiter, - stream, - span: DelimSpan::from_single(server::Span::call_site(self)), - } - } - fn delimiter(&mut self, group: &Self::Group) -> Delimiter { - group.delimiter - } - fn stream(&mut self, group: &Self::Group) -> Self::TokenStream { - group.stream.clone() - } - fn span(&mut self, group: &Self::Group) -> Self::Span { - group.span.entire() - } - fn span_open(&mut self, group: &Self::Group) -> Self::Span { - group.span.open - } - fn span_close(&mut self, group: &Self::Group) -> Self::Span { - group.span.close - } - fn set_span(&mut self, group: &mut Self::Group, span: Self::Span) { - group.span = DelimSpan::from_single(span); - } -} - -impl server::Punct for Rustc<'_> { - fn new(&mut self, ch: char, spacing: Spacing) -> Self::Punct { - Punct::new(ch, spacing == Spacing::Joint, server::Span::call_site(self)) - } - fn as_char(&mut self, punct: Self::Punct) -> char { - punct.ch - } - fn spacing(&mut self, punct: Self::Punct) -> Spacing { - if punct.joint { - Spacing::Joint - } else { - Spacing::Alone - } - } - fn span(&mut self, punct: Self::Punct) -> Self::Span { - punct.span - } - fn with_span(&mut self, punct: Self::Punct, span: Self::Span) -> Self::Punct { - Punct { span, ..punct } - } -} - -impl server::Ident for Rustc<'_> { - fn new(&mut self, string: &str, span: Self::Span, is_raw: bool) -> Self::Ident { - Ident::new(Symbol::intern(string), is_raw, span) - } - fn span(&mut self, ident: Self::Ident) -> Self::Span { - ident.span - } - fn with_span(&mut self, ident: Self::Ident, span: Self::Span) -> Self::Ident { - Ident { span, ..ident } - } -} - -impl server::Literal for Rustc<'_> { - // FIXME(eddyb) `Literal` should not expose internal `Debug` impls. - fn debug(&mut self, literal: &Self::Literal) -> String { - format!("{:?}", literal) - } - fn integer(&mut self, n: &str) -> Self::Literal { - self.lit(token::Integer, Symbol::intern(n), None) - } - fn typed_integer(&mut self, n: &str, kind: &str) -> Self::Literal { - self.lit(token::Integer, Symbol::intern(n), Some(Symbol::intern(kind))) - } - fn float(&mut self, n: &str) -> Self::Literal { - self.lit(token::Float, Symbol::intern(n), None) - } - fn f32(&mut self, n: &str) -> Self::Literal { - self.lit(token::Float, Symbol::intern(n), Some(sym::f32)) - } - fn f64(&mut self, n: &str) -> Self::Literal { - self.lit(token::Float, Symbol::intern(n), Some(sym::f64)) - } - fn string(&mut self, string: &str) -> Self::Literal { - let mut escaped = String::new(); - for ch in string.chars() { - escaped.extend(ch.escape_debug()); - } - self.lit(token::Str, Symbol::intern(&escaped), None) - } - fn character(&mut self, ch: char) -> Self::Literal { - let mut escaped = String::new(); - escaped.extend(ch.escape_unicode()); - self.lit(token::Char, Symbol::intern(&escaped), None) - } - fn byte_string(&mut self, bytes: &[u8]) -> Self::Literal { - let string = bytes - .iter() - .cloned() - .flat_map(ascii::escape_default) - .map(Into::::into) - .collect::(); - self.lit(token::ByteStr, Symbol::intern(&string), None) - } - fn span(&mut self, literal: &Self::Literal) -> Self::Span { - literal.span - } - fn set_span(&mut self, literal: &mut Self::Literal, span: Self::Span) { - literal.span = span; - } - fn subspan( - &mut self, - literal: &Self::Literal, - start: Bound, - end: Bound, - ) -> Option { - let span = literal.span; - let length = span.hi().to_usize() - span.lo().to_usize(); - - let start = match start { - Bound::Included(lo) => lo, - Bound::Excluded(lo) => lo + 1, - Bound::Unbounded => 0, - }; - - let end = match end { - Bound::Included(hi) => hi + 1, - Bound::Excluded(hi) => hi, - Bound::Unbounded => length, - }; - - // Bounds check the values, preventing addition overflow and OOB spans. - if start > u32::max_value() as usize - || end > u32::max_value() as usize - || (u32::max_value() - start as u32) < span.lo().to_u32() - || (u32::max_value() - end as u32) < span.lo().to_u32() - || start >= end - || end > length - { - return None; - } - - let new_lo = span.lo() + BytePos::from_usize(start); - let new_hi = span.lo() + BytePos::from_usize(end); - Some(span.with_lo(new_lo).with_hi(new_hi)) - } -} - -impl server::SourceFile for Rustc<'_> { - fn eq(&mut self, file1: &Self::SourceFile, file2: &Self::SourceFile) -> bool { - Lrc::ptr_eq(file1, file2) - } - fn path(&mut self, file: &Self::SourceFile) -> String { - match file.name { - FileName::Real(ref path) => path - .to_str() - .expect("non-UTF8 file path in `proc_macro::SourceFile::path`") - .to_string(), - _ => file.name.to_string(), - } - } - fn is_real(&mut self, file: &Self::SourceFile) -> bool { - file.is_real_file() - } -} - -impl server::MultiSpan for Rustc<'_> { - fn new(&mut self) -> Self::MultiSpan { - vec![] - } - fn push(&mut self, spans: &mut Self::MultiSpan, span: Self::Span) { - spans.push(span) - } -} - -impl server::Diagnostic for Rustc<'_> { - fn new(&mut self, level: Level, msg: &str, spans: Self::MultiSpan) -> Self::Diagnostic { - let mut diag = Diagnostic::new(level.to_internal(), msg); - diag.set_span(MultiSpan::from_spans(spans)); - diag - } - fn sub( - &mut self, - diag: &mut Self::Diagnostic, - level: Level, - msg: &str, - spans: Self::MultiSpan, - ) { - diag.sub(level.to_internal(), msg, MultiSpan::from_spans(spans), None); - } - fn emit(&mut self, diag: Self::Diagnostic) { - self.sess.span_diagnostic.emit_diagnostic(&diag); - } -} - -impl server::Span for Rustc<'_> { - fn debug(&mut self, span: Self::Span) -> String { - format!("{:?} bytes({}..{})", span.ctxt(), span.lo().0, span.hi().0) - } - fn def_site(&mut self) -> Self::Span { - self.def_site - } - fn call_site(&mut self) -> Self::Span { - self.call_site - } - fn mixed_site(&mut self) -> Self::Span { - self.mixed_site - } - fn source_file(&mut self, span: Self::Span) -> Self::SourceFile { - self.sess.source_map().lookup_char_pos(span.lo()).file - } - fn parent(&mut self, span: Self::Span) -> Option { - span.parent() - } - fn source(&mut self, span: Self::Span) -> Self::Span { - span.source_callsite() - } - fn start(&mut self, span: Self::Span) -> LineColumn { - let loc = self.sess.source_map().lookup_char_pos(span.lo()); - LineColumn { - line: loc.line, - column: loc.col.to_usize(), - } - } - fn end(&mut self, span: Self::Span) -> LineColumn { - let loc = self.sess.source_map().lookup_char_pos(span.hi()); - LineColumn { - line: loc.line, - column: loc.col.to_usize(), - } - } - fn join(&mut self, first: Self::Span, second: Self::Span) -> Option { - let self_loc = self.sess.source_map().lookup_char_pos(first.lo()); - let other_loc = self.sess.source_map().lookup_char_pos(second.lo()); - - if self_loc.file.name != other_loc.file.name { - return None; - } - - Some(first.to(second)) - } - fn resolved_at(&mut self, span: Self::Span, at: Self::Span) -> Self::Span { - span.with_ctxt(at.ctxt()) - } - fn source_text(&mut self, span: Self::Span) -> Option { - self.sess.source_map().span_to_snippet(span).ok() - } -} diff --git a/src/libsyntax/feature_gate/check.rs b/src/libsyntax/feature_gate/check.rs index c07b6050afe..172511f0f09 100644 --- a/src/libsyntax/feature_gate/check.rs +++ b/src/libsyntax/feature_gate/check.rs @@ -56,7 +56,7 @@ macro_rules! gate_feature { }; } -crate fn check_attribute(attr: &ast::Attribute, parse_sess: &ParseSess, features: &Features) { +pub fn check_attribute(attr: &ast::Attribute, parse_sess: &ParseSess, features: &Features) { PostExpansionVisitor { parse_sess, features }.visit_attribute(attr) } diff --git a/src/libsyntax/feature_gate/mod.rs b/src/libsyntax/feature_gate/mod.rs index ca13ab36205..ba970618c0e 100644 --- a/src/libsyntax/feature_gate/mod.rs +++ b/src/libsyntax/feature_gate/mod.rs @@ -58,8 +58,7 @@ pub use builtin_attrs::{ deprecated_attributes, is_builtin_attr, is_builtin_attr_name, }; pub use check::{ - check_crate, get_features, feature_err, emit_feature_err, + check_crate, check_attribute, get_features, feature_err, emit_feature_err, Stability, GateIssue, UnstableFeatures, EXPLAIN_STMT_ATTR_SYNTAX, EXPLAIN_UNSIZED_TUPLE_COERCION, }; -crate use check::check_attribute; diff --git a/src/libsyntax/lib.rs b/src/libsyntax/lib.rs index fa75f5624f8..a68b7fdf931 100644 --- a/src/libsyntax/lib.rs +++ b/src/libsyntax/lib.rs @@ -13,17 +13,12 @@ #![feature(crate_visibility_modifier)] #![feature(label_break_value)] #![feature(nll)] -#![feature(proc_macro_diagnostic)] -#![feature(proc_macro_internals)] -#![feature(proc_macro_span)] #![feature(try_trait)] #![feature(slice_patterns)] #![feature(unicode_internals)] #![recursion_limit="256"] -extern crate proc_macro; - pub use errors; use rustc_data_structures::sync::Lock; use rustc_index::bit_set::GrowableBitSet; @@ -34,26 +29,7 @@ use syntax_pos::edition::Edition; #[cfg(test)] mod tests; -const MACRO_ARGUMENTS: Option<&'static str> = Some("macro arguments"); - -// A variant of 'try!' that panics on an Err. This is used as a crutch on the -// way towards a non-panic!-prone parser. It should be used for fatal parsing -// errors; eventually we plan to convert all code using panictry to just use -// normal try. -#[macro_export] -macro_rules! panictry { - ($e:expr) => ({ - use std::result::Result::{Ok, Err}; - use errors::FatalError; - match $e { - Ok(e) => e, - Err(mut e) => { - e.emit(); - FatalError.raise() - } - } - }) -} +pub const MACRO_ARGUMENTS: Option<&'static str> = Some("macro arguments"); // A variant of 'panictry!' that works on a Vec instead of a single DiagnosticBuilder. macro_rules! panictry_buffer { @@ -157,19 +133,4 @@ pub mod print { mod helpers; } -pub mod ext { - mod placeholders; - mod proc_macro_server; - - pub use syntax_pos::hygiene; - pub use mbe::macro_rules::compile_declarative_macro; - pub mod allocator; - pub mod base; - pub mod build; - pub mod expand; - pub mod proc_macro; - - crate mod mbe; -} - pub mod early_buffered_lints; diff --git a/src/libsyntax/parse/literal.rs b/src/libsyntax/parse/literal.rs index 54e523430e4..14e1696610a 100644 --- a/src/libsyntax/parse/literal.rs +++ b/src/libsyntax/parse/literal.rs @@ -212,7 +212,7 @@ impl Lit { /// Attempts to recover an AST literal from semantic literal. /// This function is used when the original token doesn't exist (e.g. the literal is created /// by an AST-based macro) or unavailable (e.g. from HIR pretty-printing). - crate fn from_lit_kind(kind: LitKind, span: Span) -> Lit { + pub fn from_lit_kind(kind: LitKind, span: Span) -> Lit { Lit { token: kind.to_lit_token(), kind, span } } diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index 7914fdbf978..64ec943294a 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -5,7 +5,7 @@ mod item; mod module; mod ty; mod path; -crate use path::PathStyle; +pub use path::PathStyle; mod stmt; mod generics; mod diagnostics; @@ -130,7 +130,7 @@ pub struct Parser<'a> { /// Name of the root module this parser originated from. If `None`, then the /// name is not known. This does not change while the parser is descending /// into modules, and sub-parsers have new values for this name. - crate root_module_name: Option, + pub root_module_name: Option, expected_tokens: Vec, token_cursor: TokenCursor, desugar_doc_comments: bool, @@ -148,7 +148,7 @@ pub struct Parser<'a> { /// error. pub(super) unclosed_delims: Vec, last_unexpected_token_span: Option, - crate last_type_ascription: Option<(Span, bool /* likely path typo */)>, + pub last_type_ascription: Option<(Span, bool /* likely path typo */)>, /// If present, this `Parser` is not parsing Rust code but rather a macro call. subparser_name: Option<&'static str>, } @@ -330,7 +330,7 @@ enum TokenExpectType { } impl<'a> Parser<'a> { - crate fn new( + pub fn new( sess: &'a ParseSess, tokens: TokenStream, directory: Option>, @@ -1003,7 +1003,7 @@ impl<'a> Parser<'a> { } } - crate fn process_potential_macro_variable(&mut self) { + pub fn process_potential_macro_variable(&mut self) { self.token = match self.token.kind { token::Dollar if self.token.span.from_expansion() && self.look_ahead(1, |t| t.is_ident()) => { @@ -1037,7 +1037,7 @@ impl<'a> Parser<'a> { } /// Parses a single token tree from the input. - crate fn parse_token_tree(&mut self) -> TokenTree { + pub fn parse_token_tree(&mut self) -> TokenTree { match self.token.kind { token::OpenDelim(..) => { let frame = mem::replace(&mut self.token_cursor.frame, @@ -1099,7 +1099,7 @@ impl<'a> Parser<'a> { /// If the following element can't be a tuple (i.e., it's a function definition), then /// it's not a tuple struct field), and the contents within the parentheses isn't valid, /// so emit a proper diagnostic. - crate fn parse_visibility(&mut self, can_take_tuple: bool) -> PResult<'a, Visibility> { + pub fn parse_visibility(&mut self, can_take_tuple: bool) -> PResult<'a, Visibility> { maybe_whole!(self, NtVis, |x| x); self.expected_tokens.push(TokenType::Keyword(kw::Crate)); diff --git a/src/libsyntax/parse/parser/diagnostics.rs b/src/libsyntax/parse/parser/diagnostics.rs index a6884ec2c72..32a41bb0e27 100644 --- a/src/libsyntax/parse/parser/diagnostics.rs +++ b/src/libsyntax/parse/parser/diagnostics.rs @@ -170,7 +170,7 @@ impl RecoverQPath for Expr { } impl<'a> Parser<'a> { - crate fn fatal(&self, m: &str) -> DiagnosticBuilder<'a> { + pub fn fatal(&self, m: &str) -> DiagnosticBuilder<'a> { self.span_fatal(self.token.span, m) } @@ -190,11 +190,11 @@ impl<'a> Parser<'a> { self.sess.span_diagnostic.span_err(sp, m) } - crate fn struct_span_err>(&self, sp: S, m: &str) -> DiagnosticBuilder<'a> { + pub fn struct_span_err>(&self, sp: S, m: &str) -> DiagnosticBuilder<'a> { self.sess.span_diagnostic.struct_span_err(sp, m) } - crate fn span_bug>(&self, sp: S, m: &str) -> ! { + pub fn span_bug>(&self, sp: S, m: &str) -> ! { self.sess.span_diagnostic.span_bug(sp, m) } diff --git a/src/libsyntax/parse/parser/expr.rs b/src/libsyntax/parse/parser/expr.rs index bf9526dece2..273f5a5ffa3 100644 --- a/src/libsyntax/parse/parser/expr.rs +++ b/src/libsyntax/parse/parser/expr.rs @@ -1233,7 +1233,7 @@ impl<'a> Parser<'a> { } /// Matches `'-' lit | lit` (cf. `ast_validation::AstValidator::check_expr_within_pat`). - crate fn parse_literal_maybe_minus(&mut self) -> PResult<'a, P> { + pub fn parse_literal_maybe_minus(&mut self) -> PResult<'a, P> { maybe_whole_expr!(self); let minus_lo = self.token.span; diff --git a/src/libsyntax/parse/parser/item.rs b/src/libsyntax/parse/parser/item.rs index 6a1cbd2ebb7..0acfd1450d8 100644 --- a/src/libsyntax/parse/parser/item.rs +++ b/src/libsyntax/parse/parser/item.rs @@ -674,7 +674,7 @@ impl<'a> Parser<'a> { } /// Parses an impl item. - crate fn parse_impl_item(&mut self, at_end: &mut bool) -> PResult<'a, ImplItem> { + pub fn parse_impl_item(&mut self, at_end: &mut bool) -> PResult<'a, ImplItem> { maybe_whole!(self, NtImplItem, |x| x); let attrs = self.parse_outer_attributes()?; let mut unclosed_delims = vec![]; @@ -850,7 +850,7 @@ impl<'a> Parser<'a> { } /// Parses the items in a trait declaration. - crate fn parse_trait_item(&mut self, at_end: &mut bool) -> PResult<'a, TraitItem> { + pub fn parse_trait_item(&mut self, at_end: &mut bool) -> PResult<'a, TraitItem> { maybe_whole!(self, NtTraitItem, |x| x); let attrs = self.parse_outer_attributes()?; let mut unclosed_delims = vec![]; @@ -1116,7 +1116,7 @@ impl<'a> Parser<'a> { } /// Parses a foreign item. - crate fn parse_foreign_item(&mut self, extern_sp: Span) -> PResult<'a, ForeignItem> { + pub fn parse_foreign_item(&mut self, extern_sp: Span) -> PResult<'a, ForeignItem> { maybe_whole!(self, NtForeignItem, |ni| ni); let attrs = self.parse_outer_attributes()?; diff --git a/src/libsyntax/parse/parser/pat.rs b/src/libsyntax/parse/parser/pat.rs index b30e0636bb1..af795e51792 100644 --- a/src/libsyntax/parse/parser/pat.rs +++ b/src/libsyntax/parse/parser/pat.rs @@ -34,7 +34,7 @@ impl<'a> Parser<'a> { /// Corresponds to `pat` in RFC 2535 and does not admit or-patterns /// at the top level. Used when parsing the parameters of lambda expressions, /// functions, function pointers, and `pat` macro fragments. - crate fn parse_pat(&mut self, expected: Expected) -> PResult<'a, P> { + pub fn parse_pat(&mut self, expected: Expected) -> PResult<'a, P> { self.parse_pat_with_range_pat(true, expected) } diff --git a/src/libsyntax/parse/parser/path.rs b/src/libsyntax/parse/parser/path.rs index 0f28fb11cde..639d61a2b5c 100644 --- a/src/libsyntax/parse/parser/path.rs +++ b/src/libsyntax/parse/parser/path.rs @@ -13,7 +13,7 @@ use errors::{Applicability, pluralise}; /// Specifies how to parse a path. #[derive(Copy, Clone, PartialEq)] -crate enum PathStyle { +pub enum PathStyle { /// In some contexts, notably in expressions, paths with generic arguments are ambiguous /// with something else. For example, in expressions `segment < ....` can be interpreted /// as a comparison and `segment ( ....` can be interpreted as a function call. @@ -88,7 +88,7 @@ impl<'a> Parser<'a> { /// `a::b::C::` (with disambiguator) /// `Fn(Args)` (without disambiguator) /// `Fn::(Args)` (with disambiguator) - crate fn parse_path(&mut self, style: PathStyle) -> PResult<'a, Path> { + pub fn parse_path(&mut self, style: PathStyle) -> PResult<'a, Path> { maybe_whole!(self, NtPath, |path| { if style == PathStyle::Mod && path.segments.iter().any(|segment| segment.args.is_some()) { diff --git a/src/libsyntax/parse/parser/stmt.rs b/src/libsyntax/parse/parser/stmt.rs index f656bc11e1c..4b1fdb2eef3 100644 --- a/src/libsyntax/parse/parser/stmt.rs +++ b/src/libsyntax/parse/parser/stmt.rs @@ -19,7 +19,7 @@ use errors::Applicability; impl<'a> Parser<'a> { /// Parses a statement. This stops just before trailing semicolons on everything but items. /// e.g., a `StmtKind::Semi` parses to a `StmtKind::Expr`, leaving the trailing `;` unconsumed. - crate fn parse_stmt(&mut self) -> PResult<'a, Option> { + pub fn parse_stmt(&mut self) -> PResult<'a, Option> { Ok(self.parse_stmt_(true)) } @@ -299,7 +299,7 @@ impl<'a> Parser<'a> { } /// Parses a block. No inner attributes are allowed. - crate fn parse_block(&mut self) -> PResult<'a, P> { + pub fn parse_block(&mut self) -> PResult<'a, P> { maybe_whole!(self, NtBlock, |x| x); let lo = self.token.span; @@ -421,7 +421,7 @@ impl<'a> Parser<'a> { } /// Parses a statement, including the trailing semicolon. - crate fn parse_full_stmt(&mut self, macro_legacy_warnings: bool) -> PResult<'a, Option> { + pub fn parse_full_stmt(&mut self, macro_legacy_warnings: bool) -> PResult<'a, Option> { // Skip looking for a trailing semicolon when we have an interpolated statement. maybe_whole!(self, NtStmt, |x| Some(x)); diff --git a/src/libsyntax/parse/parser/ty.rs b/src/libsyntax/parse/parser/ty.rs index ae9f4d06c05..86c94b680b2 100644 --- a/src/libsyntax/parse/parser/ty.rs +++ b/src/libsyntax/parse/parser/ty.rs @@ -433,13 +433,13 @@ impl<'a> Parser<'a> { } } - crate fn check_lifetime(&mut self) -> bool { + pub fn check_lifetime(&mut self) -> bool { self.expected_tokens.push(TokenType::Lifetime); self.token.is_lifetime() } /// Parses a single lifetime `'a` or panics. - crate fn expect_lifetime(&mut self) -> Lifetime { + pub fn expect_lifetime(&mut self) -> Lifetime { if let Some(ident) = self.token.lifetime() { let span = self.token.span; self.bump(); diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index eb74ab2b919..e527989fb0b 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -285,7 +285,7 @@ impl TokenKind { } impl Token { - crate fn new(kind: TokenKind, span: Span) -> Self { + pub fn new(kind: TokenKind, span: Span) -> Self { Token { kind, span } } @@ -295,12 +295,12 @@ impl Token { } /// Recovers a `Token` from an `ast::Ident`. This creates a raw identifier if necessary. - crate fn from_ast_ident(ident: ast::Ident) -> Self { + pub fn from_ast_ident(ident: ast::Ident) -> Self { Token::new(Ident(ident.name, ident.is_raw_guess()), ident.span) } /// Return this token by value and leave a dummy token in its place. - crate fn take(&mut self) -> Self { + pub fn take(&mut self) -> Self { mem::replace(self, Token::dummy()) } @@ -321,7 +321,7 @@ impl Token { } /// Returns `true` if the token can appear at the start of an expression. - crate fn can_begin_expr(&self) -> bool { + pub fn can_begin_expr(&self) -> bool { match self.kind { Ident(name, is_raw) => ident_can_begin_expr(name, self.span, is_raw), // value name or keyword @@ -353,7 +353,7 @@ impl Token { } /// Returns `true` if the token can appear at the start of a type. - crate fn can_begin_type(&self) -> bool { + pub fn can_begin_type(&self) -> bool { match self.kind { Ident(name, is_raw) => ident_can_begin_type(name, self.span, is_raw), // type name or keyword @@ -396,7 +396,7 @@ impl Token { } /// Returns `true` if the token is any literal - crate fn is_lit(&self) -> bool { + pub fn is_lit(&self) -> bool { match self.kind { Literal(..) => true, _ => false, @@ -412,7 +412,7 @@ impl Token { /// Returns `true` if the token is any literal, a minus (which can prefix a literal, /// for example a '-42', or one of the boolean idents). - crate fn can_begin_literal_or_bool(&self) -> bool { + pub fn can_begin_literal_or_bool(&self) -> bool { match self.kind { Literal(..) | BinOp(Minus) => true, Ident(name, false) if name.is_bool_lit() => true, diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index 0ff1c26bac2..db6832d6423 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -156,7 +156,7 @@ use IsJoint::*; impl TokenStream { /// Given a `TokenStream` with a `Stream` of only two arguments, return a new `TokenStream` /// separating the two arguments with a comma for diagnostic suggestions. - pub(crate) fn add_comma(&self) -> Option<(TokenStream, Span)> { + pub fn add_comma(&self) -> Option<(TokenStream, Span)> { // Used to suggest if a user writes `foo!(a b);` let mut suggestion = None; let mut iter = self.0.iter().enumerate().peekable(); diff --git a/src/libsyntax_expand/Cargo.toml b/src/libsyntax_expand/Cargo.toml new file mode 100644 index 00000000000..f063753f599 --- /dev/null +++ b/src/libsyntax_expand/Cargo.toml @@ -0,0 +1,26 @@ +[package] +authors = ["The Rust Project Developers"] +name = "syntax_expand" +version = "0.0.0" +edition = "2018" +build = false + +[lib] +name = "syntax_expand" +path = "lib.rs" +doctest = false + +[dependencies] +bitflags = "1.0" +rustc_serialize = { path = "../libserialize", package = "serialize" } +log = "0.4" +scoped-tls = "1.0" +lazy_static = "1.0.0" +syntax_pos = { path = "../libsyntax_pos" } +errors = { path = "../librustc_errors", package = "rustc_errors" } +rustc_data_structures = { path = "../librustc_data_structures" } +rustc_index = { path = "../librustc_index" } +rustc_lexer = { path = "../librustc_lexer" } +rustc_target = { path = "../librustc_target" } +smallvec = { version = "0.6.7", features = ["union", "may_dangle"] } +syntax = { path = "../libsyntax" } diff --git a/src/libsyntax_expand/allocator.rs b/src/libsyntax_expand/allocator.rs new file mode 100644 index 00000000000..3526be17721 --- /dev/null +++ b/src/libsyntax_expand/allocator.rs @@ -0,0 +1,75 @@ +use syntax::{ast, attr, visit}; +use syntax::symbol::{sym, Symbol}; +use syntax_pos::Span; + +#[derive(Clone, Copy)] +pub enum AllocatorKind { + Global, + DefaultLib, + DefaultExe, +} + +impl AllocatorKind { + pub fn fn_name(&self, base: &str) -> String { + match *self { + AllocatorKind::Global => format!("__rg_{}", base), + AllocatorKind::DefaultLib => format!("__rdl_{}", base), + AllocatorKind::DefaultExe => format!("__rde_{}", base), + } + } +} + +pub enum AllocatorTy { + Layout, + Ptr, + ResultPtr, + Unit, + Usize, +} + +pub struct AllocatorMethod { + pub name: &'static str, + pub inputs: &'static [AllocatorTy], + pub output: AllocatorTy, +} + +pub static ALLOCATOR_METHODS: &[AllocatorMethod] = &[ + AllocatorMethod { + name: "alloc", + inputs: &[AllocatorTy::Layout], + output: AllocatorTy::ResultPtr, + }, + AllocatorMethod { + name: "dealloc", + inputs: &[AllocatorTy::Ptr, AllocatorTy::Layout], + output: AllocatorTy::Unit, + }, + AllocatorMethod { + name: "realloc", + inputs: &[AllocatorTy::Ptr, AllocatorTy::Layout, AllocatorTy::Usize], + output: AllocatorTy::ResultPtr, + }, + AllocatorMethod { + name: "alloc_zeroed", + inputs: &[AllocatorTy::Layout], + output: AllocatorTy::ResultPtr, + }, +]; + +pub fn global_allocator_spans(krate: &ast::Crate) -> Vec { + struct Finder { name: Symbol, spans: Vec } + impl<'ast> visit::Visitor<'ast> for Finder { + fn visit_item(&mut self, item: &'ast ast::Item) { + if item.ident.name == self.name && + attr::contains_name(&item.attrs, sym::rustc_std_internal_symbol) { + self.spans.push(item.span); + } + visit::walk_item(self, item) + } + } + + let name = Symbol::intern(&AllocatorKind::Global.fn_name("alloc")); + let mut f = Finder { name, spans: Vec::new() }; + visit::walk_crate(&mut f, krate); + f.spans +} diff --git a/src/libsyntax_expand/base.rs b/src/libsyntax_expand/base.rs new file mode 100644 index 00000000000..593e06f29b9 --- /dev/null +++ b/src/libsyntax_expand/base.rs @@ -0,0 +1,1189 @@ +use crate::expand::{self, AstFragment, Invocation}; +use crate::hygiene::ExpnId; + +use syntax::ast::{self, NodeId, Attribute, Name, PatKind}; +use syntax::attr::{self, HasAttrs, Stability, Deprecation}; +use syntax::source_map::SourceMap; +use syntax::edition::Edition; +use syntax::mut_visit::{self, MutVisitor}; +use syntax::parse::{self, parser, DirectoryOwnership}; +use syntax::parse::token; +use syntax::ptr::P; +use syntax::sess::ParseSess; +use syntax::symbol::{kw, sym, Ident, Symbol}; +use syntax::{ThinVec, MACRO_ARGUMENTS}; +use syntax::tokenstream::{self, TokenStream}; +use syntax::visit::Visitor; + +use errors::{DiagnosticBuilder, DiagnosticId}; +use smallvec::{smallvec, SmallVec}; +use syntax_pos::{FileName, Span, MultiSpan, DUMMY_SP}; +use syntax_pos::hygiene::{AstPass, ExpnData, ExpnKind}; + +use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::sync::{self, Lrc}; +use std::iter; +use std::path::PathBuf; +use std::rc::Rc; +use std::default::Default; + +pub use syntax_pos::hygiene::MacroKind; + +#[derive(Debug,Clone)] +pub enum Annotatable { + Item(P), + TraitItem(P), + ImplItem(P), + ForeignItem(P), + Stmt(P), + Expr(P), + Arm(ast::Arm), + Field(ast::Field), + FieldPat(ast::FieldPat), + GenericParam(ast::GenericParam), + Param(ast::Param), + StructField(ast::StructField), + Variant(ast::Variant), +} + +impl HasAttrs for Annotatable { + fn attrs(&self) -> &[Attribute] { + match *self { + Annotatable::Item(ref item) => &item.attrs, + Annotatable::TraitItem(ref trait_item) => &trait_item.attrs, + Annotatable::ImplItem(ref impl_item) => &impl_item.attrs, + Annotatable::ForeignItem(ref foreign_item) => &foreign_item.attrs, + Annotatable::Stmt(ref stmt) => stmt.attrs(), + Annotatable::Expr(ref expr) => &expr.attrs, + Annotatable::Arm(ref arm) => &arm.attrs, + Annotatable::Field(ref field) => &field.attrs, + Annotatable::FieldPat(ref fp) => &fp.attrs, + Annotatable::GenericParam(ref gp) => &gp.attrs, + Annotatable::Param(ref p) => &p.attrs, + Annotatable::StructField(ref sf) => &sf.attrs, + Annotatable::Variant(ref v) => &v.attrs(), + } + } + + fn visit_attrs)>(&mut self, f: F) { + match self { + Annotatable::Item(item) => item.visit_attrs(f), + Annotatable::TraitItem(trait_item) => trait_item.visit_attrs(f), + Annotatable::ImplItem(impl_item) => impl_item.visit_attrs(f), + Annotatable::ForeignItem(foreign_item) => foreign_item.visit_attrs(f), + Annotatable::Stmt(stmt) => stmt.visit_attrs(f), + Annotatable::Expr(expr) => expr.visit_attrs(f), + Annotatable::Arm(arm) => arm.visit_attrs(f), + Annotatable::Field(field) => field.visit_attrs(f), + Annotatable::FieldPat(fp) => fp.visit_attrs(f), + Annotatable::GenericParam(gp) => gp.visit_attrs(f), + Annotatable::Param(p) => p.visit_attrs(f), + Annotatable::StructField(sf) => sf.visit_attrs(f), + Annotatable::Variant(v) => v.visit_attrs(f), + } + } +} + +impl Annotatable { + pub fn span(&self) -> Span { + match *self { + Annotatable::Item(ref item) => item.span, + Annotatable::TraitItem(ref trait_item) => trait_item.span, + Annotatable::ImplItem(ref impl_item) => impl_item.span, + Annotatable::ForeignItem(ref foreign_item) => foreign_item.span, + Annotatable::Stmt(ref stmt) => stmt.span, + Annotatable::Expr(ref expr) => expr.span, + Annotatable::Arm(ref arm) => arm.span, + Annotatable::Field(ref field) => field.span, + Annotatable::FieldPat(ref fp) => fp.pat.span, + Annotatable::GenericParam(ref gp) => gp.ident.span, + Annotatable::Param(ref p) => p.span, + Annotatable::StructField(ref sf) => sf.span, + Annotatable::Variant(ref v) => v.span, + } + } + + pub fn visit_with<'a, V: Visitor<'a>>(&'a self, visitor: &mut V) { + match self { + Annotatable::Item(item) => visitor.visit_item(item), + Annotatable::TraitItem(trait_item) => visitor.visit_trait_item(trait_item), + Annotatable::ImplItem(impl_item) => visitor.visit_impl_item(impl_item), + Annotatable::ForeignItem(foreign_item) => visitor.visit_foreign_item(foreign_item), + Annotatable::Stmt(stmt) => visitor.visit_stmt(stmt), + Annotatable::Expr(expr) => visitor.visit_expr(expr), + Annotatable::Arm(arm) => visitor.visit_arm(arm), + Annotatable::Field(field) => visitor.visit_field(field), + Annotatable::FieldPat(fp) => visitor.visit_field_pattern(fp), + Annotatable::GenericParam(gp) => visitor.visit_generic_param(gp), + Annotatable::Param(p) => visitor.visit_param(p), + Annotatable::StructField(sf) =>visitor.visit_struct_field(sf), + Annotatable::Variant(v) => visitor.visit_variant(v), + } + } + + pub fn expect_item(self) -> P { + match self { + Annotatable::Item(i) => i, + _ => panic!("expected Item") + } + } + + pub fn map_item_or(self, mut f: F, mut or: G) -> Annotatable + where F: FnMut(P) -> P, + G: FnMut(Annotatable) -> Annotatable + { + match self { + Annotatable::Item(i) => Annotatable::Item(f(i)), + _ => or(self) + } + } + + pub fn expect_trait_item(self) -> ast::TraitItem { + match self { + Annotatable::TraitItem(i) => i.into_inner(), + _ => panic!("expected Item") + } + } + + pub fn expect_impl_item(self) -> ast::ImplItem { + match self { + Annotatable::ImplItem(i) => i.into_inner(), + _ => panic!("expected Item") + } + } + + pub fn expect_foreign_item(self) -> ast::ForeignItem { + match self { + Annotatable::ForeignItem(i) => i.into_inner(), + _ => panic!("expected foreign item") + } + } + + pub fn expect_stmt(self) -> ast::Stmt { + match self { + Annotatable::Stmt(stmt) => stmt.into_inner(), + _ => panic!("expected statement"), + } + } + + pub fn expect_expr(self) -> P { + match self { + Annotatable::Expr(expr) => expr, + _ => panic!("expected expression"), + } + } + + pub fn expect_arm(self) -> ast::Arm { + match self { + Annotatable::Arm(arm) => arm, + _ => panic!("expected match arm") + } + } + + pub fn expect_field(self) -> ast::Field { + match self { + Annotatable::Field(field) => field, + _ => panic!("expected field") + } + } + + pub fn expect_field_pattern(self) -> ast::FieldPat { + match self { + Annotatable::FieldPat(fp) => fp, + _ => panic!("expected field pattern") + } + } + + pub fn expect_generic_param(self) -> ast::GenericParam { + match self { + Annotatable::GenericParam(gp) => gp, + _ => panic!("expected generic parameter") + } + } + + pub fn expect_param(self) -> ast::Param { + match self { + Annotatable::Param(param) => param, + _ => panic!("expected parameter") + } + } + + pub fn expect_struct_field(self) -> ast::StructField { + match self { + Annotatable::StructField(sf) => sf, + _ => panic!("expected struct field") + } + } + + pub fn expect_variant(self) -> ast::Variant { + match self { + Annotatable::Variant(v) => v, + _ => panic!("expected variant") + } + } + + pub fn derive_allowed(&self) -> bool { + match *self { + Annotatable::Item(ref item) => match item.kind { + ast::ItemKind::Struct(..) | + ast::ItemKind::Enum(..) | + ast::ItemKind::Union(..) => true, + _ => false, + }, + _ => false, + } + } +} + +// `meta_item` is the annotation, and `item` is the item being modified. +// FIXME Decorators should follow the same pattern too. +pub trait MultiItemModifier { + fn expand(&self, + ecx: &mut ExtCtxt<'_>, + span: Span, + meta_item: &ast::MetaItem, + item: Annotatable) + -> Vec; +} + +impl MultiItemModifier for F + where F: Fn(&mut ExtCtxt<'_>, Span, &ast::MetaItem, Annotatable) -> T, + T: Into>, +{ + fn expand(&self, + ecx: &mut ExtCtxt<'_>, + span: Span, + meta_item: &ast::MetaItem, + item: Annotatable) + -> Vec { + (*self)(ecx, span, meta_item, item).into() + } +} + +impl Into> for Annotatable { + fn into(self) -> Vec { + vec![self] + } +} + +pub trait ProcMacro { + fn expand<'cx>(&self, + ecx: &'cx mut ExtCtxt<'_>, + span: Span, + ts: TokenStream) + -> TokenStream; +} + +impl ProcMacro for F + where F: Fn(TokenStream) -> TokenStream +{ + fn expand<'cx>(&self, + _ecx: &'cx mut ExtCtxt<'_>, + _span: Span, + ts: TokenStream) + -> TokenStream { + // FIXME setup implicit context in TLS before calling self. + (*self)(ts) + } +} + +pub trait AttrProcMacro { + fn expand<'cx>(&self, + ecx: &'cx mut ExtCtxt<'_>, + span: Span, + annotation: TokenStream, + annotated: TokenStream) + -> TokenStream; +} + +impl AttrProcMacro for F + where F: Fn(TokenStream, TokenStream) -> TokenStream +{ + fn expand<'cx>(&self, + _ecx: &'cx mut ExtCtxt<'_>, + _span: Span, + annotation: TokenStream, + annotated: TokenStream) + -> TokenStream { + // FIXME setup implicit context in TLS before calling self. + (*self)(annotation, annotated) + } +} + +/// Represents a thing that maps token trees to Macro Results +pub trait TTMacroExpander { + fn expand<'cx>( + &self, + ecx: &'cx mut ExtCtxt<'_>, + span: Span, + input: TokenStream, + ) -> Box; +} + +pub type MacroExpanderFn = + for<'cx> fn(&'cx mut ExtCtxt<'_>, Span, TokenStream) + -> Box; + +impl TTMacroExpander for F + where F: for<'cx> Fn(&'cx mut ExtCtxt<'_>, Span, TokenStream) + -> Box +{ + fn expand<'cx>( + &self, + ecx: &'cx mut ExtCtxt<'_>, + span: Span, + mut input: TokenStream, + ) -> Box { + struct AvoidInterpolatedIdents; + + impl MutVisitor for AvoidInterpolatedIdents { + fn visit_tt(&mut self, tt: &mut tokenstream::TokenTree) { + if let tokenstream::TokenTree::Token(token) = tt { + if let token::Interpolated(nt) = &token.kind { + if let token::NtIdent(ident, is_raw) = **nt { + *tt = tokenstream::TokenTree::token( + token::Ident(ident.name, is_raw), ident.span + ); + } + } + } + mut_visit::noop_visit_tt(tt, self) + } + + fn visit_mac(&mut self, mac: &mut ast::Mac) { + mut_visit::noop_visit_mac(mac, self) + } + } + AvoidInterpolatedIdents.visit_tts(&mut input); + (*self)(ecx, span, input) + } +} + +// Use a macro because forwarding to a simple function has type system issues +macro_rules! make_stmts_default { + ($me:expr) => { + $me.make_expr().map(|e| smallvec![ast::Stmt { + id: ast::DUMMY_NODE_ID, + span: e.span, + kind: ast::StmtKind::Expr(e), + }]) + } +} + +/// The result of a macro expansion. The return values of the various +/// methods are spliced into the AST at the callsite of the macro. +pub trait MacResult { + /// Creates an expression. + fn make_expr(self: Box) -> Option> { + None + } + /// Creates zero or more items. + fn make_items(self: Box) -> Option; 1]>> { + None + } + + /// Creates zero or more impl items. + fn make_impl_items(self: Box) -> Option> { + None + } + + /// Creates zero or more trait items. + fn make_trait_items(self: Box) -> Option> { + None + } + + /// Creates zero or more items in an `extern {}` block + fn make_foreign_items(self: Box) -> Option> { None } + + /// Creates a pattern. + fn make_pat(self: Box) -> Option> { + None + } + + /// Creates zero or more statements. + /// + /// By default this attempts to create an expression statement, + /// returning None if that fails. + fn make_stmts(self: Box) -> Option> { + make_stmts_default!(self) + } + + fn make_ty(self: Box) -> Option> { + None + } + + fn make_arms(self: Box) -> Option> { + None + } + + fn make_fields(self: Box) -> Option> { + None + } + + fn make_field_patterns(self: Box) -> Option> { + None + } + + fn make_generic_params(self: Box) -> Option> { + None + } + + fn make_params(self: Box) -> Option> { + None + } + + fn make_struct_fields(self: Box) -> Option> { + None + } + + fn make_variants(self: Box) -> Option> { + None + } +} + +macro_rules! make_MacEager { + ( $( $fld:ident: $t:ty, )* ) => { + /// `MacResult` implementation for the common case where you've already + /// built each form of AST that you might return. + #[derive(Default)] + pub struct MacEager { + $( + pub $fld: Option<$t>, + )* + } + + impl MacEager { + $( + pub fn $fld(v: $t) -> Box { + Box::new(MacEager { + $fld: Some(v), + ..Default::default() + }) + } + )* + } + } +} + +make_MacEager! { + expr: P, + pat: P, + items: SmallVec<[P; 1]>, + impl_items: SmallVec<[ast::ImplItem; 1]>, + trait_items: SmallVec<[ast::TraitItem; 1]>, + foreign_items: SmallVec<[ast::ForeignItem; 1]>, + stmts: SmallVec<[ast::Stmt; 1]>, + ty: P, +} + +impl MacResult for MacEager { + fn make_expr(self: Box) -> Option> { + self.expr + } + + fn make_items(self: Box) -> Option; 1]>> { + self.items + } + + fn make_impl_items(self: Box) -> Option> { + self.impl_items + } + + fn make_trait_items(self: Box) -> Option> { + self.trait_items + } + + fn make_foreign_items(self: Box) -> Option> { + self.foreign_items + } + + fn make_stmts(self: Box) -> Option> { + match self.stmts.as_ref().map_or(0, |s| s.len()) { + 0 => make_stmts_default!(self), + _ => self.stmts, + } + } + + fn make_pat(self: Box) -> Option> { + if let Some(p) = self.pat { + return Some(p); + } + if let Some(e) = self.expr { + if let ast::ExprKind::Lit(_) = e.kind { + return Some(P(ast::Pat { + id: ast::DUMMY_NODE_ID, + span: e.span, + kind: PatKind::Lit(e), + })); + } + } + None + } + + fn make_ty(self: Box) -> Option> { + self.ty + } +} + +/// Fill-in macro expansion result, to allow compilation to continue +/// after hitting errors. +#[derive(Copy, Clone)] +pub struct DummyResult { + is_error: bool, + span: Span, +} + +impl DummyResult { + /// Creates a default MacResult that can be anything. + /// + /// Use this as a return value after hitting any errors and + /// calling `span_err`. + pub fn any(span: Span) -> Box { + Box::new(DummyResult { is_error: true, span }) + } + + /// Same as `any`, but must be a valid fragment, not error. + pub fn any_valid(span: Span) -> Box { + Box::new(DummyResult { is_error: false, span }) + } + + /// A plain dummy expression. + pub fn raw_expr(sp: Span, is_error: bool) -> P { + P(ast::Expr { + id: ast::DUMMY_NODE_ID, + kind: if is_error { ast::ExprKind::Err } else { ast::ExprKind::Tup(Vec::new()) }, + span: sp, + attrs: ThinVec::new(), + }) + } + + /// A plain dummy pattern. + pub fn raw_pat(sp: Span) -> ast::Pat { + ast::Pat { + id: ast::DUMMY_NODE_ID, + kind: PatKind::Wild, + span: sp, + } + } + + /// A plain dummy type. + pub fn raw_ty(sp: Span, is_error: bool) -> P { + P(ast::Ty { + id: ast::DUMMY_NODE_ID, + kind: if is_error { ast::TyKind::Err } else { ast::TyKind::Tup(Vec::new()) }, + span: sp + }) + } +} + +impl MacResult for DummyResult { + fn make_expr(self: Box) -> Option> { + Some(DummyResult::raw_expr(self.span, self.is_error)) + } + + fn make_pat(self: Box) -> Option> { + Some(P(DummyResult::raw_pat(self.span))) + } + + fn make_items(self: Box) -> Option; 1]>> { + Some(SmallVec::new()) + } + + fn make_impl_items(self: Box) -> Option> { + Some(SmallVec::new()) + } + + fn make_trait_items(self: Box) -> Option> { + Some(SmallVec::new()) + } + + fn make_foreign_items(self: Box) -> Option> { + Some(SmallVec::new()) + } + + fn make_stmts(self: Box) -> Option> { + Some(smallvec![ast::Stmt { + id: ast::DUMMY_NODE_ID, + kind: ast::StmtKind::Expr(DummyResult::raw_expr(self.span, self.is_error)), + span: self.span, + }]) + } + + fn make_ty(self: Box) -> Option> { + Some(DummyResult::raw_ty(self.span, self.is_error)) + } + + fn make_arms(self: Box) -> Option> { + Some(SmallVec::new()) + } + + fn make_fields(self: Box) -> Option> { + Some(SmallVec::new()) + } + + fn make_field_patterns(self: Box) -> Option> { + Some(SmallVec::new()) + } + + fn make_generic_params(self: Box) -> Option> { + Some(SmallVec::new()) + } + + fn make_params(self: Box) -> Option> { + Some(SmallVec::new()) + } + + fn make_struct_fields(self: Box) -> Option> { + Some(SmallVec::new()) + } + + fn make_variants(self: Box) -> Option> { + Some(SmallVec::new()) + } +} + +/// A syntax extension kind. +pub enum SyntaxExtensionKind { + /// A token-based function-like macro. + Bang( + /// An expander with signature TokenStream -> TokenStream. + Box, + ), + + /// An AST-based function-like macro. + LegacyBang( + /// An expander with signature TokenStream -> AST. + Box, + ), + + /// A token-based attribute macro. + Attr( + /// An expander with signature (TokenStream, TokenStream) -> TokenStream. + /// The first TokenSteam is the attribute itself, the second is the annotated item. + /// The produced TokenSteam replaces the input TokenSteam. + Box, + ), + + /// An AST-based attribute macro. + LegacyAttr( + /// An expander with signature (AST, AST) -> AST. + /// The first AST fragment is the attribute itself, the second is the annotated item. + /// The produced AST fragment replaces the input AST fragment. + Box, + ), + + /// A trivial attribute "macro" that does nothing, + /// only keeps the attribute and marks it as inert, + /// thus making it ineligible for further expansion. + NonMacroAttr { + /// Suppresses the `unused_attributes` lint for this attribute. + mark_used: bool, + }, + + /// A token-based derive macro. + Derive( + /// An expander with signature TokenStream -> TokenStream (not yet). + /// The produced TokenSteam is appended to the input TokenSteam. + Box, + ), + + /// An AST-based derive macro. + LegacyDerive( + /// An expander with signature AST -> AST. + /// The produced AST fragment is appended to the input AST fragment. + Box, + ), +} + +/// A struct representing a macro definition in "lowered" form ready for expansion. +pub struct SyntaxExtension { + /// A syntax extension kind. + pub kind: SyntaxExtensionKind, + /// Span of the macro definition. + pub span: Span, + /// Whitelist of unstable features that are treated as stable inside this macro. + pub allow_internal_unstable: Option>, + /// Suppresses the `unsafe_code` lint for code produced by this macro. + pub allow_internal_unsafe: bool, + /// Enables the macro helper hack (`ident!(...)` -> `$crate::ident!(...)`) for this macro. + pub local_inner_macros: bool, + /// The macro's stability info. + pub stability: Option, + /// The macro's deprecation info. + pub deprecation: Option, + /// Names of helper attributes registered by this macro. + pub helper_attrs: Vec, + /// Edition of the crate in which this macro is defined. + pub edition: Edition, + /// Built-in macros have a couple of special properties like availability + /// in `#[no_implicit_prelude]` modules, so we have to keep this flag. + pub is_builtin: bool, + /// We have to identify macros providing a `Copy` impl early for compatibility reasons. + pub is_derive_copy: bool, +} + +impl SyntaxExtension { + /// Returns which kind of macro calls this syntax extension. + pub fn macro_kind(&self) -> MacroKind { + match self.kind { + SyntaxExtensionKind::Bang(..) | + SyntaxExtensionKind::LegacyBang(..) => MacroKind::Bang, + SyntaxExtensionKind::Attr(..) | + SyntaxExtensionKind::LegacyAttr(..) | + SyntaxExtensionKind::NonMacroAttr { .. } => MacroKind::Attr, + SyntaxExtensionKind::Derive(..) | + SyntaxExtensionKind::LegacyDerive(..) => MacroKind::Derive, + } + } + + /// Constructs a syntax extension with default properties. + pub fn default(kind: SyntaxExtensionKind, edition: Edition) -> SyntaxExtension { + SyntaxExtension { + span: DUMMY_SP, + allow_internal_unstable: None, + allow_internal_unsafe: false, + local_inner_macros: false, + stability: None, + deprecation: None, + helper_attrs: Vec::new(), + edition, + is_builtin: false, + is_derive_copy: false, + kind, + } + } + + /// Constructs a syntax extension with the given properties + /// and other properties converted from attributes. + pub fn new( + sess: &ParseSess, + kind: SyntaxExtensionKind, + span: Span, + helper_attrs: Vec, + edition: Edition, + name: Name, + attrs: &[ast::Attribute], + ) -> SyntaxExtension { + let allow_internal_unstable = attr::allow_internal_unstable( + &attrs, &sess.span_diagnostic, + ).map(|features| features.collect::>().into()); + + let mut local_inner_macros = false; + if let Some(macro_export) = attr::find_by_name(attrs, sym::macro_export) { + if let Some(l) = macro_export.meta_item_list() { + local_inner_macros = attr::list_contains_name(&l, sym::local_inner_macros); + } + } + + let is_builtin = attr::contains_name(attrs, sym::rustc_builtin_macro); + + SyntaxExtension { + kind, + span, + allow_internal_unstable, + allow_internal_unsafe: attr::contains_name(attrs, sym::allow_internal_unsafe), + local_inner_macros, + stability: attr::find_stability(&sess, attrs, span), + deprecation: attr::find_deprecation(&sess, attrs, span), + helper_attrs, + edition, + is_builtin, + is_derive_copy: is_builtin && name == sym::Copy, + } + } + + pub fn dummy_bang(edition: Edition) -> SyntaxExtension { + fn expander<'cx>(_: &'cx mut ExtCtxt<'_>, span: Span, _: TokenStream) + -> Box { + DummyResult::any(span) + } + SyntaxExtension::default(SyntaxExtensionKind::LegacyBang(Box::new(expander)), edition) + } + + pub fn dummy_derive(edition: Edition) -> SyntaxExtension { + fn expander(_: &mut ExtCtxt<'_>, _: Span, _: &ast::MetaItem, _: Annotatable) + -> Vec { + Vec::new() + } + SyntaxExtension::default(SyntaxExtensionKind::Derive(Box::new(expander)), edition) + } + + pub fn non_macro_attr(mark_used: bool, edition: Edition) -> SyntaxExtension { + SyntaxExtension::default(SyntaxExtensionKind::NonMacroAttr { mark_used }, edition) + } + + pub fn expn_data(&self, parent: ExpnId, call_site: Span, descr: Symbol) -> ExpnData { + ExpnData { + kind: ExpnKind::Macro(self.macro_kind(), descr), + parent, + call_site, + def_site: self.span, + allow_internal_unstable: self.allow_internal_unstable.clone(), + allow_internal_unsafe: self.allow_internal_unsafe, + local_inner_macros: self.local_inner_macros, + edition: self.edition, + } + } +} + +pub type NamedSyntaxExtension = (Name, SyntaxExtension); + +/// Result of resolving a macro invocation. +pub enum InvocationRes { + Single(Lrc), + DeriveContainer(Vec>), +} + +/// Error type that denotes indeterminacy. +pub struct Indeterminate; + +bitflags::bitflags! { + /// Built-in derives that need some extra tracking beyond the usual macro functionality. + #[derive(Default)] + pub struct SpecialDerives: u8 { + const PARTIAL_EQ = 1 << 0; + const EQ = 1 << 1; + const COPY = 1 << 2; + } +} + +pub trait Resolver { + fn next_node_id(&mut self) -> NodeId; + + fn resolve_dollar_crates(&mut self); + fn visit_ast_fragment_with_placeholders(&mut self, expn_id: ExpnId, fragment: &AstFragment, + extra_placeholders: &[NodeId]); + fn register_builtin_macro(&mut self, ident: ast::Ident, ext: SyntaxExtension); + + fn expansion_for_ast_pass( + &mut self, + call_site: Span, + pass: AstPass, + features: &[Symbol], + parent_module_id: Option, + ) -> ExpnId; + + fn resolve_imports(&mut self); + + fn resolve_macro_invocation( + &mut self, invoc: &Invocation, eager_expansion_root: ExpnId, force: bool + ) -> Result; + + fn check_unused_macros(&self); + + fn has_derives(&self, expn_id: ExpnId, derives: SpecialDerives) -> bool; + fn add_derives(&mut self, expn_id: ExpnId, derives: SpecialDerives); +} + +#[derive(Clone)] +pub struct ModuleData { + pub mod_path: Vec, + pub directory: PathBuf, +} + +#[derive(Clone)] +pub struct ExpansionData { + pub id: ExpnId, + pub depth: usize, + pub module: Rc, + pub directory_ownership: DirectoryOwnership, + pub prior_type_ascription: Option<(Span, bool)>, +} + +/// One of these is made during expansion and incrementally updated as we go; +/// when a macro expansion occurs, the resulting nodes have the `backtrace() +/// -> expn_data` of their expansion context stored into their span. +pub struct ExtCtxt<'a> { + pub parse_sess: &'a ParseSess, + pub ecfg: expand::ExpansionConfig<'a>, + pub root_path: PathBuf, + pub resolver: &'a mut dyn Resolver, + pub current_expansion: ExpansionData, + pub expansions: FxHashMap>, +} + +impl<'a> ExtCtxt<'a> { + pub fn new(parse_sess: &'a ParseSess, + ecfg: expand::ExpansionConfig<'a>, + resolver: &'a mut dyn Resolver) + -> ExtCtxt<'a> { + ExtCtxt { + parse_sess, + ecfg, + root_path: PathBuf::new(), + resolver, + current_expansion: ExpansionData { + id: ExpnId::root(), + depth: 0, + module: Rc::new(ModuleData { mod_path: Vec::new(), directory: PathBuf::new() }), + directory_ownership: DirectoryOwnership::Owned { relative: None }, + prior_type_ascription: None, + }, + expansions: FxHashMap::default(), + } + } + + /// Returns a `Folder` for deeply expanding all macros in an AST node. + pub fn expander<'b>(&'b mut self) -> expand::MacroExpander<'b, 'a> { + expand::MacroExpander::new(self, false) + } + + /// Returns a `Folder` that deeply expands all macros and assigns all `NodeId`s in an AST node. + /// Once `NodeId`s are assigned, the node may not be expanded, removed, or otherwise modified. + pub fn monotonic_expander<'b>(&'b mut self) -> expand::MacroExpander<'b, 'a> { + expand::MacroExpander::new(self, true) + } + pub fn new_parser_from_tts(&self, stream: TokenStream) -> parser::Parser<'a> { + parse::stream_to_parser(self.parse_sess, stream, MACRO_ARGUMENTS) + } + pub fn source_map(&self) -> &'a SourceMap { self.parse_sess.source_map() } + pub fn parse_sess(&self) -> &'a ParseSess { self.parse_sess } + pub fn cfg(&self) -> &ast::CrateConfig { &self.parse_sess.config } + pub fn call_site(&self) -> Span { + self.current_expansion.id.expn_data().call_site + } + + /// Equivalent of `Span::def_site` from the proc macro API, + /// except that the location is taken from the span passed as an argument. + pub fn with_def_site_ctxt(&self, span: Span) -> Span { + span.with_def_site_ctxt(self.current_expansion.id) + } + + /// Equivalent of `Span::call_site` from the proc macro API, + /// except that the location is taken from the span passed as an argument. + pub fn with_call_site_ctxt(&self, span: Span) -> Span { + span.with_call_site_ctxt(self.current_expansion.id) + } + + /// Equivalent of `Span::mixed_site` from the proc macro API, + /// except that the location is taken from the span passed as an argument. + pub fn with_mixed_site_ctxt(&self, span: Span) -> Span { + span.with_mixed_site_ctxt(self.current_expansion.id) + } + + /// Returns span for the macro which originally caused the current expansion to happen. + /// + /// Stops backtracing at include! boundary. + pub fn expansion_cause(&self) -> Option { + let mut expn_id = self.current_expansion.id; + let mut last_macro = None; + loop { + let expn_data = expn_id.expn_data(); + // Stop going up the backtrace once include! is encountered + if expn_data.is_root() || expn_data.kind.descr() == sym::include { + break; + } + expn_id = expn_data.call_site.ctxt().outer_expn(); + last_macro = Some(expn_data.call_site); + } + last_macro + } + + pub fn struct_span_warn>(&self, + sp: S, + msg: &str) + -> DiagnosticBuilder<'a> { + self.parse_sess.span_diagnostic.struct_span_warn(sp, msg) + } + pub fn struct_span_err>(&self, + sp: S, + msg: &str) + -> DiagnosticBuilder<'a> { + self.parse_sess.span_diagnostic.struct_span_err(sp, msg) + } + pub fn struct_span_fatal>(&self, + sp: S, + msg: &str) + -> DiagnosticBuilder<'a> { + self.parse_sess.span_diagnostic.struct_span_fatal(sp, msg) + } + + /// Emit `msg` attached to `sp`, and stop compilation immediately. + /// + /// `span_err` should be strongly preferred where-ever possible: + /// this should *only* be used when: + /// + /// - continuing has a high risk of flow-on errors (e.g., errors in + /// declaring a macro would cause all uses of that macro to + /// complain about "undefined macro"), or + /// - there is literally nothing else that can be done (however, + /// in most cases one can construct a dummy expression/item to + /// substitute; we never hit resolve/type-checking so the dummy + /// value doesn't have to match anything) + pub fn span_fatal>(&self, sp: S, msg: &str) -> ! { + self.parse_sess.span_diagnostic.span_fatal(sp, msg).raise(); + } + + /// Emit `msg` attached to `sp`, without immediately stopping + /// compilation. + /// + /// Compilation will be stopped in the near future (at the end of + /// the macro expansion phase). + pub fn span_err>(&self, sp: S, msg: &str) { + self.parse_sess.span_diagnostic.span_err(sp, msg); + } + pub fn span_err_with_code>(&self, sp: S, msg: &str, code: DiagnosticId) { + self.parse_sess.span_diagnostic.span_err_with_code(sp, msg, code); + } + pub fn span_warn>(&self, sp: S, msg: &str) { + self.parse_sess.span_diagnostic.span_warn(sp, msg); + } + pub fn span_bug>(&self, sp: S, msg: &str) -> ! { + self.parse_sess.span_diagnostic.span_bug(sp, msg); + } + pub fn trace_macros_diag(&mut self) { + for (sp, notes) in self.expansions.iter() { + let mut db = self.parse_sess.span_diagnostic.span_note_diag(*sp, "trace_macro"); + for note in notes { + db.note(note); + } + db.emit(); + } + // Fixme: does this result in errors? + self.expansions.clear(); + } + pub fn bug(&self, msg: &str) -> ! { + self.parse_sess.span_diagnostic.bug(msg); + } + pub fn trace_macros(&self) -> bool { + self.ecfg.trace_mac + } + pub fn set_trace_macros(&mut self, x: bool) { + self.ecfg.trace_mac = x + } + pub fn ident_of(&self, st: &str, sp: Span) -> ast::Ident { + ast::Ident::from_str_and_span(st, sp) + } + pub fn std_path(&self, components: &[Symbol]) -> Vec { + let def_site = self.with_def_site_ctxt(DUMMY_SP); + iter::once(Ident::new(kw::DollarCrate, def_site)) + .chain(components.iter().map(|&s| Ident::with_dummy_span(s))) + .collect() + } + pub fn name_of(&self, st: &str) -> ast::Name { + Symbol::intern(st) + } + + pub fn check_unused_macros(&self) { + self.resolver.check_unused_macros(); + } + + /// Resolves a path mentioned inside Rust code. + /// + /// This unifies the logic used for resolving `include_X!`, and `#[doc(include)]` file paths. + /// + /// Returns an absolute path to the file that `path` refers to. + pub fn resolve_path(&self, path: impl Into, span: Span) -> PathBuf { + let path = path.into(); + + // Relative paths are resolved relative to the file in which they are found + // after macro expansion (that is, they are unhygienic). + if !path.is_absolute() { + let callsite = span.source_callsite(); + let mut result = match self.source_map().span_to_unmapped_path(callsite) { + FileName::Real(path) => path, + FileName::DocTest(path, _) => path, + other => panic!("cannot resolve relative path in non-file source `{}`", other), + }; + result.pop(); + result.push(path); + result + } else { + path + } + } +} + +/// Extracts a string literal from the macro expanded version of `expr`, +/// emitting `err_msg` if `expr` is not a string literal. This does not stop +/// compilation on error, merely emits a non-fatal error and returns `None`. +pub fn expr_to_spanned_string<'a>( + cx: &'a mut ExtCtxt<'_>, + expr: P, + err_msg: &str, +) -> Result<(Symbol, ast::StrStyle, Span), Option>> { + // Perform eager expansion on the expression. + // We want to be able to handle e.g., `concat!("foo", "bar")`. + let expr = cx.expander().fully_expand_fragment(AstFragment::Expr(expr)).make_expr(); + + Err(match expr.kind { + ast::ExprKind::Lit(ref l) => match l.kind { + ast::LitKind::Str(s, style) => return Ok((s, style, expr.span)), + ast::LitKind::Err(_) => None, + _ => Some(cx.struct_span_err(l.span, err_msg)) + }, + ast::ExprKind::Err => None, + _ => Some(cx.struct_span_err(expr.span, err_msg)) + }) +} + +pub fn expr_to_string(cx: &mut ExtCtxt<'_>, expr: P, err_msg: &str) + -> Option<(Symbol, ast::StrStyle)> { + expr_to_spanned_string(cx, expr, err_msg) + .map_err(|err| err.map(|mut err| err.emit())) + .ok() + .map(|(symbol, style, _)| (symbol, style)) +} + +/// Non-fatally assert that `tts` is empty. Note that this function +/// returns even when `tts` is non-empty, macros that *need* to stop +/// compilation should call +/// `cx.parse_sess.span_diagnostic.abort_if_errors()` (this should be +/// done as rarely as possible). +pub fn check_zero_tts(cx: &ExtCtxt<'_>, + sp: Span, + tts: TokenStream, + name: &str) { + if !tts.is_empty() { + cx.span_err(sp, &format!("{} takes no arguments", name)); + } +} + +/// Interpreting `tts` as a comma-separated sequence of expressions, +/// expect exactly one string literal, or emit an error and return `None`. +pub fn get_single_str_from_tts(cx: &mut ExtCtxt<'_>, + sp: Span, + tts: TokenStream, + name: &str) + -> Option { + let mut p = cx.new_parser_from_tts(tts); + if p.token == token::Eof { + cx.span_err(sp, &format!("{} takes 1 argument", name)); + return None + } + let ret = panictry!(p.parse_expr()); + let _ = p.eat(&token::Comma); + + if p.token != token::Eof { + cx.span_err(sp, &format!("{} takes 1 argument", name)); + } + expr_to_string(cx, ret, "argument must be a string literal").map(|(s, _)| { + s.to_string() + }) +} + +/// Extracts comma-separated expressions from `tts`. If there is a +/// parsing error, emit a non-fatal error and return `None`. +pub fn get_exprs_from_tts(cx: &mut ExtCtxt<'_>, + sp: Span, + tts: TokenStream) -> Option>> { + let mut p = cx.new_parser_from_tts(tts); + let mut es = Vec::new(); + while p.token != token::Eof { + let expr = panictry!(p.parse_expr()); + + // Perform eager expansion on the expression. + // We want to be able to handle e.g., `concat!("foo", "bar")`. + let expr = cx.expander().fully_expand_fragment(AstFragment::Expr(expr)).make_expr(); + + es.push(expr); + if p.eat(&token::Comma) { + continue; + } + if p.token != token::Eof { + cx.span_err(sp, "expected token: `,`"); + return None; + } + } + Some(es) +} diff --git a/src/libsyntax_expand/build.rs b/src/libsyntax_expand/build.rs new file mode 100644 index 00000000000..105ffe3ee8a --- /dev/null +++ b/src/libsyntax_expand/build.rs @@ -0,0 +1,640 @@ +use crate::base::ExtCtxt; + +use syntax::ast::{self, Ident, Expr, BlockCheckMode, UnOp, PatKind}; +use syntax::attr; +use syntax::source_map::{respan, Spanned}; +use syntax::ptr::P; +use syntax::symbol::{kw, sym, Symbol}; +use syntax::ThinVec; + +use syntax_pos::{Pos, Span}; + +impl<'a> ExtCtxt<'a> { + pub fn path(&self, span: Span, strs: Vec ) -> ast::Path { + self.path_all(span, false, strs, vec![]) + } + pub fn path_ident(&self, span: Span, id: ast::Ident) -> ast::Path { + self.path(span, vec![id]) + } + pub fn path_global(&self, span: Span, strs: Vec ) -> ast::Path { + self.path_all(span, true, strs, vec![]) + } + pub fn path_all(&self, + span: Span, + global: bool, + mut idents: Vec , + args: Vec) + -> ast::Path { + assert!(!idents.is_empty()); + let add_root = global && !idents[0].is_path_segment_keyword(); + let mut segments = Vec::with_capacity(idents.len() + add_root as usize); + if add_root { + segments.push(ast::PathSegment::path_root(span)); + } + let last_ident = idents.pop().unwrap(); + segments.extend(idents.into_iter().map(|ident| { + ast::PathSegment::from_ident(ident.with_span_pos(span)) + })); + let args = if !args.is_empty() { + ast::AngleBracketedArgs { args, constraints: Vec::new(), span }.into() + } else { + None + }; + segments.push(ast::PathSegment { + ident: last_ident.with_span_pos(span), + id: ast::DUMMY_NODE_ID, + args, + }); + ast::Path { span, segments } + } + + pub fn ty_mt(&self, ty: P, mutbl: ast::Mutability) -> ast::MutTy { + ast::MutTy { + ty, + mutbl, + } + } + + pub fn ty(&self, span: Span, kind: ast::TyKind) -> P { + P(ast::Ty { + id: ast::DUMMY_NODE_ID, + span, + kind, + }) + } + + pub fn ty_path(&self, path: ast::Path) -> P { + self.ty(path.span, ast::TyKind::Path(None, path)) + } + + // Might need to take bounds as an argument in the future, if you ever want + // to generate a bounded existential trait type. + pub fn ty_ident(&self, span: Span, ident: ast::Ident) + -> P { + self.ty_path(self.path_ident(span, ident)) + } + + pub fn anon_const(&self, span: Span, kind: ast::ExprKind) -> ast::AnonConst { + ast::AnonConst { + id: ast::DUMMY_NODE_ID, + value: P(ast::Expr { + id: ast::DUMMY_NODE_ID, + kind, + span, + attrs: ThinVec::new(), + }) + } + } + + pub fn const_ident(&self, span: Span, ident: ast::Ident) -> ast::AnonConst { + self.anon_const(span, ast::ExprKind::Path(None, self.path_ident(span, ident))) + } + + pub fn ty_rptr(&self, + span: Span, + ty: P, + lifetime: Option, + mutbl: ast::Mutability) + -> P { + self.ty(span, + ast::TyKind::Rptr(lifetime, self.ty_mt(ty, mutbl))) + } + + pub fn ty_ptr(&self, + span: Span, + ty: P, + mutbl: ast::Mutability) + -> P { + self.ty(span, + ast::TyKind::Ptr(self.ty_mt(ty, mutbl))) + } + + pub fn typaram(&self, + span: Span, + ident: ast::Ident, + attrs: Vec, + bounds: ast::GenericBounds, + default: Option>) -> ast::GenericParam { + ast::GenericParam { + ident: ident.with_span_pos(span), + id: ast::DUMMY_NODE_ID, + attrs: attrs.into(), + bounds, + kind: ast::GenericParamKind::Type { + default, + }, + is_placeholder: false + } + } + + pub fn trait_ref(&self, path: ast::Path) -> ast::TraitRef { + ast::TraitRef { + path, + ref_id: ast::DUMMY_NODE_ID, + } + } + + pub fn poly_trait_ref(&self, span: Span, path: ast::Path) -> ast::PolyTraitRef { + ast::PolyTraitRef { + bound_generic_params: Vec::new(), + trait_ref: self.trait_ref(path), + span, + } + } + + pub fn trait_bound(&self, path: ast::Path) -> ast::GenericBound { + ast::GenericBound::Trait(self.poly_trait_ref(path.span, path), + ast::TraitBoundModifier::None) + } + + pub fn lifetime(&self, span: Span, ident: ast::Ident) -> ast::Lifetime { + ast::Lifetime { id: ast::DUMMY_NODE_ID, ident: ident.with_span_pos(span) } + } + + pub fn lifetime_def(&self, + span: Span, + ident: ast::Ident, + attrs: Vec, + bounds: ast::GenericBounds) + -> ast::GenericParam { + let lifetime = self.lifetime(span, ident); + ast::GenericParam { + ident: lifetime.ident, + id: lifetime.id, + attrs: attrs.into(), + bounds, + kind: ast::GenericParamKind::Lifetime, + is_placeholder: false + } + } + + pub fn stmt_expr(&self, expr: P) -> ast::Stmt { + ast::Stmt { + id: ast::DUMMY_NODE_ID, + span: expr.span, + kind: ast::StmtKind::Expr(expr), + } + } + + pub fn stmt_let(&self, sp: Span, mutbl: bool, ident: ast::Ident, + ex: P) -> ast::Stmt { + let pat = if mutbl { + let binding_mode = ast::BindingMode::ByValue(ast::Mutability::Mutable); + self.pat_ident_binding_mode(sp, ident, binding_mode) + } else { + self.pat_ident(sp, ident) + }; + let local = P(ast::Local { + pat, + ty: None, + init: Some(ex), + id: ast::DUMMY_NODE_ID, + span: sp, + attrs: ThinVec::new(), + }); + ast::Stmt { + id: ast::DUMMY_NODE_ID, + kind: ast::StmtKind::Local(local), + span: sp, + } + } + + // Generates `let _: Type;`, which is usually used for type assertions. + pub fn stmt_let_type_only(&self, span: Span, ty: P) -> ast::Stmt { + let local = P(ast::Local { + pat: self.pat_wild(span), + ty: Some(ty), + init: None, + id: ast::DUMMY_NODE_ID, + span, + attrs: ThinVec::new(), + }); + ast::Stmt { + id: ast::DUMMY_NODE_ID, + kind: ast::StmtKind::Local(local), + span, + } + } + + pub fn stmt_item(&self, sp: Span, item: P) -> ast::Stmt { + ast::Stmt { + id: ast::DUMMY_NODE_ID, + kind: ast::StmtKind::Item(item), + span: sp, + } + } + + pub fn block_expr(&self, expr: P) -> P { + self.block(expr.span, vec![ast::Stmt { + id: ast::DUMMY_NODE_ID, + span: expr.span, + kind: ast::StmtKind::Expr(expr), + }]) + } + pub fn block(&self, span: Span, stmts: Vec) -> P { + P(ast::Block { + stmts, + id: ast::DUMMY_NODE_ID, + rules: BlockCheckMode::Default, + span, + }) + } + + pub fn expr(&self, span: Span, kind: ast::ExprKind) -> P { + P(ast::Expr { + id: ast::DUMMY_NODE_ID, + kind, + span, + attrs: ThinVec::new(), + }) + } + + pub fn expr_path(&self, path: ast::Path) -> P { + self.expr(path.span, ast::ExprKind::Path(None, path)) + } + + pub fn expr_ident(&self, span: Span, id: ast::Ident) -> P { + self.expr_path(self.path_ident(span, id)) + } + pub fn expr_self(&self, span: Span) -> P { + self.expr_ident(span, Ident::with_dummy_span(kw::SelfLower)) + } + + pub fn expr_binary(&self, sp: Span, op: ast::BinOpKind, + lhs: P, rhs: P) -> P { + self.expr(sp, ast::ExprKind::Binary(Spanned { node: op, span: sp }, lhs, rhs)) + } + + pub fn expr_deref(&self, sp: Span, e: P) -> P { + self.expr(sp, ast::ExprKind::Unary(UnOp::Deref, e)) + } + + pub fn expr_addr_of(&self, sp: Span, e: P) -> P { + self.expr(sp, ast::ExprKind::AddrOf(ast::Mutability::Immutable, e)) + } + + pub fn expr_call( + &self, span: Span, expr: P, args: Vec>, + ) -> P { + self.expr(span, ast::ExprKind::Call(expr, args)) + } + pub fn expr_call_ident(&self, span: Span, id: ast::Ident, + args: Vec>) -> P { + self.expr(span, ast::ExprKind::Call(self.expr_ident(span, id), args)) + } + pub fn expr_call_global(&self, sp: Span, fn_path: Vec , + args: Vec> ) -> P { + let pathexpr = self.expr_path(self.path_global(sp, fn_path)); + self.expr_call(sp, pathexpr, args) + } + pub fn expr_method_call(&self, span: Span, + expr: P, + ident: ast::Ident, + mut args: Vec> ) -> P { + args.insert(0, expr); + let segment = ast::PathSegment::from_ident(ident.with_span_pos(span)); + self.expr(span, ast::ExprKind::MethodCall(segment, args)) + } + pub fn expr_block(&self, b: P) -> P { + self.expr(b.span, ast::ExprKind::Block(b, None)) + } + pub fn field_imm(&self, span: Span, ident: Ident, e: P) -> ast::Field { + ast::Field { + ident: ident.with_span_pos(span), + expr: e, + span, + is_shorthand: false, + attrs: ThinVec::new(), + id: ast::DUMMY_NODE_ID, + is_placeholder: false, + } + } + pub fn expr_struct( + &self, span: Span, path: ast::Path, fields: Vec + ) -> P { + self.expr(span, ast::ExprKind::Struct(path, fields, None)) + } + pub fn expr_struct_ident(&self, span: Span, + id: ast::Ident, fields: Vec) -> P { + self.expr_struct(span, self.path_ident(span, id), fields) + } + + pub fn expr_lit(&self, span: Span, lit_kind: ast::LitKind) -> P { + let lit = ast::Lit::from_lit_kind(lit_kind, span); + self.expr(span, ast::ExprKind::Lit(lit)) + } + pub fn expr_usize(&self, span: Span, i: usize) -> P { + self.expr_lit(span, ast::LitKind::Int(i as u128, + ast::LitIntType::Unsigned(ast::UintTy::Usize))) + } + pub fn expr_u32(&self, sp: Span, u: u32) -> P { + self.expr_lit(sp, ast::LitKind::Int(u as u128, + ast::LitIntType::Unsigned(ast::UintTy::U32))) + } + pub fn expr_bool(&self, sp: Span, value: bool) -> P { + self.expr_lit(sp, ast::LitKind::Bool(value)) + } + + pub fn expr_vec(&self, sp: Span, exprs: Vec>) -> P { + self.expr(sp, ast::ExprKind::Array(exprs)) + } + pub fn expr_vec_slice(&self, sp: Span, exprs: Vec>) -> P { + self.expr_addr_of(sp, self.expr_vec(sp, exprs)) + } + pub fn expr_str(&self, sp: Span, s: Symbol) -> P { + self.expr_lit(sp, ast::LitKind::Str(s, ast::StrStyle::Cooked)) + } + + pub fn expr_cast(&self, sp: Span, expr: P, ty: P) -> P { + self.expr(sp, ast::ExprKind::Cast(expr, ty)) + } + + pub fn expr_some(&self, sp: Span, expr: P) -> P { + let some = self.std_path(&[sym::option, sym::Option, sym::Some]); + self.expr_call_global(sp, some, vec![expr]) + } + + pub fn expr_tuple(&self, sp: Span, exprs: Vec>) -> P { + self.expr(sp, ast::ExprKind::Tup(exprs)) + } + + pub fn expr_fail(&self, span: Span, msg: Symbol) -> P { + let loc = self.source_map().lookup_char_pos(span.lo()); + let expr_file = self.expr_str(span, Symbol::intern(&loc.file.name.to_string())); + let expr_line = self.expr_u32(span, loc.line as u32); + let expr_col = self.expr_u32(span, loc.col.to_usize() as u32 + 1); + let expr_loc_tuple = self.expr_tuple(span, vec![expr_file, expr_line, expr_col]); + let expr_loc_ptr = self.expr_addr_of(span, expr_loc_tuple); + self.expr_call_global( + span, + [sym::std, sym::rt, sym::begin_panic].iter().map(|s| Ident::new(*s, span)).collect(), + vec![ + self.expr_str(span, msg), + expr_loc_ptr]) + } + + pub fn expr_unreachable(&self, span: Span) -> P { + self.expr_fail(span, Symbol::intern("internal error: entered unreachable code")) + } + + pub fn expr_ok(&self, sp: Span, expr: P) -> P { + let ok = self.std_path(&[sym::result, sym::Result, sym::Ok]); + self.expr_call_global(sp, ok, vec![expr]) + } + + pub fn expr_try(&self, sp: Span, head: P) -> P { + let ok = self.std_path(&[sym::result, sym::Result, sym::Ok]); + let ok_path = self.path_global(sp, ok); + let err = self.std_path(&[sym::result, sym::Result, sym::Err]); + let err_path = self.path_global(sp, err); + + let binding_variable = self.ident_of("__try_var", sp); + let binding_pat = self.pat_ident(sp, binding_variable); + let binding_expr = self.expr_ident(sp, binding_variable); + + // `Ok(__try_var)` pattern + let ok_pat = self.pat_tuple_struct(sp, ok_path, vec![binding_pat.clone()]); + + // `Err(__try_var)` (pattern and expression respectively) + let err_pat = self.pat_tuple_struct(sp, err_path.clone(), vec![binding_pat]); + let err_inner_expr = self.expr_call(sp, self.expr_path(err_path), + vec![binding_expr.clone()]); + // `return Err(__try_var)` + let err_expr = self.expr(sp, ast::ExprKind::Ret(Some(err_inner_expr))); + + // `Ok(__try_var) => __try_var` + let ok_arm = self.arm(sp, ok_pat, binding_expr); + // `Err(__try_var) => return Err(__try_var)` + let err_arm = self.arm(sp, err_pat, err_expr); + + // `match head { Ok() => ..., Err() => ... }` + self.expr_match(sp, head, vec![ok_arm, err_arm]) + } + + + pub fn pat(&self, span: Span, kind: PatKind) -> P { + P(ast::Pat { id: ast::DUMMY_NODE_ID, kind, span }) + } + pub fn pat_wild(&self, span: Span) -> P { + self.pat(span, PatKind::Wild) + } + pub fn pat_lit(&self, span: Span, expr: P) -> P { + self.pat(span, PatKind::Lit(expr)) + } + pub fn pat_ident(&self, span: Span, ident: ast::Ident) -> P { + let binding_mode = ast::BindingMode::ByValue(ast::Mutability::Immutable); + self.pat_ident_binding_mode(span, ident, binding_mode) + } + + pub fn pat_ident_binding_mode(&self, + span: Span, + ident: ast::Ident, + bm: ast::BindingMode) -> P { + let pat = PatKind::Ident(bm, ident.with_span_pos(span), None); + self.pat(span, pat) + } + pub fn pat_path(&self, span: Span, path: ast::Path) -> P { + self.pat(span, PatKind::Path(None, path)) + } + pub fn pat_tuple_struct(&self, span: Span, path: ast::Path, + subpats: Vec>) -> P { + self.pat(span, PatKind::TupleStruct(path, subpats)) + } + pub fn pat_struct(&self, span: Span, path: ast::Path, + field_pats: Vec) -> P { + self.pat(span, PatKind::Struct(path, field_pats, false)) + } + pub fn pat_tuple(&self, span: Span, pats: Vec>) -> P { + self.pat(span, PatKind::Tuple(pats)) + } + + pub fn pat_some(&self, span: Span, pat: P) -> P { + let some = self.std_path(&[sym::option, sym::Option, sym::Some]); + let path = self.path_global(span, some); + self.pat_tuple_struct(span, path, vec![pat]) + } + + pub fn pat_none(&self, span: Span) -> P { + let some = self.std_path(&[sym::option, sym::Option, sym::None]); + let path = self.path_global(span, some); + self.pat_path(span, path) + } + + pub fn pat_ok(&self, span: Span, pat: P) -> P { + let some = self.std_path(&[sym::result, sym::Result, sym::Ok]); + let path = self.path_global(span, some); + self.pat_tuple_struct(span, path, vec![pat]) + } + + pub fn pat_err(&self, span: Span, pat: P) -> P { + let some = self.std_path(&[sym::result, sym::Result, sym::Err]); + let path = self.path_global(span, some); + self.pat_tuple_struct(span, path, vec![pat]) + } + + pub fn arm(&self, span: Span, pat: P, expr: P) -> ast::Arm { + ast::Arm { + attrs: vec![], + pat, + guard: None, + body: expr, + span, + id: ast::DUMMY_NODE_ID, + is_placeholder: false, + } + } + + pub fn arm_unreachable(&self, span: Span) -> ast::Arm { + self.arm(span, self.pat_wild(span), self.expr_unreachable(span)) + } + + pub fn expr_match(&self, span: Span, arg: P, arms: Vec) -> P { + self.expr(span, ast::ExprKind::Match(arg, arms)) + } + + pub fn expr_if(&self, span: Span, cond: P, + then: P, els: Option>) -> P { + let els = els.map(|x| self.expr_block(self.block_expr(x))); + self.expr(span, ast::ExprKind::If(cond, self.block_expr(then), els)) + } + + pub fn lambda_fn_decl(&self, + span: Span, + fn_decl: P, + body: P, + fn_decl_span: Span) // span of the `|...|` part + -> P { + self.expr(span, ast::ExprKind::Closure(ast::CaptureBy::Ref, + ast::IsAsync::NotAsync, + ast::Movability::Movable, + fn_decl, + body, + fn_decl_span)) + } + + pub fn lambda(&self, + span: Span, + ids: Vec, + body: P) + -> P { + let fn_decl = self.fn_decl( + ids.iter().map(|id| self.param(span, *id, self.ty(span, ast::TyKind::Infer))).collect(), + ast::FunctionRetTy::Default(span)); + + // FIXME -- We are using `span` as the span of the `|...|` + // part of the lambda, but it probably (maybe?) corresponds to + // the entire lambda body. Probably we should extend the API + // here, but that's not entirely clear. + self.expr(span, ast::ExprKind::Closure(ast::CaptureBy::Ref, + ast::IsAsync::NotAsync, + ast::Movability::Movable, + fn_decl, + body, + span)) + } + + pub fn lambda0(&self, span: Span, body: P) -> P { + self.lambda(span, Vec::new(), body) + } + + pub fn lambda1(&self, span: Span, body: P, ident: ast::Ident) -> P { + self.lambda(span, vec![ident], body) + } + + pub fn lambda_stmts_1(&self, span: Span, stmts: Vec, + ident: ast::Ident) -> P { + self.lambda1(span, self.expr_block(self.block(span, stmts)), ident) + } + + pub fn param(&self, span: Span, ident: ast::Ident, ty: P) -> ast::Param { + let arg_pat = self.pat_ident(span, ident); + ast::Param { + attrs: ThinVec::default(), + id: ast::DUMMY_NODE_ID, + pat: arg_pat, + span, + ty, + is_placeholder: false, + } + } + + // FIXME: unused `self` + pub fn fn_decl(&self, inputs: Vec, output: ast::FunctionRetTy) -> P { + P(ast::FnDecl { + inputs, + output, + }) + } + + pub fn item(&self, span: Span, name: Ident, + attrs: Vec, kind: ast::ItemKind) -> P { + // FIXME: Would be nice if our generated code didn't violate + // Rust coding conventions + P(ast::Item { + ident: name, + attrs, + id: ast::DUMMY_NODE_ID, + kind, + vis: respan(span.shrink_to_lo(), ast::VisibilityKind::Inherited), + span, + tokens: None, + }) + } + + pub fn variant(&self, span: Span, ident: Ident, tys: Vec> ) -> ast::Variant { + let fields: Vec<_> = tys.into_iter().map(|ty| { + ast::StructField { + span: ty.span, + ty, + ident: None, + vis: respan(span.shrink_to_lo(), ast::VisibilityKind::Inherited), + attrs: Vec::new(), + id: ast::DUMMY_NODE_ID, + is_placeholder: false, + } + }).collect(); + + let vdata = if fields.is_empty() { + ast::VariantData::Unit(ast::DUMMY_NODE_ID) + } else { + ast::VariantData::Tuple(fields, ast::DUMMY_NODE_ID) + }; + + ast::Variant { + attrs: Vec::new(), + data: vdata, + disr_expr: None, + id: ast::DUMMY_NODE_ID, + ident, + span, + is_placeholder: false, + } + } + + pub fn item_static(&self, + span: Span, + name: Ident, + ty: P, + mutbl: ast::Mutability, + expr: P) + -> P { + self.item(span, name, Vec::new(), ast::ItemKind::Static(ty, mutbl, expr)) + } + + pub fn item_const(&self, + span: Span, + name: Ident, + ty: P, + expr: P) + -> P { + self.item(span, name, Vec::new(), ast::ItemKind::Const(ty, expr)) + } + + pub fn attribute(&self, mi: ast::MetaItem) -> ast::Attribute { + attr::mk_attr_outer(mi) + } + + pub fn meta_word(&self, sp: Span, w: ast::Name) -> ast::MetaItem { + attr::mk_word_item(Ident::new(w, sp)) + } +} diff --git a/src/libsyntax_expand/expand.rs b/src/libsyntax_expand/expand.rs new file mode 100644 index 00000000000..47b4bca314a --- /dev/null +++ b/src/libsyntax_expand/expand.rs @@ -0,0 +1,1551 @@ +use crate::base::*; +use crate::proc_macro::{collect_derives, MarkAttrs}; +use crate::hygiene::{ExpnId, SyntaxContext, ExpnData, ExpnKind}; +use crate::mbe::macro_rules::annotate_err_with_kind; +use crate::placeholders::{placeholder, PlaceholderExpander}; + +use syntax::ast::{self, AttrItem, Block, Ident, LitKind, NodeId, PatKind, Path}; +use syntax::ast::{MacStmtStyle, StmtKind, ItemKind}; +use syntax::attr::{self, HasAttrs}; +use syntax::source_map::respan; +use syntax::configure; +use syntax::config::StripUnconfigured; +use syntax::feature_gate::{self, Features, GateIssue, is_builtin_attr, emit_feature_err}; +use syntax::mut_visit::*; +use syntax::parse::{DirectoryOwnership, PResult}; +use syntax::parse::token; +use syntax::parse::parser::Parser; +use syntax::print::pprust; +use syntax::ptr::P; +use syntax::symbol::{sym, Symbol}; +use syntax::tokenstream::{TokenStream, TokenTree}; +use syntax::visit::Visitor; +use syntax::util::map_in_place::MapInPlace; + +use errors::{Applicability, FatalError}; +use smallvec::{smallvec, SmallVec}; +use syntax_pos::{Span, DUMMY_SP, FileName}; + +use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::sync::Lrc; +use std::io::ErrorKind; +use std::{iter, mem, slice}; +use std::ops::DerefMut; +use std::rc::Rc; +use std::path::PathBuf; + +macro_rules! ast_fragments { + ( + $($Kind:ident($AstTy:ty) { + $kind_name:expr; + $(one fn $mut_visit_ast:ident; fn $visit_ast:ident;)? + $(many fn $flat_map_ast_elt:ident; fn $visit_ast_elt:ident;)? + fn $make_ast:ident; + })* + ) => { + /// A fragment of AST that can be produced by a single macro expansion. + /// Can also serve as an input and intermediate result for macro expansion operations. + pub enum AstFragment { + OptExpr(Option>), + $($Kind($AstTy),)* + } + + /// "Discriminant" of an AST fragment. + #[derive(Copy, Clone, PartialEq, Eq)] + pub enum AstFragmentKind { + OptExpr, + $($Kind,)* + } + + impl AstFragmentKind { + pub fn name(self) -> &'static str { + match self { + AstFragmentKind::OptExpr => "expression", + $(AstFragmentKind::$Kind => $kind_name,)* + } + } + + fn make_from<'a>(self, result: Box) -> Option { + match self { + AstFragmentKind::OptExpr => + result.make_expr().map(Some).map(AstFragment::OptExpr), + $(AstFragmentKind::$Kind => result.$make_ast().map(AstFragment::$Kind),)* + } + } + } + + impl AstFragment { + pub fn make_opt_expr(self) -> Option> { + match self { + AstFragment::OptExpr(expr) => expr, + _ => panic!("AstFragment::make_* called on the wrong kind of fragment"), + } + } + + $(pub fn $make_ast(self) -> $AstTy { + match self { + AstFragment::$Kind(ast) => ast, + _ => panic!("AstFragment::make_* called on the wrong kind of fragment"), + } + })* + + pub fn mut_visit_with(&mut self, vis: &mut F) { + match self { + AstFragment::OptExpr(opt_expr) => { + visit_clobber(opt_expr, |opt_expr| { + if let Some(expr) = opt_expr { + vis.filter_map_expr(expr) + } else { + None + } + }); + } + $($(AstFragment::$Kind(ast) => vis.$mut_visit_ast(ast),)?)* + $($(AstFragment::$Kind(ast) => + ast.flat_map_in_place(|ast| vis.$flat_map_ast_elt(ast)),)?)* + } + } + + pub fn visit_with<'a, V: Visitor<'a>>(&'a self, visitor: &mut V) { + match *self { + AstFragment::OptExpr(Some(ref expr)) => visitor.visit_expr(expr), + AstFragment::OptExpr(None) => {} + $($(AstFragment::$Kind(ref ast) => visitor.$visit_ast(ast),)?)* + $($(AstFragment::$Kind(ref ast) => for ast_elt in &ast[..] { + visitor.$visit_ast_elt(ast_elt); + })?)* + } + } + } + + impl<'a> MacResult for crate::mbe::macro_rules::ParserAnyMacro<'a> { + $(fn $make_ast(self: Box>) + -> Option<$AstTy> { + Some(self.make(AstFragmentKind::$Kind).$make_ast()) + })* + } + } +} + +ast_fragments! { + Expr(P) { "expression"; one fn visit_expr; fn visit_expr; fn make_expr; } + Pat(P) { "pattern"; one fn visit_pat; fn visit_pat; fn make_pat; } + Ty(P) { "type"; one fn visit_ty; fn visit_ty; fn make_ty; } + Stmts(SmallVec<[ast::Stmt; 1]>) { + "statement"; many fn flat_map_stmt; fn visit_stmt; fn make_stmts; + } + Items(SmallVec<[P; 1]>) { + "item"; many fn flat_map_item; fn visit_item; fn make_items; + } + TraitItems(SmallVec<[ast::TraitItem; 1]>) { + "trait item"; many fn flat_map_trait_item; fn visit_trait_item; fn make_trait_items; + } + ImplItems(SmallVec<[ast::ImplItem; 1]>) { + "impl item"; many fn flat_map_impl_item; fn visit_impl_item; fn make_impl_items; + } + ForeignItems(SmallVec<[ast::ForeignItem; 1]>) { + "foreign item"; + many fn flat_map_foreign_item; + fn visit_foreign_item; + fn make_foreign_items; + } + Arms(SmallVec<[ast::Arm; 1]>) { + "match arm"; many fn flat_map_arm; fn visit_arm; fn make_arms; + } + Fields(SmallVec<[ast::Field; 1]>) { + "field expression"; many fn flat_map_field; fn visit_field; fn make_fields; + } + FieldPats(SmallVec<[ast::FieldPat; 1]>) { + "field pattern"; + many fn flat_map_field_pattern; + fn visit_field_pattern; + fn make_field_patterns; + } + GenericParams(SmallVec<[ast::GenericParam; 1]>) { + "generic parameter"; + many fn flat_map_generic_param; + fn visit_generic_param; + fn make_generic_params; + } + Params(SmallVec<[ast::Param; 1]>) { + "function parameter"; many fn flat_map_param; fn visit_param; fn make_params; + } + StructFields(SmallVec<[ast::StructField; 1]>) { + "field"; + many fn flat_map_struct_field; + fn visit_struct_field; + fn make_struct_fields; + } + Variants(SmallVec<[ast::Variant; 1]>) { + "variant"; many fn flat_map_variant; fn visit_variant; fn make_variants; + } +} + +impl AstFragmentKind { + fn dummy(self, span: Span) -> AstFragment { + self.make_from(DummyResult::any(span)).expect("couldn't create a dummy AST fragment") + } + + fn expect_from_annotatables>(self, items: I) + -> AstFragment { + let mut items = items.into_iter(); + match self { + AstFragmentKind::Arms => + AstFragment::Arms(items.map(Annotatable::expect_arm).collect()), + AstFragmentKind::Fields => + AstFragment::Fields(items.map(Annotatable::expect_field).collect()), + AstFragmentKind::FieldPats => + AstFragment::FieldPats(items.map(Annotatable::expect_field_pattern).collect()), + AstFragmentKind::GenericParams => + AstFragment::GenericParams(items.map(Annotatable::expect_generic_param).collect()), + AstFragmentKind::Params => + AstFragment::Params(items.map(Annotatable::expect_param).collect()), + AstFragmentKind::StructFields => AstFragment::StructFields( + items.map(Annotatable::expect_struct_field).collect() + ), + AstFragmentKind::Variants => + AstFragment::Variants(items.map(Annotatable::expect_variant).collect()), + AstFragmentKind::Items => + AstFragment::Items(items.map(Annotatable::expect_item).collect()), + AstFragmentKind::ImplItems => + AstFragment::ImplItems(items.map(Annotatable::expect_impl_item).collect()), + AstFragmentKind::TraitItems => + AstFragment::TraitItems(items.map(Annotatable::expect_trait_item).collect()), + AstFragmentKind::ForeignItems => + AstFragment::ForeignItems(items.map(Annotatable::expect_foreign_item).collect()), + AstFragmentKind::Stmts => + AstFragment::Stmts(items.map(Annotatable::expect_stmt).collect()), + AstFragmentKind::Expr => AstFragment::Expr( + items.next().expect("expected exactly one expression").expect_expr() + ), + AstFragmentKind::OptExpr => + AstFragment::OptExpr(items.next().map(Annotatable::expect_expr)), + AstFragmentKind::Pat | AstFragmentKind::Ty => + panic!("patterns and types aren't annotatable"), + } + } +} + +pub struct Invocation { + pub kind: InvocationKind, + pub fragment_kind: AstFragmentKind, + pub expansion_data: ExpansionData, +} + +pub enum InvocationKind { + Bang { + mac: ast::Mac, + span: Span, + }, + Attr { + attr: ast::Attribute, + item: Annotatable, + // Required for resolving derive helper attributes. + derives: Vec, + // We temporarily report errors for attribute macros placed after derives + after_derive: bool, + }, + Derive { + path: Path, + item: Annotatable, + }, + /// "Invocation" that contains all derives from an item, + /// broken into multiple `Derive` invocations when expanded. + /// FIXME: Find a way to remove it. + DeriveContainer { + derives: Vec, + item: Annotatable, + }, +} + +impl Invocation { + pub fn span(&self) -> Span { + match &self.kind { + InvocationKind::Bang { span, .. } => *span, + InvocationKind::Attr { attr, .. } => attr.span, + InvocationKind::Derive { path, .. } => path.span, + InvocationKind::DeriveContainer { item, .. } => item.span(), + } + } +} + +pub struct MacroExpander<'a, 'b> { + pub cx: &'a mut ExtCtxt<'b>, + monotonic: bool, // cf. `cx.monotonic_expander()` +} + +impl<'a, 'b> MacroExpander<'a, 'b> { + pub fn new(cx: &'a mut ExtCtxt<'b>, monotonic: bool) -> Self { + MacroExpander { cx, monotonic } + } + + pub fn expand_crate(&mut self, mut krate: ast::Crate) -> ast::Crate { + let mut module = ModuleData { + mod_path: vec![Ident::from_str(&self.cx.ecfg.crate_name)], + directory: match self.cx.source_map().span_to_unmapped_path(krate.span) { + FileName::Real(path) => path, + other => PathBuf::from(other.to_string()), + }, + }; + module.directory.pop(); + self.cx.root_path = module.directory.clone(); + self.cx.current_expansion.module = Rc::new(module); + + let orig_mod_span = krate.module.inner; + + let krate_item = AstFragment::Items(smallvec![P(ast::Item { + attrs: krate.attrs, + span: krate.span, + kind: ast::ItemKind::Mod(krate.module), + ident: Ident::invalid(), + id: ast::DUMMY_NODE_ID, + vis: respan(krate.span.shrink_to_lo(), ast::VisibilityKind::Public), + tokens: None, + })]); + + match self.fully_expand_fragment(krate_item).make_items().pop().map(P::into_inner) { + Some(ast::Item { attrs, kind: ast::ItemKind::Mod(module), .. }) => { + krate.attrs = attrs; + krate.module = module; + }, + None => { + // Resolution failed so we return an empty expansion + krate.attrs = vec![]; + krate.module = ast::Mod { + inner: orig_mod_span, + items: vec![], + inline: true, + }; + }, + _ => unreachable!(), + }; + self.cx.trace_macros_diag(); + krate + } + + // Recursively expand all macro invocations in this AST fragment. + pub fn fully_expand_fragment(&mut self, input_fragment: AstFragment) -> AstFragment { + let orig_expansion_data = self.cx.current_expansion.clone(); + self.cx.current_expansion.depth = 0; + + // Collect all macro invocations and replace them with placeholders. + let (mut fragment_with_placeholders, mut invocations) + = self.collect_invocations(input_fragment, &[]); + + // Optimization: if we resolve all imports now, + // we'll be able to immediately resolve most of imported macros. + self.resolve_imports(); + + // Resolve paths in all invocations and produce output expanded fragments for them, but + // do not insert them into our input AST fragment yet, only store in `expanded_fragments`. + // The output fragments also go through expansion recursively until no invocations are left. + // Unresolved macros produce dummy outputs as a recovery measure. + invocations.reverse(); + let mut expanded_fragments = Vec::new(); + let mut all_derive_placeholders: FxHashMap> = FxHashMap::default(); + let mut undetermined_invocations = Vec::new(); + let (mut progress, mut force) = (false, !self.monotonic); + loop { + let invoc = if let Some(invoc) = invocations.pop() { + invoc + } else { + self.resolve_imports(); + if undetermined_invocations.is_empty() { break } + invocations = mem::take(&mut undetermined_invocations); + force = !mem::replace(&mut progress, false); + continue + }; + + let eager_expansion_root = + if self.monotonic { invoc.expansion_data.id } else { orig_expansion_data.id }; + let res = match self.cx.resolver.resolve_macro_invocation( + &invoc, eager_expansion_root, force + ) { + Ok(res) => res, + Err(Indeterminate) => { + undetermined_invocations.push(invoc); + continue + } + }; + + progress = true; + let ExpansionData { depth, id: expn_id, .. } = invoc.expansion_data; + self.cx.current_expansion = invoc.expansion_data.clone(); + + // FIXME(jseyfried): Refactor out the following logic + let (expanded_fragment, new_invocations) = match res { + InvocationRes::Single(ext) => { + let fragment = self.expand_invoc(invoc, &ext.kind); + self.collect_invocations(fragment, &[]) + } + InvocationRes::DeriveContainer(exts) => { + let (derives, item) = match invoc.kind { + InvocationKind::DeriveContainer { derives, item } => (derives, item), + _ => unreachable!(), + }; + if !item.derive_allowed() { + let attr = attr::find_by_name(item.attrs(), sym::derive) + .expect("`derive` attribute should exist"); + let span = attr.span; + let mut err = self.cx.struct_span_err(span, + "`derive` may only be applied to structs, enums and unions"); + if let ast::AttrStyle::Inner = attr.style { + let trait_list = derives.iter() + .map(|t| pprust::path_to_string(t)) + .collect::>(); + let suggestion = format!("#[derive({})]", trait_list.join(", ")); + err.span_suggestion( + span, "try an outer attribute", suggestion, + // We don't 𝑘𝑛𝑜𝑤 that the following item is an ADT + Applicability::MaybeIncorrect + ); + } + err.emit(); + } + + let mut item = self.fully_configure(item); + item.visit_attrs(|attrs| attrs.retain(|a| a.path != sym::derive)); + let mut helper_attrs = Vec::new(); + let mut has_copy = false; + for ext in exts { + helper_attrs.extend(&ext.helper_attrs); + has_copy |= ext.is_derive_copy; + } + // Mark derive helpers inside this item as known and used. + // FIXME: This is a hack, derive helpers should be integrated with regular name + // resolution instead. For example, helpers introduced by a derive container + // can be in scope for all code produced by that container's expansion. + item.visit_with(&mut MarkAttrs(&helper_attrs)); + if has_copy { + self.cx.resolver.add_derives(invoc.expansion_data.id, SpecialDerives::COPY); + } + + let derive_placeholders = + all_derive_placeholders.entry(invoc.expansion_data.id).or_default(); + derive_placeholders.reserve(derives.len()); + invocations.reserve(derives.len()); + for path in derives { + let expn_id = ExpnId::fresh(None); + derive_placeholders.push(NodeId::placeholder_from_expn_id(expn_id)); + invocations.push(Invocation { + kind: InvocationKind::Derive { path, item: item.clone() }, + fragment_kind: invoc.fragment_kind, + expansion_data: ExpansionData { + id: expn_id, + ..invoc.expansion_data.clone() + }, + }); + } + let fragment = invoc.fragment_kind + .expect_from_annotatables(::std::iter::once(item)); + self.collect_invocations(fragment, derive_placeholders) + } + }; + + if expanded_fragments.len() < depth { + expanded_fragments.push(Vec::new()); + } + expanded_fragments[depth - 1].push((expn_id, expanded_fragment)); + if !self.cx.ecfg.single_step { + invocations.extend(new_invocations.into_iter().rev()); + } + } + + self.cx.current_expansion = orig_expansion_data; + + // Finally incorporate all the expanded macros into the input AST fragment. + let mut placeholder_expander = PlaceholderExpander::new(self.cx, self.monotonic); + while let Some(expanded_fragments) = expanded_fragments.pop() { + for (expn_id, expanded_fragment) in expanded_fragments.into_iter().rev() { + let derive_placeholders = + all_derive_placeholders.remove(&expn_id).unwrap_or_else(Vec::new); + placeholder_expander.add(NodeId::placeholder_from_expn_id(expn_id), + expanded_fragment, derive_placeholders); + } + } + fragment_with_placeholders.mut_visit_with(&mut placeholder_expander); + fragment_with_placeholders + } + + fn resolve_imports(&mut self) { + if self.monotonic { + self.cx.resolver.resolve_imports(); + } + } + + /// Collects all macro invocations reachable at this time in this AST fragment, and replace + /// them with "placeholders" - dummy macro invocations with specially crafted `NodeId`s. + /// Then call into resolver that builds a skeleton ("reduced graph") of the fragment and + /// prepares data for resolving paths of macro invocations. + fn collect_invocations(&mut self, mut fragment: AstFragment, extra_placeholders: &[NodeId]) + -> (AstFragment, Vec) { + // Resolve `$crate`s in the fragment for pretty-printing. + self.cx.resolver.resolve_dollar_crates(); + + let invocations = { + let mut collector = InvocationCollector { + cfg: StripUnconfigured { + sess: self.cx.parse_sess, + features: self.cx.ecfg.features, + }, + cx: self.cx, + invocations: Vec::new(), + monotonic: self.monotonic, + }; + fragment.mut_visit_with(&mut collector); + collector.invocations + }; + + // FIXME: Merge `extra_placeholders` into the `fragment` as regular placeholders. + if self.monotonic { + self.cx.resolver.visit_ast_fragment_with_placeholders( + self.cx.current_expansion.id, &fragment, extra_placeholders); + } + + (fragment, invocations) + } + + fn fully_configure(&mut self, item: Annotatable) -> Annotatable { + let mut cfg = StripUnconfigured { + sess: self.cx.parse_sess, + features: self.cx.ecfg.features, + }; + // Since the item itself has already been configured by the InvocationCollector, + // we know that fold result vector will contain exactly one element + match item { + Annotatable::Item(item) => { + Annotatable::Item(cfg.flat_map_item(item).pop().unwrap()) + } + Annotatable::TraitItem(item) => { + Annotatable::TraitItem( + item.map(|item| cfg.flat_map_trait_item(item).pop().unwrap())) + } + Annotatable::ImplItem(item) => { + Annotatable::ImplItem(item.map(|item| cfg.flat_map_impl_item(item).pop().unwrap())) + } + Annotatable::ForeignItem(item) => { + Annotatable::ForeignItem( + item.map(|item| cfg.flat_map_foreign_item(item).pop().unwrap()) + ) + } + Annotatable::Stmt(stmt) => { + Annotatable::Stmt(stmt.map(|stmt| cfg.flat_map_stmt(stmt).pop().unwrap())) + } + Annotatable::Expr(mut expr) => { + Annotatable::Expr({ cfg.visit_expr(&mut expr); expr }) + } + Annotatable::Arm(arm) => { + Annotatable::Arm(cfg.flat_map_arm(arm).pop().unwrap()) + } + Annotatable::Field(field) => { + Annotatable::Field(cfg.flat_map_field(field).pop().unwrap()) + } + Annotatable::FieldPat(fp) => { + Annotatable::FieldPat(cfg.flat_map_field_pattern(fp).pop().unwrap()) + } + Annotatable::GenericParam(param) => { + Annotatable::GenericParam(cfg.flat_map_generic_param(param).pop().unwrap()) + } + Annotatable::Param(param) => { + Annotatable::Param(cfg.flat_map_param(param).pop().unwrap()) + } + Annotatable::StructField(sf) => { + Annotatable::StructField(cfg.flat_map_struct_field(sf).pop().unwrap()) + } + Annotatable::Variant(v) => { + Annotatable::Variant(cfg.flat_map_variant(v).pop().unwrap()) + } + } + } + + fn expand_invoc(&mut self, invoc: Invocation, ext: &SyntaxExtensionKind) -> AstFragment { + if self.cx.current_expansion.depth > self.cx.ecfg.recursion_limit { + let expn_data = self.cx.current_expansion.id.expn_data(); + let suggested_limit = self.cx.ecfg.recursion_limit * 2; + let mut err = self.cx.struct_span_err(expn_data.call_site, + &format!("recursion limit reached while expanding the macro `{}`", + expn_data.kind.descr())); + err.help(&format!( + "consider adding a `#![recursion_limit=\"{}\"]` attribute to your crate", + suggested_limit)); + err.emit(); + self.cx.trace_macros_diag(); + FatalError.raise(); + } + + let (fragment_kind, span) = (invoc.fragment_kind, invoc.span()); + match invoc.kind { + InvocationKind::Bang { mac, .. } => match ext { + SyntaxExtensionKind::Bang(expander) => { + self.gate_proc_macro_expansion_kind(span, fragment_kind); + let tok_result = expander.expand(self.cx, span, mac.stream()); + self.parse_ast_fragment(tok_result, fragment_kind, &mac.path, span) + } + SyntaxExtensionKind::LegacyBang(expander) => { + let prev = self.cx.current_expansion.prior_type_ascription; + self.cx.current_expansion.prior_type_ascription = mac.prior_type_ascription; + let tok_result = expander.expand(self.cx, span, mac.stream()); + let result = if let Some(result) = fragment_kind.make_from(tok_result) { + result + } else { + let msg = format!( + "non-{kind} macro in {kind} position: {path}", + kind = fragment_kind.name(), + path = pprust::path_to_string(&mac.path), + ); + self.cx.span_err(span, &msg); + self.cx.trace_macros_diag(); + fragment_kind.dummy(span) + }; + self.cx.current_expansion.prior_type_ascription = prev; + result + } + _ => unreachable!() + } + InvocationKind::Attr { attr, mut item, .. } => match ext { + SyntaxExtensionKind::Attr(expander) => { + self.gate_proc_macro_attr_item(span, &item); + let item_tok = TokenTree::token(token::Interpolated(Lrc::new(match item { + Annotatable::Item(item) => token::NtItem(item), + Annotatable::TraitItem(item) => token::NtTraitItem(item.into_inner()), + Annotatable::ImplItem(item) => token::NtImplItem(item.into_inner()), + Annotatable::ForeignItem(item) => token::NtForeignItem(item.into_inner()), + Annotatable::Stmt(stmt) => token::NtStmt(stmt.into_inner()), + Annotatable::Expr(expr) => token::NtExpr(expr), + Annotatable::Arm(..) + | Annotatable::Field(..) + | Annotatable::FieldPat(..) + | Annotatable::GenericParam(..) + | Annotatable::Param(..) + | Annotatable::StructField(..) + | Annotatable::Variant(..) + => panic!("unexpected annotatable"), + })), DUMMY_SP).into(); + let input = self.extract_proc_macro_attr_input(attr.item.tokens, span); + let tok_result = expander.expand(self.cx, span, input, item_tok); + self.parse_ast_fragment(tok_result, fragment_kind, &attr.item.path, span) + } + SyntaxExtensionKind::LegacyAttr(expander) => { + match attr.parse_meta(self.cx.parse_sess) { + Ok(meta) => { + let item = expander.expand(self.cx, span, &meta, item); + fragment_kind.expect_from_annotatables(item) + } + Err(mut err) => { + err.emit(); + fragment_kind.dummy(span) + } + } + } + SyntaxExtensionKind::NonMacroAttr { mark_used } => { + attr::mark_known(&attr); + if *mark_used { + attr::mark_used(&attr); + } + item.visit_attrs(|attrs| attrs.push(attr)); + fragment_kind.expect_from_annotatables(iter::once(item)) + } + _ => unreachable!() + } + InvocationKind::Derive { path, item } => match ext { + SyntaxExtensionKind::Derive(expander) | + SyntaxExtensionKind::LegacyDerive(expander) => { + if !item.derive_allowed() { + return fragment_kind.dummy(span); + } + let meta = ast::MetaItem { kind: ast::MetaItemKind::Word, span, path }; + let items = expander.expand(self.cx, span, &meta, item); + fragment_kind.expect_from_annotatables(items) + } + _ => unreachable!() + } + InvocationKind::DeriveContainer { .. } => unreachable!() + } + } + + fn extract_proc_macro_attr_input(&self, tokens: TokenStream, span: Span) -> TokenStream { + let mut trees = tokens.trees(); + match trees.next() { + Some(TokenTree::Delimited(_, _, tts)) => { + if trees.next().is_none() { + return tts.into() + } + } + Some(TokenTree::Token(..)) => {} + None => return TokenStream::default(), + } + self.cx.span_err(span, "custom attribute invocations must be \ + of the form `#[foo]` or `#[foo(..)]`, the macro name must only be \ + followed by a delimiter token"); + TokenStream::default() + } + + fn gate_proc_macro_attr_item(&self, span: Span, item: &Annotatable) { + let (kind, gate) = match *item { + Annotatable::Item(ref item) => { + match item.kind { + ItemKind::Mod(_) if self.cx.ecfg.proc_macro_hygiene() => return, + ItemKind::Mod(_) => ("modules", sym::proc_macro_hygiene), + _ => return, + } + } + Annotatable::TraitItem(_) => return, + Annotatable::ImplItem(_) => return, + Annotatable::ForeignItem(_) => return, + Annotatable::Stmt(_) | + Annotatable::Expr(_) if self.cx.ecfg.proc_macro_hygiene() => return, + Annotatable::Stmt(_) => ("statements", sym::proc_macro_hygiene), + Annotatable::Expr(_) => ("expressions", sym::proc_macro_hygiene), + Annotatable::Arm(..) + | Annotatable::Field(..) + | Annotatable::FieldPat(..) + | Annotatable::GenericParam(..) + | Annotatable::Param(..) + | Annotatable::StructField(..) + | Annotatable::Variant(..) + => panic!("unexpected annotatable"), + }; + emit_feature_err( + self.cx.parse_sess, + gate, + span, + GateIssue::Language, + &format!("custom attributes cannot be applied to {}", kind), + ); + } + + fn gate_proc_macro_expansion_kind(&self, span: Span, kind: AstFragmentKind) { + let kind = match kind { + AstFragmentKind::Expr | + AstFragmentKind::OptExpr => "expressions", + AstFragmentKind::Pat => "patterns", + AstFragmentKind::Stmts => "statements", + AstFragmentKind::Ty | + AstFragmentKind::Items | + AstFragmentKind::TraitItems | + AstFragmentKind::ImplItems | + AstFragmentKind::ForeignItems => return, + AstFragmentKind::Arms + | AstFragmentKind::Fields + | AstFragmentKind::FieldPats + | AstFragmentKind::GenericParams + | AstFragmentKind::Params + | AstFragmentKind::StructFields + | AstFragmentKind::Variants + => panic!("unexpected AST fragment kind"), + }; + if self.cx.ecfg.proc_macro_hygiene() { + return + } + emit_feature_err( + self.cx.parse_sess, + sym::proc_macro_hygiene, + span, + GateIssue::Language, + &format!("procedural macros cannot be expanded to {}", kind), + ); + } + + fn parse_ast_fragment( + &mut self, + toks: TokenStream, + kind: AstFragmentKind, + path: &Path, + span: Span, + ) -> AstFragment { + let mut parser = self.cx.new_parser_from_tts(toks); + match parse_ast_fragment(&mut parser, kind, false) { + Ok(fragment) => { + ensure_complete_parse(&mut parser, path, kind.name(), span); + fragment + } + Err(mut err) => { + err.set_span(span); + annotate_err_with_kind(&mut err, kind, span); + err.emit(); + self.cx.trace_macros_diag(); + kind.dummy(span) + } + } + } +} + +pub fn parse_ast_fragment<'a>( + this: &mut Parser<'a>, + kind: AstFragmentKind, + macro_legacy_warnings: bool, +) -> PResult<'a, AstFragment> { + Ok(match kind { + AstFragmentKind::Items => { + let mut items = SmallVec::new(); + while let Some(item) = this.parse_item()? { + items.push(item); + } + AstFragment::Items(items) + } + AstFragmentKind::TraitItems => { + let mut items = SmallVec::new(); + while this.token != token::Eof { + items.push(this.parse_trait_item(&mut false)?); + } + AstFragment::TraitItems(items) + } + AstFragmentKind::ImplItems => { + let mut items = SmallVec::new(); + while this.token != token::Eof { + items.push(this.parse_impl_item(&mut false)?); + } + AstFragment::ImplItems(items) + } + AstFragmentKind::ForeignItems => { + let mut items = SmallVec::new(); + while this.token != token::Eof { + items.push(this.parse_foreign_item(DUMMY_SP)?); + } + AstFragment::ForeignItems(items) + } + AstFragmentKind::Stmts => { + let mut stmts = SmallVec::new(); + while this.token != token::Eof && + // won't make progress on a `}` + this.token != token::CloseDelim(token::Brace) { + if let Some(stmt) = this.parse_full_stmt(macro_legacy_warnings)? { + stmts.push(stmt); + } + } + AstFragment::Stmts(stmts) + } + AstFragmentKind::Expr => AstFragment::Expr(this.parse_expr()?), + AstFragmentKind::OptExpr => { + if this.token != token::Eof { + AstFragment::OptExpr(Some(this.parse_expr()?)) + } else { + AstFragment::OptExpr(None) + } + }, + AstFragmentKind::Ty => AstFragment::Ty(this.parse_ty()?), + AstFragmentKind::Pat => AstFragment::Pat(this.parse_pat(None)?), + AstFragmentKind::Arms + | AstFragmentKind::Fields + | AstFragmentKind::FieldPats + | AstFragmentKind::GenericParams + | AstFragmentKind::Params + | AstFragmentKind::StructFields + | AstFragmentKind::Variants + => panic!("unexpected AST fragment kind"), + }) +} + +pub fn ensure_complete_parse<'a>( + this: &mut Parser<'a>, + macro_path: &Path, + kind_name: &str, + span: Span, +) { + if this.token != token::Eof { + let msg = format!("macro expansion ignores token `{}` and any following", + this.this_token_to_string()); + // Avoid emitting backtrace info twice. + let def_site_span = this.token.span.with_ctxt(SyntaxContext::root()); + let mut err = this.struct_span_err(def_site_span, &msg); + err.span_label(span, "caused by the macro expansion here"); + let msg = format!( + "the usage of `{}!` is likely invalid in {} context", + pprust::path_to_string(macro_path), + kind_name, + ); + err.note(&msg); + let semi_span = this.sess.source_map().next_point(span); + + let semi_full_span = semi_span.to(this.sess.source_map().next_point(semi_span)); + match this.sess.source_map().span_to_snippet(semi_full_span) { + Ok(ref snippet) if &snippet[..] != ";" && kind_name == "expression" => { + err.span_suggestion( + semi_span, + "you might be missing a semicolon here", + ";".to_owned(), + Applicability::MaybeIncorrect, + ); + } + _ => {} + } + err.emit(); + } +} + +struct InvocationCollector<'a, 'b> { + cx: &'a mut ExtCtxt<'b>, + cfg: StripUnconfigured<'a>, + invocations: Vec, + monotonic: bool, +} + +impl<'a, 'b> InvocationCollector<'a, 'b> { + fn collect(&mut self, fragment_kind: AstFragmentKind, kind: InvocationKind) -> AstFragment { + // Expansion data for all the collected invocations is set upon their resolution, + // with exception of the derive container case which is not resolved and can get + // its expansion data immediately. + let expn_data = match &kind { + InvocationKind::DeriveContainer { item, .. } => Some(ExpnData { + parent: self.cx.current_expansion.id, + ..ExpnData::default( + ExpnKind::Macro(MacroKind::Attr, sym::derive), + item.span(), self.cx.parse_sess.edition, + ) + }), + _ => None, + }; + let expn_id = ExpnId::fresh(expn_data); + self.invocations.push(Invocation { + kind, + fragment_kind, + expansion_data: ExpansionData { + id: expn_id, + depth: self.cx.current_expansion.depth + 1, + ..self.cx.current_expansion.clone() + }, + }); + placeholder(fragment_kind, NodeId::placeholder_from_expn_id(expn_id)) + } + + fn collect_bang(&mut self, mac: ast::Mac, span: Span, kind: AstFragmentKind) -> AstFragment { + self.collect(kind, InvocationKind::Bang { mac, span }) + } + + fn collect_attr(&mut self, + attr: Option, + derives: Vec, + item: Annotatable, + kind: AstFragmentKind, + after_derive: bool) + -> AstFragment { + self.collect(kind, match attr { + Some(attr) => InvocationKind::Attr { attr, item, derives, after_derive }, + None => InvocationKind::DeriveContainer { derives, item }, + }) + } + + fn find_attr_invoc(&self, attrs: &mut Vec, after_derive: &mut bool) + -> Option { + let attr = attrs.iter() + .position(|a| { + if a.path == sym::derive { + *after_derive = true; + } + !attr::is_known(a) && !is_builtin_attr(a) + }) + .map(|i| attrs.remove(i)); + if let Some(attr) = &attr { + if !self.cx.ecfg.custom_inner_attributes() && + attr.style == ast::AttrStyle::Inner && attr.path != sym::test { + emit_feature_err(&self.cx.parse_sess, sym::custom_inner_attributes, + attr.span, GateIssue::Language, + "non-builtin inner attributes are unstable"); + } + } + attr + } + + /// If `item` is an attr invocation, remove and return the macro attribute and derive traits. + fn classify_item(&mut self, item: &mut T) + -> (Option, Vec, /* after_derive */ bool) + where T: HasAttrs, + { + let (mut attr, mut traits, mut after_derive) = (None, Vec::new(), false); + + item.visit_attrs(|mut attrs| { + attr = self.find_attr_invoc(&mut attrs, &mut after_derive); + traits = collect_derives(&mut self.cx, &mut attrs); + }); + + (attr, traits, after_derive) + } + + /// Alternative to `classify_item()` that ignores `#[derive]` so invocations fallthrough + /// to the unused-attributes lint (making it an error on statements and expressions + /// is a breaking change) + fn classify_nonitem(&mut self, nonitem: &mut T) + -> (Option, /* after_derive */ bool) { + let (mut attr, mut after_derive) = (None, false); + + nonitem.visit_attrs(|mut attrs| { + attr = self.find_attr_invoc(&mut attrs, &mut after_derive); + }); + + (attr, after_derive) + } + + fn configure(&mut self, node: T) -> Option { + self.cfg.configure(node) + } + + // Detect use of feature-gated or invalid attributes on macro invocations + // since they will not be detected after macro expansion. + fn check_attributes(&mut self, attrs: &[ast::Attribute]) { + let features = self.cx.ecfg.features.unwrap(); + for attr in attrs.iter() { + feature_gate::check_attribute(attr, self.cx.parse_sess, features); + + // macros are expanded before any lint passes so this warning has to be hardcoded + if attr.path == sym::derive { + self.cx.struct_span_warn(attr.span, "`#[derive]` does nothing on macro invocations") + .note("this may become a hard error in a future release") + .emit(); + } + } + } +} + +impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> { + fn visit_expr(&mut self, expr: &mut P) { + self.cfg.configure_expr(expr); + visit_clobber(expr.deref_mut(), |mut expr| { + self.cfg.configure_expr_kind(&mut expr.kind); + + // ignore derives so they remain unused + let (attr, after_derive) = self.classify_nonitem(&mut expr); + + if attr.is_some() { + // Collect the invoc regardless of whether or not attributes are permitted here + // expansion will eat the attribute so it won't error later. + attr.as_ref().map(|a| self.cfg.maybe_emit_expr_attr_err(a)); + + // AstFragmentKind::Expr requires the macro to emit an expression. + return self.collect_attr(attr, vec![], Annotatable::Expr(P(expr)), + AstFragmentKind::Expr, after_derive) + .make_expr() + .into_inner() + } + + if let ast::ExprKind::Mac(mac) = expr.kind { + self.check_attributes(&expr.attrs); + self.collect_bang(mac, expr.span, AstFragmentKind::Expr) + .make_expr() + .into_inner() + } else { + noop_visit_expr(&mut expr, self); + expr + } + }); + } + + fn flat_map_arm(&mut self, arm: ast::Arm) -> SmallVec<[ast::Arm; 1]> { + let mut arm = configure!(self, arm); + + let (attr, traits, after_derive) = self.classify_item(&mut arm); + if attr.is_some() || !traits.is_empty() { + return self.collect_attr(attr, traits, Annotatable::Arm(arm), + AstFragmentKind::Arms, after_derive) + .make_arms(); + } + + noop_flat_map_arm(arm, self) + } + + fn flat_map_field(&mut self, field: ast::Field) -> SmallVec<[ast::Field; 1]> { + let mut field = configure!(self, field); + + let (attr, traits, after_derive) = self.classify_item(&mut field); + if attr.is_some() || !traits.is_empty() { + return self.collect_attr(attr, traits, Annotatable::Field(field), + AstFragmentKind::Fields, after_derive) + .make_fields(); + } + + noop_flat_map_field(field, self) + } + + fn flat_map_field_pattern(&mut self, fp: ast::FieldPat) -> SmallVec<[ast::FieldPat; 1]> { + let mut fp = configure!(self, fp); + + let (attr, traits, after_derive) = self.classify_item(&mut fp); + if attr.is_some() || !traits.is_empty() { + return self.collect_attr(attr, traits, Annotatable::FieldPat(fp), + AstFragmentKind::FieldPats, after_derive) + .make_field_patterns(); + } + + noop_flat_map_field_pattern(fp, self) + } + + fn flat_map_param(&mut self, p: ast::Param) -> SmallVec<[ast::Param; 1]> { + let mut p = configure!(self, p); + + let (attr, traits, after_derive) = self.classify_item(&mut p); + if attr.is_some() || !traits.is_empty() { + return self.collect_attr(attr, traits, Annotatable::Param(p), + AstFragmentKind::Params, after_derive) + .make_params(); + } + + noop_flat_map_param(p, self) + } + + fn flat_map_struct_field(&mut self, sf: ast::StructField) -> SmallVec<[ast::StructField; 1]> { + let mut sf = configure!(self, sf); + + let (attr, traits, after_derive) = self.classify_item(&mut sf); + if attr.is_some() || !traits.is_empty() { + return self.collect_attr(attr, traits, Annotatable::StructField(sf), + AstFragmentKind::StructFields, after_derive) + .make_struct_fields(); + } + + noop_flat_map_struct_field(sf, self) + } + + fn flat_map_variant(&mut self, variant: ast::Variant) -> SmallVec<[ast::Variant; 1]> { + let mut variant = configure!(self, variant); + + let (attr, traits, after_derive) = self.classify_item(&mut variant); + if attr.is_some() || !traits.is_empty() { + return self.collect_attr(attr, traits, Annotatable::Variant(variant), + AstFragmentKind::Variants, after_derive) + .make_variants(); + } + + noop_flat_map_variant(variant, self) + } + + fn filter_map_expr(&mut self, expr: P) -> Option> { + let expr = configure!(self, expr); + expr.filter_map(|mut expr| { + self.cfg.configure_expr_kind(&mut expr.kind); + + // Ignore derives so they remain unused. + let (attr, after_derive) = self.classify_nonitem(&mut expr); + + if attr.is_some() { + attr.as_ref().map(|a| self.cfg.maybe_emit_expr_attr_err(a)); + + return self.collect_attr(attr, vec![], Annotatable::Expr(P(expr)), + AstFragmentKind::OptExpr, after_derive) + .make_opt_expr() + .map(|expr| expr.into_inner()) + } + + if let ast::ExprKind::Mac(mac) = expr.kind { + self.check_attributes(&expr.attrs); + self.collect_bang(mac, expr.span, AstFragmentKind::OptExpr) + .make_opt_expr() + .map(|expr| expr.into_inner()) + } else { + Some({ noop_visit_expr(&mut expr, self); expr }) + } + }) + } + + fn visit_pat(&mut self, pat: &mut P) { + self.cfg.configure_pat(pat); + match pat.kind { + PatKind::Mac(_) => {} + _ => return noop_visit_pat(pat, self), + } + + visit_clobber(pat, |mut pat| { + match mem::replace(&mut pat.kind, PatKind::Wild) { + PatKind::Mac(mac) => + self.collect_bang(mac, pat.span, AstFragmentKind::Pat).make_pat(), + _ => unreachable!(), + } + }); + } + + fn flat_map_stmt(&mut self, stmt: ast::Stmt) -> SmallVec<[ast::Stmt; 1]> { + let mut stmt = configure!(self, stmt); + + // we'll expand attributes on expressions separately + if !stmt.is_expr() { + let (attr, derives, after_derive) = if stmt.is_item() { + self.classify_item(&mut stmt) + } else { + // ignore derives on non-item statements so it falls through + // to the unused-attributes lint + let (attr, after_derive) = self.classify_nonitem(&mut stmt); + (attr, vec![], after_derive) + }; + + if attr.is_some() || !derives.is_empty() { + return self.collect_attr(attr, derives, Annotatable::Stmt(P(stmt)), + AstFragmentKind::Stmts, after_derive).make_stmts(); + } + } + + if let StmtKind::Mac(mac) = stmt.kind { + let (mac, style, attrs) = mac.into_inner(); + self.check_attributes(&attrs); + let mut placeholder = self.collect_bang(mac, stmt.span, AstFragmentKind::Stmts) + .make_stmts(); + + // If this is a macro invocation with a semicolon, then apply that + // semicolon to the final statement produced by expansion. + if style == MacStmtStyle::Semicolon { + if let Some(stmt) = placeholder.pop() { + placeholder.push(stmt.add_trailing_semicolon()); + } + } + + return placeholder; + } + + // The placeholder expander gives ids to statements, so we avoid folding the id here. + let ast::Stmt { id, kind, span } = stmt; + noop_flat_map_stmt_kind(kind, self).into_iter().map(|kind| { + ast::Stmt { id, kind, span } + }).collect() + + } + + fn visit_block(&mut self, block: &mut P) { + let old_directory_ownership = self.cx.current_expansion.directory_ownership; + self.cx.current_expansion.directory_ownership = DirectoryOwnership::UnownedViaBlock; + noop_visit_block(block, self); + self.cx.current_expansion.directory_ownership = old_directory_ownership; + } + + fn flat_map_item(&mut self, item: P) -> SmallVec<[P; 1]> { + let mut item = configure!(self, item); + + let (attr, traits, after_derive) = self.classify_item(&mut item); + if attr.is_some() || !traits.is_empty() { + return self.collect_attr(attr, traits, Annotatable::Item(item), + AstFragmentKind::Items, after_derive).make_items(); + } + + match item.kind { + ast::ItemKind::Mac(..) => { + self.check_attributes(&item.attrs); + item.and_then(|item| match item.kind { + ItemKind::Mac(mac) => self.collect( + AstFragmentKind::Items, InvocationKind::Bang { mac, span: item.span } + ).make_items(), + _ => unreachable!(), + }) + } + ast::ItemKind::Mod(ast::Mod { inner, .. }) => { + if item.ident == Ident::invalid() { + return noop_flat_map_item(item, self); + } + + let orig_directory_ownership = self.cx.current_expansion.directory_ownership; + let mut module = (*self.cx.current_expansion.module).clone(); + module.mod_path.push(item.ident); + + // Detect if this is an inline module (`mod m { ... }` as opposed to `mod m;`). + // In the non-inline case, `inner` is never the dummy span (cf. `parse_item_mod`). + // Thus, if `inner` is the dummy span, we know the module is inline. + let inline_module = item.span.contains(inner) || inner.is_dummy(); + + if inline_module { + if let Some(path) = attr::first_attr_value_str_by_name(&item.attrs, sym::path) { + self.cx.current_expansion.directory_ownership = + DirectoryOwnership::Owned { relative: None }; + module.directory.push(&*path.as_str()); + } else { + module.directory.push(&*item.ident.as_str()); + } + } else { + let path = self.cx.parse_sess.source_map().span_to_unmapped_path(inner); + let mut path = match path { + FileName::Real(path) => path, + other => PathBuf::from(other.to_string()), + }; + let directory_ownership = match path.file_name().unwrap().to_str() { + Some("mod.rs") => DirectoryOwnership::Owned { relative: None }, + Some(_) => DirectoryOwnership::Owned { + relative: Some(item.ident), + }, + None => DirectoryOwnership::UnownedViaMod(false), + }; + path.pop(); + module.directory = path; + self.cx.current_expansion.directory_ownership = directory_ownership; + } + + let orig_module = + mem::replace(&mut self.cx.current_expansion.module, Rc::new(module)); + let result = noop_flat_map_item(item, self); + self.cx.current_expansion.module = orig_module; + self.cx.current_expansion.directory_ownership = orig_directory_ownership; + result + } + + _ => noop_flat_map_item(item, self), + } + } + + fn flat_map_trait_item(&mut self, item: ast::TraitItem) -> SmallVec<[ast::TraitItem; 1]> { + let mut item = configure!(self, item); + + let (attr, traits, after_derive) = self.classify_item(&mut item); + if attr.is_some() || !traits.is_empty() { + return self.collect_attr(attr, traits, Annotatable::TraitItem(P(item)), + AstFragmentKind::TraitItems, after_derive).make_trait_items() + } + + match item.kind { + ast::TraitItemKind::Macro(mac) => { + let ast::TraitItem { attrs, span, .. } = item; + self.check_attributes(&attrs); + self.collect_bang(mac, span, AstFragmentKind::TraitItems).make_trait_items() + } + _ => noop_flat_map_trait_item(item, self), + } + } + + fn flat_map_impl_item(&mut self, item: ast::ImplItem) -> SmallVec<[ast::ImplItem; 1]> { + let mut item = configure!(self, item); + + let (attr, traits, after_derive) = self.classify_item(&mut item); + if attr.is_some() || !traits.is_empty() { + return self.collect_attr(attr, traits, Annotatable::ImplItem(P(item)), + AstFragmentKind::ImplItems, after_derive).make_impl_items(); + } + + match item.kind { + ast::ImplItemKind::Macro(mac) => { + let ast::ImplItem { attrs, span, .. } = item; + self.check_attributes(&attrs); + self.collect_bang(mac, span, AstFragmentKind::ImplItems).make_impl_items() + } + _ => noop_flat_map_impl_item(item, self), + } + } + + fn visit_ty(&mut self, ty: &mut P) { + match ty.kind { + ast::TyKind::Mac(_) => {} + _ => return noop_visit_ty(ty, self), + }; + + visit_clobber(ty, |mut ty| { + match mem::replace(&mut ty.kind, ast::TyKind::Err) { + ast::TyKind::Mac(mac) => + self.collect_bang(mac, ty.span, AstFragmentKind::Ty).make_ty(), + _ => unreachable!(), + } + }); + } + + fn visit_foreign_mod(&mut self, foreign_mod: &mut ast::ForeignMod) { + self.cfg.configure_foreign_mod(foreign_mod); + noop_visit_foreign_mod(foreign_mod, self); + } + + fn flat_map_foreign_item(&mut self, mut foreign_item: ast::ForeignItem) + -> SmallVec<[ast::ForeignItem; 1]> + { + let (attr, traits, after_derive) = self.classify_item(&mut foreign_item); + + if attr.is_some() || !traits.is_empty() { + return self.collect_attr(attr, traits, Annotatable::ForeignItem(P(foreign_item)), + AstFragmentKind::ForeignItems, after_derive) + .make_foreign_items(); + } + + if let ast::ForeignItemKind::Macro(mac) = foreign_item.kind { + self.check_attributes(&foreign_item.attrs); + return self.collect_bang(mac, foreign_item.span, AstFragmentKind::ForeignItems) + .make_foreign_items(); + } + + noop_flat_map_foreign_item(foreign_item, self) + } + + fn visit_item_kind(&mut self, item: &mut ast::ItemKind) { + match item { + ast::ItemKind::MacroDef(..) => {} + _ => { + self.cfg.configure_item_kind(item); + noop_visit_item_kind(item, self); + } + } + } + + fn flat_map_generic_param( + &mut self, + param: ast::GenericParam + ) -> SmallVec<[ast::GenericParam; 1]> + { + let mut param = configure!(self, param); + + let (attr, traits, after_derive) = self.classify_item(&mut param); + if attr.is_some() || !traits.is_empty() { + return self.collect_attr(attr, traits, Annotatable::GenericParam(param), + AstFragmentKind::GenericParams, after_derive) + .make_generic_params(); + } + + noop_flat_map_generic_param(param, self) + } + + fn visit_attribute(&mut self, at: &mut ast::Attribute) { + // turn `#[doc(include="filename")]` attributes into `#[doc(include(file="filename", + // contents="file contents")]` attributes + if !at.check_name(sym::doc) { + return noop_visit_attribute(at, self); + } + + if let Some(list) = at.meta_item_list() { + if !list.iter().any(|it| it.check_name(sym::include)) { + return noop_visit_attribute(at, self); + } + + let mut items = vec![]; + + for mut it in list { + if !it.check_name(sym::include) { + items.push({ noop_visit_meta_list_item(&mut it, self); it }); + continue; + } + + if let Some(file) = it.value_str() { + let err_count = self.cx.parse_sess.span_diagnostic.err_count(); + self.check_attributes(slice::from_ref(at)); + if self.cx.parse_sess.span_diagnostic.err_count() > err_count { + // avoid loading the file if they haven't enabled the feature + return noop_visit_attribute(at, self); + } + + let filename = self.cx.resolve_path(&*file.as_str(), it.span()); + match self.cx.source_map().load_file(&filename) { + Ok(source_file) => { + let src = source_file.src.as_ref() + .expect("freshly loaded file should have a source"); + let src_interned = Symbol::intern(src.as_str()); + + let include_info = vec![ + ast::NestedMetaItem::MetaItem( + attr::mk_name_value_item_str( + Ident::with_dummy_span(sym::file), + file, + DUMMY_SP, + ), + ), + ast::NestedMetaItem::MetaItem( + attr::mk_name_value_item_str( + Ident::with_dummy_span(sym::contents), + src_interned, + DUMMY_SP, + ), + ), + ]; + + let include_ident = Ident::with_dummy_span(sym::include); + let item = attr::mk_list_item(include_ident, include_info); + items.push(ast::NestedMetaItem::MetaItem(item)); + } + Err(e) => { + let lit = it + .meta_item() + .and_then(|item| item.name_value_literal()) + .unwrap(); + + if e.kind() == ErrorKind::InvalidData { + self.cx + .struct_span_err( + lit.span, + &format!("{} wasn't a utf-8 file", filename.display()), + ) + .span_label(lit.span, "contains invalid utf-8") + .emit(); + } else { + let mut err = self.cx.struct_span_err( + lit.span, + &format!("couldn't read {}: {}", filename.display(), e), + ); + err.span_label(lit.span, "couldn't read file"); + + err.emit(); + } + } + } + } else { + let mut err = self.cx.struct_span_err( + it.span(), + &format!("expected path to external documentation"), + ); + + // Check if the user erroneously used `doc(include(...))` syntax. + let literal = it.meta_item_list().and_then(|list| { + if list.len() == 1 { + list[0].literal().map(|literal| &literal.kind) + } else { + None + } + }); + + let (path, applicability) = match &literal { + Some(LitKind::Str(path, ..)) => { + (path.to_string(), Applicability::MachineApplicable) + } + _ => (String::from(""), Applicability::HasPlaceholders), + }; + + err.span_suggestion( + it.span(), + "provide a file path with `=`", + format!("include = \"{}\"", path), + applicability, + ); + + err.emit(); + } + } + + let meta = attr::mk_list_item(Ident::with_dummy_span(sym::doc), items); + *at = attr::Attribute { + item: AttrItem { path: meta.path, tokens: meta.kind.tokens(meta.span) }, + span: at.span, + id: at.id, + style: at.style, + is_sugared_doc: false, + }; + } else { + noop_visit_attribute(at, self) + } + } + + fn visit_id(&mut self, id: &mut ast::NodeId) { + if self.monotonic { + debug_assert_eq!(*id, ast::DUMMY_NODE_ID); + *id = self.cx.resolver.next_node_id() + } + } + + fn visit_fn_decl(&mut self, mut fn_decl: &mut P) { + self.cfg.configure_fn_decl(&mut fn_decl); + noop_visit_fn_decl(fn_decl, self); + } +} + +pub struct ExpansionConfig<'feat> { + pub crate_name: String, + pub features: Option<&'feat Features>, + pub recursion_limit: usize, + pub trace_mac: bool, + pub should_test: bool, // If false, strip `#[test]` nodes + pub single_step: bool, + pub keep_macs: bool, +} + +impl<'feat> ExpansionConfig<'feat> { + pub fn default(crate_name: String) -> ExpansionConfig<'static> { + ExpansionConfig { + crate_name, + features: None, + recursion_limit: 1024, + trace_mac: false, + should_test: false, + single_step: false, + keep_macs: false, + } + } + + fn proc_macro_hygiene(&self) -> bool { + self.features.map_or(false, |features| features.proc_macro_hygiene) + } + fn custom_inner_attributes(&self) -> bool { + self.features.map_or(false, |features| features.custom_inner_attributes) + } +} diff --git a/src/libsyntax_expand/lib.rs b/src/libsyntax_expand/lib.rs new file mode 100644 index 00000000000..88e69d79397 --- /dev/null +++ b/src/libsyntax_expand/lib.rs @@ -0,0 +1,38 @@ +#![feature(crate_visibility_modifier)] +#![feature(proc_macro_diagnostic)] +#![feature(proc_macro_internals)] +#![feature(proc_macro_span)] + +extern crate proc_macro as pm; + +// A variant of 'try!' that panics on an Err. This is used as a crutch on the +// way towards a non-panic!-prone parser. It should be used for fatal parsing +// errors; eventually we plan to convert all code using panictry to just use +// normal try. +#[macro_export] +macro_rules! panictry { + ($e:expr) => ({ + use std::result::Result::{Ok, Err}; + use errors::FatalError; + match $e { + Ok(e) => e, + Err(mut e) => { + e.emit(); + FatalError.raise() + } + } + }) +} + +mod placeholders; +mod proc_macro_server; + +pub use syntax_pos::hygiene; +pub use mbe::macro_rules::compile_declarative_macro; +pub mod allocator; +pub mod base; +pub mod build; +pub mod expand; +pub mod proc_macro; + +crate mod mbe; diff --git a/src/libsyntax_expand/mbe.rs b/src/libsyntax_expand/mbe.rs new file mode 100644 index 00000000000..453fe94f1de --- /dev/null +++ b/src/libsyntax_expand/mbe.rs @@ -0,0 +1,166 @@ +//! This module implements declarative macros: old `macro_rules` and the newer +//! `macro`. Declarative macros are also known as "macro by example", and that's +//! why we call this module `mbe`. For external documentation, prefer the +//! official terminology: "declarative macros". + +crate mod transcribe; +crate mod macro_check; +crate mod macro_parser; +crate mod macro_rules; +crate mod quoted; + +use syntax::ast; +use syntax::parse::token::{self, Token, TokenKind}; +use syntax::tokenstream::{DelimSpan}; + +use syntax_pos::{BytePos, Span}; + +use rustc_data_structures::sync::Lrc; + +/// Contains the sub-token-trees of a "delimited" token tree, such as the contents of `(`. Note +/// that the delimiter itself might be `NoDelim`. +#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug)] +struct Delimited { + delim: token::DelimToken, + tts: Vec, +} + +impl Delimited { + /// Returns a `self::TokenTree` with a `Span` corresponding to the opening delimiter. + fn open_tt(&self, span: Span) -> TokenTree { + let open_span = if span.is_dummy() { + span + } else { + span.with_hi(span.lo() + BytePos(self.delim.len() as u32)) + }; + TokenTree::token(token::OpenDelim(self.delim), open_span) + } + + /// Returns a `self::TokenTree` with a `Span` corresponding to the closing delimiter. + fn close_tt(&self, span: Span) -> TokenTree { + let close_span = if span.is_dummy() { + span + } else { + span.with_lo(span.hi() - BytePos(self.delim.len() as u32)) + }; + TokenTree::token(token::CloseDelim(self.delim), close_span) + } +} + +#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug)] +struct SequenceRepetition { + /// The sequence of token trees + tts: Vec, + /// The optional separator + separator: Option, + /// Whether the sequence can be repeated zero (*), or one or more times (+) + kleene: KleeneToken, + /// The number of `Match`s that appear in the sequence (and subsequences) + num_captures: usize, +} + +#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug, Copy)] +struct KleeneToken { + span: Span, + op: KleeneOp, +} + +impl KleeneToken { + fn new(op: KleeneOp, span: Span) -> KleeneToken { + KleeneToken { span, op } + } +} + +/// A Kleene-style [repetition operator](http://en.wikipedia.org/wiki/Kleene_star) +/// for token sequences. +#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] +enum KleeneOp { + /// Kleene star (`*`) for zero or more repetitions + ZeroOrMore, + /// Kleene plus (`+`) for one or more repetitions + OneOrMore, + /// Kleene optional (`?`) for zero or one reptitions + ZeroOrOne, +} + +/// Similar to `tokenstream::TokenTree`, except that `$i`, `$i:ident`, and `$(...)` +/// are "first-class" token trees. Useful for parsing macros. +#[derive(Debug, Clone, PartialEq, RustcEncodable, RustcDecodable)] +enum TokenTree { + Token(Token), + Delimited(DelimSpan, Lrc), + /// A kleene-style repetition sequence + Sequence(DelimSpan, Lrc), + /// e.g., `$var` + MetaVar(Span, ast::Ident), + /// e.g., `$var:expr`. This is only used in the left hand side of MBE macros. + MetaVarDecl( + Span, + ast::Ident, /* name to bind */ + ast::Ident, /* kind of nonterminal */ + ), +} + +impl TokenTree { + /// Return the number of tokens in the tree. + fn len(&self) -> usize { + match *self { + TokenTree::Delimited(_, ref delimed) => match delimed.delim { + token::NoDelim => delimed.tts.len(), + _ => delimed.tts.len() + 2, + }, + TokenTree::Sequence(_, ref seq) => seq.tts.len(), + _ => 0, + } + } + + /// Returns `true` if the given token tree is delimited. + fn is_delimited(&self) -> bool { + match *self { + TokenTree::Delimited(..) => true, + _ => false, + } + } + + /// Returns `true` if the given token tree is a token of the given kind. + fn is_token(&self, expected_kind: &TokenKind) -> bool { + match self { + TokenTree::Token(Token { kind: actual_kind, .. }) => actual_kind == expected_kind, + _ => false, + } + } + + /// Gets the `index`-th sub-token-tree. This only makes sense for delimited trees and sequences. + fn get_tt(&self, index: usize) -> TokenTree { + match (self, index) { + (&TokenTree::Delimited(_, ref delimed), _) if delimed.delim == token::NoDelim => { + delimed.tts[index].clone() + } + (&TokenTree::Delimited(span, ref delimed), _) => { + if index == 0 { + return delimed.open_tt(span.open); + } + if index == delimed.tts.len() + 1 { + return delimed.close_tt(span.close); + } + delimed.tts[index - 1].clone() + } + (&TokenTree::Sequence(_, ref seq), _) => seq.tts[index].clone(), + _ => panic!("Cannot expand a token tree"), + } + } + + /// Retrieves the `TokenTree`'s span. + fn span(&self) -> Span { + match *self { + TokenTree::Token(Token { span, .. }) + | TokenTree::MetaVar(span, _) + | TokenTree::MetaVarDecl(span, _, _) => span, + TokenTree::Delimited(span, _) | TokenTree::Sequence(span, _) => span.entire(), + } + } + + fn token(kind: TokenKind, span: Span) -> TokenTree { + TokenTree::Token(Token::new(kind, span)) + } +} diff --git a/src/libsyntax_expand/mbe/macro_check.rs b/src/libsyntax_expand/mbe/macro_check.rs new file mode 100644 index 00000000000..50abda8d45e --- /dev/null +++ b/src/libsyntax_expand/mbe/macro_check.rs @@ -0,0 +1,626 @@ +//! Checks that meta-variables in macro definition are correctly declared and used. +//! +//! # What is checked +//! +//! ## Meta-variables must not be bound twice +//! +//! ``` +//! macro_rules! foo { ($x:tt $x:tt) => { $x }; } +//! ``` +//! +//! This check is sound (no false-negative) and complete (no false-positive). +//! +//! ## Meta-variables must not be free +//! +//! ``` +//! macro_rules! foo { () => { $x }; } +//! ``` +//! +//! This check is also done at macro instantiation but only if the branch is taken. +//! +//! ## Meta-variables must repeat at least as many times as their binder +//! +//! ``` +//! macro_rules! foo { ($($x:tt)*) => { $x }; } +//! ``` +//! +//! This check is also done at macro instantiation but only if the branch is taken. +//! +//! ## Meta-variables must repeat with the same Kleene operators as their binder +//! +//! ``` +//! macro_rules! foo { ($($x:tt)+) => { $($x)* }; } +//! ``` +//! +//! This check is not done at macro instantiation. +//! +//! # Disclaimer +//! +//! In the presence of nested macros (a macro defined in a macro), those checks may have false +//! positives and false negatives. We try to detect those cases by recognizing potential macro +//! definitions in RHSes, but nested macros may be hidden through the use of particular values of +//! meta-variables. +//! +//! ## Examples of false positive +//! +//! False positives can come from cases where we don't recognize a nested macro, because it depends +//! on particular values of meta-variables. In the following example, we think both instances of +//! `$x` are free, which is a correct statement if `$name` is anything but `macro_rules`. But when +//! `$name` is `macro_rules`, like in the instantiation below, then `$x:tt` is actually a binder of +//! the nested macro and `$x` is bound to it. +//! +//! ``` +//! macro_rules! foo { ($name:ident) => { $name! bar { ($x:tt) => { $x }; } }; } +//! foo!(macro_rules); +//! ``` +//! +//! False positives can also come from cases where we think there is a nested macro while there +//! isn't. In the following example, we think `$x` is free, which is incorrect because `bar` is not +//! a nested macro since it is not evaluated as code by `stringify!`. +//! +//! ``` +//! macro_rules! foo { () => { stringify!(macro_rules! bar { () => { $x }; }) }; } +//! ``` +//! +//! ## Examples of false negative +//! +//! False negatives can come from cases where we don't recognize a meta-variable, because it depends +//! on particular values of meta-variables. In the following examples, we don't see that if `$d` is +//! instantiated with `$` then `$d z` becomes `$z` in the nested macro definition and is thus a free +//! meta-variable. Note however, that if `foo` is instantiated, then we would check the definition +//! of `bar` and would see the issue. +//! +//! ``` +//! macro_rules! foo { ($d:tt) => { macro_rules! bar { ($y:tt) => { $d z }; } }; } +//! ``` +//! +//! # How it is checked +//! +//! There are 3 main functions: `check_binders`, `check_occurrences`, and `check_nested_macro`. They +//! all need some kind of environment. +//! +//! ## Environments +//! +//! Environments are used to pass information. +//! +//! ### From LHS to RHS +//! +//! When checking a LHS with `check_binders`, we produce (and use) an environment for binders, +//! namely `Binders`. This is a mapping from binder name to information about that binder: the span +//! of the binder for error messages and the stack of Kleene operators under which it was bound in +//! the LHS. +//! +//! This environment is used by both the LHS and RHS. The LHS uses it to detect duplicate binders. +//! The RHS uses it to detect the other errors. +//! +//! ### From outer macro to inner macro +//! +//! When checking the RHS of an outer macro and we detect a nested macro definition, we push the +//! current state, namely `MacroState`, to an environment of nested macro definitions. Each state +//! stores the LHS binders when entering the macro definition as well as the stack of Kleene +//! operators under which the inner macro is defined in the RHS. +//! +//! This environment is a stack representing the nesting of macro definitions. As such, the stack of +//! Kleene operators under which a meta-variable is repeating is the concatenation of the stacks +//! stored when entering a macro definition starting from the state in which the meta-variable is +//! bound. +use crate::mbe::{KleeneToken, TokenTree}; + +use syntax::ast::NodeId; +use syntax::early_buffered_lints::BufferedEarlyLintId; +use syntax::parse::token::{DelimToken, Token, TokenKind}; +use syntax::sess::ParseSess; +use syntax::symbol::{kw, sym}; + +use rustc_data_structures::fx::FxHashMap; +use smallvec::SmallVec; +use syntax_pos::{symbol::Ident, MultiSpan, Span}; + +/// Stack represented as linked list. +/// +/// Those are used for environments because they grow incrementally and are not mutable. +enum Stack<'a, T> { + /// Empty stack. + Empty, + /// A non-empty stack. + Push { + /// The top element. + top: T, + /// The previous elements. + prev: &'a Stack<'a, T>, + }, +} + +impl<'a, T> Stack<'a, T> { + /// Returns whether a stack is empty. + fn is_empty(&self) -> bool { + match *self { + Stack::Empty => true, + _ => false, + } + } + + /// Returns a new stack with an element of top. + fn push(&'a self, top: T) -> Stack<'a, T> { + Stack::Push { top, prev: self } + } +} + +impl<'a, T> Iterator for &'a Stack<'a, T> { + type Item = &'a T; + + // Iterates from top to bottom of the stack. + fn next(&mut self) -> Option<&'a T> { + match *self { + Stack::Empty => None, + Stack::Push { ref top, ref prev } => { + *self = prev; + Some(top) + } + } + } +} + +impl From<&Stack<'_, KleeneToken>> for SmallVec<[KleeneToken; 1]> { + fn from(ops: &Stack<'_, KleeneToken>) -> SmallVec<[KleeneToken; 1]> { + let mut ops: SmallVec<[KleeneToken; 1]> = ops.cloned().collect(); + // The stack is innermost on top. We want outermost first. + ops.reverse(); + ops + } +} + +/// Information attached to a meta-variable binder in LHS. +struct BinderInfo { + /// The span of the meta-variable in LHS. + span: Span, + /// The stack of Kleene operators (outermost first). + ops: SmallVec<[KleeneToken; 1]>, +} + +/// An environment of meta-variables to their binder information. +type Binders = FxHashMap; + +/// The state at which we entered a macro definition in the RHS of another macro definition. +struct MacroState<'a> { + /// The binders of the branch where we entered the macro definition. + binders: &'a Binders, + /// The stack of Kleene operators (outermost first) where we entered the macro definition. + ops: SmallVec<[KleeneToken; 1]>, +} + +/// Checks that meta-variables are used correctly in a macro definition. +/// +/// Arguments: +/// - `sess` is used to emit diagnostics and lints +/// - `node_id` is used to emit lints +/// - `span` is used when no spans are available +/// - `lhses` and `rhses` should have the same length and represent the macro definition +pub(super) fn check_meta_variables( + sess: &ParseSess, + node_id: NodeId, + span: Span, + lhses: &[TokenTree], + rhses: &[TokenTree], +) -> bool { + if lhses.len() != rhses.len() { + sess.span_diagnostic.span_bug(span, "length mismatch between LHSes and RHSes") + } + let mut valid = true; + for (lhs, rhs) in lhses.iter().zip(rhses.iter()) { + let mut binders = Binders::default(); + check_binders(sess, node_id, lhs, &Stack::Empty, &mut binders, &Stack::Empty, &mut valid); + check_occurrences(sess, node_id, rhs, &Stack::Empty, &binders, &Stack::Empty, &mut valid); + } + valid +} + +/// Checks `lhs` as part of the LHS of a macro definition, extends `binders` with new binders, and +/// sets `valid` to false in case of errors. +/// +/// Arguments: +/// - `sess` is used to emit diagnostics and lints +/// - `node_id` is used to emit lints +/// - `lhs` is checked as part of a LHS +/// - `macros` is the stack of possible outer macros +/// - `binders` contains the binders of the LHS +/// - `ops` is the stack of Kleene operators from the LHS +/// - `valid` is set in case of errors +fn check_binders( + sess: &ParseSess, + node_id: NodeId, + lhs: &TokenTree, + macros: &Stack<'_, MacroState<'_>>, + binders: &mut Binders, + ops: &Stack<'_, KleeneToken>, + valid: &mut bool, +) { + match *lhs { + TokenTree::Token(..) => {} + // This can only happen when checking a nested macro because this LHS is then in the RHS of + // the outer macro. See ui/macros/macro-of-higher-order.rs where $y:$fragment in the + // LHS of the nested macro (and RHS of the outer macro) is parsed as MetaVar(y) Colon + // MetaVar(fragment) and not as MetaVarDecl(y, fragment). + TokenTree::MetaVar(span, name) => { + if macros.is_empty() { + sess.span_diagnostic.span_bug(span, "unexpected MetaVar in lhs"); + } + // There are 3 possibilities: + if let Some(prev_info) = binders.get(&name) { + // 1. The meta-variable is already bound in the current LHS: This is an error. + let mut span = MultiSpan::from_span(span); + span.push_span_label(prev_info.span, "previous declaration".into()); + buffer_lint(sess, span, node_id, "duplicate matcher binding"); + } else if get_binder_info(macros, binders, name).is_none() { + // 2. The meta-variable is free: This is a binder. + binders.insert(name, BinderInfo { span, ops: ops.into() }); + } else { + // 3. The meta-variable is bound: This is an occurrence. + check_occurrences(sess, node_id, lhs, macros, binders, ops, valid); + } + } + // Similarly, this can only happen when checking a toplevel macro. + TokenTree::MetaVarDecl(span, name, _kind) => { + if !macros.is_empty() { + sess.span_diagnostic.span_bug(span, "unexpected MetaVarDecl in nested lhs"); + } + if let Some(prev_info) = get_binder_info(macros, binders, name) { + // Duplicate binders at the top-level macro definition are errors. The lint is only + // for nested macro definitions. + sess.span_diagnostic + .struct_span_err(span, "duplicate matcher binding") + .span_note(prev_info.span, "previous declaration was here") + .emit(); + *valid = false; + } else { + binders.insert(name, BinderInfo { span, ops: ops.into() }); + } + } + TokenTree::Delimited(_, ref del) => { + for tt in &del.tts { + check_binders(sess, node_id, tt, macros, binders, ops, valid); + } + } + TokenTree::Sequence(_, ref seq) => { + let ops = ops.push(seq.kleene); + for tt in &seq.tts { + check_binders(sess, node_id, tt, macros, binders, &ops, valid); + } + } + } +} + +/// Returns the binder information of a meta-variable. +/// +/// Arguments: +/// - `macros` is the stack of possible outer macros +/// - `binders` contains the current binders +/// - `name` is the name of the meta-variable we are looking for +fn get_binder_info<'a>( + mut macros: &'a Stack<'a, MacroState<'a>>, + binders: &'a Binders, + name: Ident, +) -> Option<&'a BinderInfo> { + binders.get(&name).or_else(|| macros.find_map(|state| state.binders.get(&name))) +} + +/// Checks `rhs` as part of the RHS of a macro definition and sets `valid` to false in case of +/// errors. +/// +/// Arguments: +/// - `sess` is used to emit diagnostics and lints +/// - `node_id` is used to emit lints +/// - `rhs` is checked as part of a RHS +/// - `macros` is the stack of possible outer macros +/// - `binders` contains the binders of the associated LHS +/// - `ops` is the stack of Kleene operators from the RHS +/// - `valid` is set in case of errors +fn check_occurrences( + sess: &ParseSess, + node_id: NodeId, + rhs: &TokenTree, + macros: &Stack<'_, MacroState<'_>>, + binders: &Binders, + ops: &Stack<'_, KleeneToken>, + valid: &mut bool, +) { + match *rhs { + TokenTree::Token(..) => {} + TokenTree::MetaVarDecl(span, _name, _kind) => { + sess.span_diagnostic.span_bug(span, "unexpected MetaVarDecl in rhs") + } + TokenTree::MetaVar(span, name) => { + check_ops_is_prefix(sess, node_id, macros, binders, ops, span, name); + } + TokenTree::Delimited(_, ref del) => { + check_nested_occurrences(sess, node_id, &del.tts, macros, binders, ops, valid); + } + TokenTree::Sequence(_, ref seq) => { + let ops = ops.push(seq.kleene); + check_nested_occurrences(sess, node_id, &seq.tts, macros, binders, &ops, valid); + } + } +} + +/// Represents the processed prefix of a nested macro. +#[derive(Clone, Copy, PartialEq, Eq)] +enum NestedMacroState { + /// Nothing that matches a nested macro definition was processed yet. + Empty, + /// The token `macro_rules` was processed. + MacroRules, + /// The tokens `macro_rules!` were processed. + MacroRulesNot, + /// The tokens `macro_rules!` followed by a name were processed. The name may be either directly + /// an identifier or a meta-variable (that hopefully would be instantiated by an identifier). + MacroRulesNotName, + /// The keyword `macro` was processed. + Macro, + /// The keyword `macro` followed by a name was processed. + MacroName, + /// The keyword `macro` followed by a name and a token delimited by parentheses was processed. + MacroNameParen, +} + +/// Checks `tts` as part of the RHS of a macro definition, tries to recognize nested macro +/// definitions, and sets `valid` to false in case of errors. +/// +/// Arguments: +/// - `sess` is used to emit diagnostics and lints +/// - `node_id` is used to emit lints +/// - `tts` is checked as part of a RHS and may contain macro definitions +/// - `macros` is the stack of possible outer macros +/// - `binders` contains the binders of the associated LHS +/// - `ops` is the stack of Kleene operators from the RHS +/// - `valid` is set in case of errors +fn check_nested_occurrences( + sess: &ParseSess, + node_id: NodeId, + tts: &[TokenTree], + macros: &Stack<'_, MacroState<'_>>, + binders: &Binders, + ops: &Stack<'_, KleeneToken>, + valid: &mut bool, +) { + let mut state = NestedMacroState::Empty; + let nested_macros = macros.push(MacroState { binders, ops: ops.into() }); + let mut nested_binders = Binders::default(); + for tt in tts { + match (state, tt) { + ( + NestedMacroState::Empty, + &TokenTree::Token(Token { kind: TokenKind::Ident(name, false), .. }), + ) => { + if name == sym::macro_rules { + state = NestedMacroState::MacroRules; + } else if name == kw::Macro { + state = NestedMacroState::Macro; + } + } + ( + NestedMacroState::MacroRules, + &TokenTree::Token(Token { kind: TokenKind::Not, .. }), + ) => { + state = NestedMacroState::MacroRulesNot; + } + ( + NestedMacroState::MacroRulesNot, + &TokenTree::Token(Token { kind: TokenKind::Ident(..), .. }), + ) => { + state = NestedMacroState::MacroRulesNotName; + } + (NestedMacroState::MacroRulesNot, &TokenTree::MetaVar(..)) => { + state = NestedMacroState::MacroRulesNotName; + // We check that the meta-variable is correctly used. + check_occurrences(sess, node_id, tt, macros, binders, ops, valid); + } + (NestedMacroState::MacroRulesNotName, &TokenTree::Delimited(_, ref del)) + | (NestedMacroState::MacroName, &TokenTree::Delimited(_, ref del)) + if del.delim == DelimToken::Brace => + { + let legacy = state == NestedMacroState::MacroRulesNotName; + state = NestedMacroState::Empty; + let rest = + check_nested_macro(sess, node_id, legacy, &del.tts, &nested_macros, valid); + // If we did not check the whole macro definition, then check the rest as if outside + // the macro definition. + check_nested_occurrences( + sess, + node_id, + &del.tts[rest..], + macros, + binders, + ops, + valid, + ); + } + ( + NestedMacroState::Macro, + &TokenTree::Token(Token { kind: TokenKind::Ident(..), .. }), + ) => { + state = NestedMacroState::MacroName; + } + (NestedMacroState::Macro, &TokenTree::MetaVar(..)) => { + state = NestedMacroState::MacroName; + // We check that the meta-variable is correctly used. + check_occurrences(sess, node_id, tt, macros, binders, ops, valid); + } + (NestedMacroState::MacroName, &TokenTree::Delimited(_, ref del)) + if del.delim == DelimToken::Paren => + { + state = NestedMacroState::MacroNameParen; + nested_binders = Binders::default(); + check_binders( + sess, + node_id, + tt, + &nested_macros, + &mut nested_binders, + &Stack::Empty, + valid, + ); + } + (NestedMacroState::MacroNameParen, &TokenTree::Delimited(_, ref del)) + if del.delim == DelimToken::Brace => + { + state = NestedMacroState::Empty; + check_occurrences( + sess, + node_id, + tt, + &nested_macros, + &nested_binders, + &Stack::Empty, + valid, + ); + } + (_, ref tt) => { + state = NestedMacroState::Empty; + check_occurrences(sess, node_id, tt, macros, binders, ops, valid); + } + } + } +} + +/// Checks the body of nested macro, returns where the check stopped, and sets `valid` to false in +/// case of errors. +/// +/// The token trees are checked as long as they look like a list of (LHS) => {RHS} token trees. This +/// check is a best-effort to detect a macro definition. It returns the position in `tts` where we +/// stopped checking because we detected we were not in a macro definition anymore. +/// +/// Arguments: +/// - `sess` is used to emit diagnostics and lints +/// - `node_id` is used to emit lints +/// - `legacy` specifies whether the macro is legacy +/// - `tts` is checked as a list of (LHS) => {RHS} +/// - `macros` is the stack of outer macros +/// - `valid` is set in case of errors +fn check_nested_macro( + sess: &ParseSess, + node_id: NodeId, + legacy: bool, + tts: &[TokenTree], + macros: &Stack<'_, MacroState<'_>>, + valid: &mut bool, +) -> usize { + let n = tts.len(); + let mut i = 0; + let separator = if legacy { TokenKind::Semi } else { TokenKind::Comma }; + loop { + // We expect 3 token trees: `(LHS) => {RHS}`. The separator is checked after. + if i + 2 >= n + || !tts[i].is_delimited() + || !tts[i + 1].is_token(&TokenKind::FatArrow) + || !tts[i + 2].is_delimited() + { + break; + } + let lhs = &tts[i]; + let rhs = &tts[i + 2]; + let mut binders = Binders::default(); + check_binders(sess, node_id, lhs, macros, &mut binders, &Stack::Empty, valid); + check_occurrences(sess, node_id, rhs, macros, &binders, &Stack::Empty, valid); + // Since the last semicolon is optional for legacy macros and decl_macro are not terminated, + // we increment our checked position by how many token trees we already checked (the 3 + // above) before checking for the separator. + i += 3; + if i == n || !tts[i].is_token(&separator) { + break; + } + // We increment our checked position for the semicolon. + i += 1; + } + i +} + +/// Checks that a meta-variable occurrence is valid. +/// +/// Arguments: +/// - `sess` is used to emit diagnostics and lints +/// - `node_id` is used to emit lints +/// - `macros` is the stack of possible outer macros +/// - `binders` contains the binders of the associated LHS +/// - `ops` is the stack of Kleene operators from the RHS +/// - `span` is the span of the meta-variable to check +/// - `name` is the name of the meta-variable to check +fn check_ops_is_prefix( + sess: &ParseSess, + node_id: NodeId, + macros: &Stack<'_, MacroState<'_>>, + binders: &Binders, + ops: &Stack<'_, KleeneToken>, + span: Span, + name: Ident, +) { + let macros = macros.push(MacroState { binders, ops: ops.into() }); + // Accumulates the stacks the operators of each state until (and including when) the + // meta-variable is found. The innermost stack is first. + let mut acc: SmallVec<[&SmallVec<[KleeneToken; 1]>; 1]> = SmallVec::new(); + for state in ¯os { + acc.push(&state.ops); + if let Some(binder) = state.binders.get(&name) { + // This variable concatenates the stack of operators from the RHS of the LHS where the + // meta-variable was defined to where it is used (in possibly nested macros). The + // outermost operator is first. + let mut occurrence_ops: SmallVec<[KleeneToken; 2]> = SmallVec::new(); + // We need to iterate from the end to start with outermost stack. + for ops in acc.iter().rev() { + occurrence_ops.extend_from_slice(ops); + } + ops_is_prefix(sess, node_id, span, name, &binder.ops, &occurrence_ops); + return; + } + } + buffer_lint(sess, span.into(), node_id, &format!("unknown macro variable `{}`", name)); +} + +/// Returns whether `binder_ops` is a prefix of `occurrence_ops`. +/// +/// The stack of Kleene operators of a meta-variable occurrence just needs to have the stack of +/// Kleene operators of its binder as a prefix. +/// +/// Consider $i in the following example: +/// +/// ( $( $i:ident = $($j:ident),+ );* ) => { $($( $i += $j; )+)* } +/// +/// It occurs under the Kleene stack ["*", "+"] and is bound under ["*"] only. +/// +/// Arguments: +/// - `sess` is used to emit diagnostics and lints +/// - `node_id` is used to emit lints +/// - `span` is the span of the meta-variable being check +/// - `name` is the name of the meta-variable being check +/// - `binder_ops` is the stack of Kleene operators for the binder +/// - `occurrence_ops` is the stack of Kleene operators for the occurrence +fn ops_is_prefix( + sess: &ParseSess, + node_id: NodeId, + span: Span, + name: Ident, + binder_ops: &[KleeneToken], + occurrence_ops: &[KleeneToken], +) { + for (i, binder) in binder_ops.iter().enumerate() { + if i >= occurrence_ops.len() { + let mut span = MultiSpan::from_span(span); + span.push_span_label(binder.span, "expected repetition".into()); + let message = &format!("variable '{}' is still repeating at this depth", name); + buffer_lint(sess, span, node_id, message); + return; + } + let occurrence = &occurrence_ops[i]; + if occurrence.op != binder.op { + let mut span = MultiSpan::from_span(span); + span.push_span_label(binder.span, "expected repetition".into()); + span.push_span_label(occurrence.span, "conflicting repetition".into()); + let message = "meta-variable repeats with different Kleene operator"; + buffer_lint(sess, span, node_id, message); + return; + } + } +} + +fn buffer_lint(sess: &ParseSess, span: MultiSpan, node_id: NodeId, message: &str) { + sess.buffer_lint(BufferedEarlyLintId::MetaVariableMisuse, span, node_id, message); +} diff --git a/src/libsyntax_expand/mbe/macro_parser.rs b/src/libsyntax_expand/mbe/macro_parser.rs new file mode 100644 index 00000000000..2edb4925512 --- /dev/null +++ b/src/libsyntax_expand/mbe/macro_parser.rs @@ -0,0 +1,944 @@ +//! This is an NFA-based parser, which calls out to the main rust parser for named non-terminals +//! (which it commits to fully when it hits one in a grammar). There's a set of current NFA threads +//! and a set of next ones. Instead of NTs, we have a special case for Kleene star. The big-O, in +//! pathological cases, is worse than traditional use of NFA or Earley parsing, but it's an easier +//! fit for Macro-by-Example-style rules. +//! +//! (In order to prevent the pathological case, we'd need to lazily construct the resulting +//! `NamedMatch`es at the very end. It'd be a pain, and require more memory to keep around old +//! items, but it would also save overhead) +//! +//! We don't say this parser uses the Earley algorithm, because it's unnecessarily inaccurate. +//! The macro parser restricts itself to the features of finite state automata. Earley parsers +//! can be described as an extension of NFAs with completion rules, prediction rules, and recursion. +//! +//! Quick intro to how the parser works: +//! +//! A 'position' is a dot in the middle of a matcher, usually represented as a +//! dot. For example `· a $( a )* a b` is a position, as is `a $( · a )* a b`. +//! +//! The parser walks through the input a character at a time, maintaining a list +//! of threads consistent with the current position in the input string: `cur_items`. +//! +//! As it processes them, it fills up `eof_items` with threads that would be valid if +//! the macro invocation is now over, `bb_items` with threads that are waiting on +//! a Rust non-terminal like `$e:expr`, and `next_items` with threads that are waiting +//! on a particular token. Most of the logic concerns moving the · through the +//! repetitions indicated by Kleene stars. The rules for moving the · without +//! consuming any input are called epsilon transitions. It only advances or calls +//! out to the real Rust parser when no `cur_items` threads remain. +//! +//! Example: +//! +//! ```text, ignore +//! Start parsing a a a a b against [· a $( a )* a b]. +//! +//! Remaining input: a a a a b +//! next: [· a $( a )* a b] +//! +//! - - - Advance over an a. - - - +//! +//! Remaining input: a a a b +//! cur: [a · $( a )* a b] +//! Descend/Skip (first item). +//! next: [a $( · a )* a b] [a $( a )* · a b]. +//! +//! - - - Advance over an a. - - - +//! +//! Remaining input: a a b +//! cur: [a $( a · )* a b] [a $( a )* a · b] +//! Follow epsilon transition: Finish/Repeat (first item) +//! next: [a $( a )* · a b] [a $( · a )* a b] [a $( a )* a · b] +//! +//! - - - Advance over an a. - - - (this looks exactly like the last step) +//! +//! Remaining input: a b +//! cur: [a $( a · )* a b] [a $( a )* a · b] +//! Follow epsilon transition: Finish/Repeat (first item) +//! next: [a $( a )* · a b] [a $( · a )* a b] [a $( a )* a · b] +//! +//! - - - Advance over an a. - - - (this looks exactly like the last step) +//! +//! Remaining input: b +//! cur: [a $( a · )* a b] [a $( a )* a · b] +//! Follow epsilon transition: Finish/Repeat (first item) +//! next: [a $( a )* · a b] [a $( · a )* a b] [a $( a )* a · b] +//! +//! - - - Advance over a b. - - - +//! +//! Remaining input: '' +//! eof: [a $( a )* a b ·] +//! ``` + +crate use NamedMatch::*; +crate use ParseResult::*; +use TokenTreeOrTokenTreeSlice::*; + +use crate::mbe::{self, TokenTree}; + +use syntax::ast::{Ident, Name}; +use syntax::parse::{Directory, PResult}; +use syntax::parse::parser::{Parser, PathStyle}; +use syntax::parse::token::{self, DocComment, Nonterminal, Token}; +use syntax::print::pprust; +use syntax::sess::ParseSess; +use syntax::symbol::{kw, sym, Symbol}; +use syntax::tokenstream::{DelimSpan, TokenStream}; + +use errors::FatalError; +use smallvec::{smallvec, SmallVec}; +use syntax_pos::Span; + +use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::sync::Lrc; +use std::collections::hash_map::Entry::{Occupied, Vacant}; +use std::mem; +use std::ops::{Deref, DerefMut}; + +// To avoid costly uniqueness checks, we require that `MatchSeq` always has a nonempty body. + +/// Either a sequence of token trees or a single one. This is used as the representation of the +/// sequence of tokens that make up a matcher. +#[derive(Clone)] +enum TokenTreeOrTokenTreeSlice<'tt> { + Tt(TokenTree), + TtSeq(&'tt [TokenTree]), +} + +impl<'tt> TokenTreeOrTokenTreeSlice<'tt> { + /// Returns the number of constituent top-level token trees of `self` (top-level in that it + /// will not recursively descend into subtrees). + fn len(&self) -> usize { + match *self { + TtSeq(ref v) => v.len(), + Tt(ref tt) => tt.len(), + } + } + + /// The `index`-th token tree of `self`. + fn get_tt(&self, index: usize) -> TokenTree { + match *self { + TtSeq(ref v) => v[index].clone(), + Tt(ref tt) => tt.get_tt(index), + } + } +} + +/// An unzipping of `TokenTree`s... see the `stack` field of `MatcherPos`. +/// +/// This is used by `inner_parse_loop` to keep track of delimited submatchers that we have +/// descended into. +#[derive(Clone)] +struct MatcherTtFrame<'tt> { + /// The "parent" matcher that we are descending into. + elts: TokenTreeOrTokenTreeSlice<'tt>, + /// The position of the "dot" in `elts` at the time we descended. + idx: usize, +} + +type NamedMatchVec = SmallVec<[NamedMatch; 4]>; + +/// Represents a single "position" (aka "matcher position", aka "item"), as +/// described in the module documentation. +/// +/// Here: +/// +/// - `'root` represents the lifetime of the stack slot that holds the root +/// `MatcherPos`. As described in `MatcherPosHandle`, the root `MatcherPos` +/// structure is stored on the stack, but subsequent instances are put into +/// the heap. +/// - `'tt` represents the lifetime of the token trees that this matcher +/// position refers to. +/// +/// It is important to distinguish these two lifetimes because we have a +/// `SmallVec>` below, and the destructor of +/// that is considered to possibly access the data from its elements (it lacks +/// a `#[may_dangle]` attribute). As a result, the compiler needs to know that +/// all the elements in that `SmallVec` strictly outlive the root stack slot +/// lifetime. By separating `'tt` from `'root`, we can show that. +#[derive(Clone)] +struct MatcherPos<'root, 'tt> { + /// The token or sequence of tokens that make up the matcher + top_elts: TokenTreeOrTokenTreeSlice<'tt>, + + /// The position of the "dot" in this matcher + idx: usize, + + /// The first span of source that the beginning of this matcher corresponds to. In other + /// words, the token in the source whose span is `sp_open` is matched against the first token of + /// the matcher. + sp_open: Span, + + /// For each named metavar in the matcher, we keep track of token trees matched against the + /// metavar by the black box parser. In particular, there may be more than one match per + /// metavar if we are in a repetition (each repetition matches each of the variables). + /// Moreover, matchers and repetitions can be nested; the `matches` field is shared (hence the + /// `Rc`) among all "nested" matchers. `match_lo`, `match_cur`, and `match_hi` keep track of + /// the current position of the `self` matcher position in the shared `matches` list. + /// + /// Also, note that while we are descending into a sequence, matchers are given their own + /// `matches` vector. Only once we reach the end of a full repetition of the sequence do we add + /// all bound matches from the submatcher into the shared top-level `matches` vector. If `sep` + /// and `up` are `Some`, then `matches` is _not_ the shared top-level list. Instead, if one + /// wants the shared `matches`, one should use `up.matches`. + matches: Box<[Lrc]>, + /// The position in `matches` corresponding to the first metavar in this matcher's sequence of + /// token trees. In other words, the first metavar in the first token of `top_elts` corresponds + /// to `matches[match_lo]`. + match_lo: usize, + /// The position in `matches` corresponding to the metavar we are currently trying to match + /// against the source token stream. `match_lo <= match_cur <= match_hi`. + match_cur: usize, + /// Similar to `match_lo` except `match_hi` is the position in `matches` of the _last_ metavar + /// in this matcher. + match_hi: usize, + + // The following fields are used if we are matching a repetition. If we aren't, they should be + // `None`. + + /// The KleeneOp of this sequence if we are in a repetition. + seq_op: Option, + + /// The separator if we are in a repetition. + sep: Option, + + /// The "parent" matcher position if we are in a repetition. That is, the matcher position just + /// before we enter the sequence. + up: Option>, + + /// Specifically used to "unzip" token trees. By "unzip", we mean to unwrap the delimiters from + /// a delimited token tree (e.g., something wrapped in `(` `)`) or to get the contents of a doc + /// comment... + /// + /// When matching against matchers with nested delimited submatchers (e.g., `pat ( pat ( .. ) + /// pat ) pat`), we need to keep track of the matchers we are descending into. This stack does + /// that where the bottom of the stack is the outermost matcher. + /// Also, throughout the comments, this "descent" is often referred to as "unzipping"... + stack: SmallVec<[MatcherTtFrame<'tt>; 1]>, +} + +impl<'root, 'tt> MatcherPos<'root, 'tt> { + /// Adds `m` as a named match for the `idx`-th metavar. + fn push_match(&mut self, idx: usize, m: NamedMatch) { + let matches = Lrc::make_mut(&mut self.matches[idx]); + matches.push(m); + } +} + +// Lots of MatcherPos instances are created at runtime. Allocating them on the +// heap is slow. Furthermore, using SmallVec to allocate them all +// on the stack is also slow, because MatcherPos is quite a large type and +// instances get moved around a lot between vectors, which requires lots of +// slow memcpy calls. +// +// Therefore, the initial MatcherPos is always allocated on the stack, +// subsequent ones (of which there aren't that many) are allocated on the heap, +// and this type is used to encapsulate both cases. +enum MatcherPosHandle<'root, 'tt> { + Ref(&'root mut MatcherPos<'root, 'tt>), + Box(Box>), +} + +impl<'root, 'tt> Clone for MatcherPosHandle<'root, 'tt> { + // This always produces a new Box. + fn clone(&self) -> Self { + MatcherPosHandle::Box(match *self { + MatcherPosHandle::Ref(ref r) => Box::new((**r).clone()), + MatcherPosHandle::Box(ref b) => b.clone(), + }) + } +} + +impl<'root, 'tt> Deref for MatcherPosHandle<'root, 'tt> { + type Target = MatcherPos<'root, 'tt>; + fn deref(&self) -> &Self::Target { + match *self { + MatcherPosHandle::Ref(ref r) => r, + MatcherPosHandle::Box(ref b) => b, + } + } +} + +impl<'root, 'tt> DerefMut for MatcherPosHandle<'root, 'tt> { + fn deref_mut(&mut self) -> &mut MatcherPos<'root, 'tt> { + match *self { + MatcherPosHandle::Ref(ref mut r) => r, + MatcherPosHandle::Box(ref mut b) => b, + } + } +} + +/// Represents the possible results of an attempted parse. +crate enum ParseResult { + /// Parsed successfully. + Success(T), + /// Arm failed to match. If the second parameter is `token::Eof`, it indicates an unexpected + /// end of macro invocation. Otherwise, it indicates that no rules expected the given token. + Failure(Token, &'static str), + /// Fatal error (malformed macro?). Abort compilation. + Error(syntax_pos::Span, String), +} + +/// A `ParseResult` where the `Success` variant contains a mapping of `Ident`s to `NamedMatch`es. +/// This represents the mapping of metavars to the token trees they bind to. +crate type NamedParseResult = ParseResult>; + +/// Count how many metavars are named in the given matcher `ms`. +pub(super) fn count_names(ms: &[TokenTree]) -> usize { + ms.iter().fold(0, |count, elt| { + count + match *elt { + TokenTree::Sequence(_, ref seq) => seq.num_captures, + TokenTree::Delimited(_, ref delim) => count_names(&delim.tts), + TokenTree::MetaVar(..) => 0, + TokenTree::MetaVarDecl(..) => 1, + TokenTree::Token(..) => 0, + } + }) +} + +/// `len` `Vec`s (initially shared and empty) that will store matches of metavars. +fn create_matches(len: usize) -> Box<[Lrc]> { + if len == 0 { + vec![] + } else { + let empty_matches = Lrc::new(SmallVec::new()); + vec![empty_matches; len] + }.into_boxed_slice() +} + +/// Generates the top-level matcher position in which the "dot" is before the first token of the +/// matcher `ms` and we are going to start matching at the span `open` in the source. +fn initial_matcher_pos<'root, 'tt>(ms: &'tt [TokenTree], open: Span) -> MatcherPos<'root, 'tt> { + let match_idx_hi = count_names(ms); + let matches = create_matches(match_idx_hi); + MatcherPos { + // Start with the top level matcher given to us + top_elts: TtSeq(ms), // "elts" is an abbr. for "elements" + // The "dot" is before the first token of the matcher + idx: 0, + // We start matching at the span `open` in the source code + sp_open: open, + + // Initialize `matches` to a bunch of empty `Vec`s -- one for each metavar in `top_elts`. + // `match_lo` for `top_elts` is 0 and `match_hi` is `matches.len()`. `match_cur` is 0 since + // we haven't actually matched anything yet. + matches, + match_lo: 0, + match_cur: 0, + match_hi: match_idx_hi, + + // Haven't descended into any delimiters, so empty stack + stack: smallvec![], + + // Haven't descended into any sequences, so both of these are `None`. + seq_op: None, + sep: None, + up: None, + } +} + +/// `NamedMatch` is a pattern-match result for a single `token::MATCH_NONTERMINAL`: +/// so it is associated with a single ident in a parse, and all +/// `MatchedNonterminal`s in the `NamedMatch` have the same non-terminal type +/// (expr, item, etc). Each leaf in a single `NamedMatch` corresponds to a +/// single `token::MATCH_NONTERMINAL` in the `TokenTree` that produced it. +/// +/// The in-memory structure of a particular `NamedMatch` represents the match +/// that occurred when a particular subset of a matcher was applied to a +/// particular token tree. +/// +/// The width of each `MatchedSeq` in the `NamedMatch`, and the identity of +/// the `MatchedNonterminal`s, will depend on the token tree it was applied +/// to: each `MatchedSeq` corresponds to a single `TTSeq` in the originating +/// token tree. The depth of the `NamedMatch` structure will therefore depend +/// only on the nesting depth of `ast::TTSeq`s in the originating +/// token tree it was derived from. +#[derive(Debug, Clone)] +crate enum NamedMatch { + MatchedSeq(Lrc, DelimSpan), + MatchedNonterminal(Lrc), +} + +/// Takes a sequence of token trees `ms` representing a matcher which successfully matched input +/// and an iterator of items that matched input and produces a `NamedParseResult`. +fn nameize>( + sess: &ParseSess, + ms: &[TokenTree], + mut res: I, +) -> NamedParseResult { + // Recursively descend into each type of matcher (e.g., sequences, delimited, metavars) and make + // sure that each metavar has _exactly one_ binding. If a metavar does not have exactly one + // binding, then there is an error. If it does, then we insert the binding into the + // `NamedParseResult`. + fn n_rec>( + sess: &ParseSess, + m: &TokenTree, + res: &mut I, + ret_val: &mut FxHashMap, + ) -> Result<(), (syntax_pos::Span, String)> { + match *m { + TokenTree::Sequence(_, ref seq) => for next_m in &seq.tts { + n_rec(sess, next_m, res.by_ref(), ret_val)? + }, + TokenTree::Delimited(_, ref delim) => for next_m in &delim.tts { + n_rec(sess, next_m, res.by_ref(), ret_val)?; + }, + TokenTree::MetaVarDecl(span, _, id) if id.name == kw::Invalid => { + if sess.missing_fragment_specifiers.borrow_mut().remove(&span) { + return Err((span, "missing fragment specifier".to_string())); + } + } + TokenTree::MetaVarDecl(sp, bind_name, _) => { + match ret_val.entry(bind_name) { + Vacant(spot) => { + spot.insert(res.next().unwrap()); + } + Occupied(..) => { + return Err((sp, format!("duplicated bind name: {}", bind_name))) + } + } + } + TokenTree::MetaVar(..) | TokenTree::Token(..) => (), + } + + Ok(()) + } + + let mut ret_val = FxHashMap::default(); + for m in ms { + match n_rec(sess, m, res.by_ref(), &mut ret_val) { + Ok(_) => {} + Err((sp, msg)) => return Error(sp, msg), + } + } + + Success(ret_val) +} + +/// Performs a token equality check, ignoring syntax context (that is, an unhygienic comparison) +fn token_name_eq(t1: &Token, t2: &Token) -> bool { + if let (Some((ident1, is_raw1)), Some((ident2, is_raw2))) = (t1.ident(), t2.ident()) { + ident1.name == ident2.name && is_raw1 == is_raw2 + } else if let (Some(ident1), Some(ident2)) = (t1.lifetime(), t2.lifetime()) { + ident1.name == ident2.name + } else { + t1.kind == t2.kind + } +} + +/// Process the matcher positions of `cur_items` until it is empty. In the process, this will +/// produce more items in `next_items`, `eof_items`, and `bb_items`. +/// +/// For more info about the how this happens, see the module-level doc comments and the inline +/// comments of this function. +/// +/// # Parameters +/// +/// - `sess`: the parsing session into which errors are emitted. +/// - `cur_items`: the set of current items to be processed. This should be empty by the end of a +/// successful execution of this function. +/// - `next_items`: the set of newly generated items. These are used to replenish `cur_items` in +/// the function `parse`. +/// - `eof_items`: the set of items that would be valid if this was the EOF. +/// - `bb_items`: the set of items that are waiting for the black-box parser. +/// - `token`: the current token of the parser. +/// - `span`: the `Span` in the source code corresponding to the token trees we are trying to match +/// against the matcher positions in `cur_items`. +/// +/// # Returns +/// +/// A `ParseResult`. Note that matches are kept track of through the items generated. +fn inner_parse_loop<'root, 'tt>( + sess: &ParseSess, + cur_items: &mut SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>, + next_items: &mut Vec>, + eof_items: &mut SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>, + bb_items: &mut SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>, + token: &Token, +) -> ParseResult<()> { + // Pop items from `cur_items` until it is empty. + while let Some(mut item) = cur_items.pop() { + // When unzipped trees end, remove them. This corresponds to backtracking out of a + // delimited submatcher into which we already descended. In backtracking out again, we need + // to advance the "dot" past the delimiters in the outer matcher. + while item.idx >= item.top_elts.len() { + match item.stack.pop() { + Some(MatcherTtFrame { elts, idx }) => { + item.top_elts = elts; + item.idx = idx + 1; + } + None => break, + } + } + + // Get the current position of the "dot" (`idx`) in `item` and the number of token trees in + // the matcher (`len`). + let idx = item.idx; + let len = item.top_elts.len(); + + // If `idx >= len`, then we are at or past the end of the matcher of `item`. + if idx >= len { + // We are repeating iff there is a parent. If the matcher is inside of a repetition, + // then we could be at the end of a sequence or at the beginning of the next + // repetition. + if item.up.is_some() { + // At this point, regardless of whether there is a separator, we should add all + // matches from the complete repetition of the sequence to the shared, top-level + // `matches` list (actually, `up.matches`, which could itself not be the top-level, + // but anyway...). Moreover, we add another item to `cur_items` in which the "dot" + // is at the end of the `up` matcher. This ensures that the "dot" in the `up` + // matcher is also advanced sufficiently. + // + // NOTE: removing the condition `idx == len` allows trailing separators. + if idx == len { + // Get the `up` matcher + let mut new_pos = item.up.clone().unwrap(); + + // Add matches from this repetition to the `matches` of `up` + for idx in item.match_lo..item.match_hi { + let sub = item.matches[idx].clone(); + let span = DelimSpan::from_pair(item.sp_open, token.span); + new_pos.push_match(idx, MatchedSeq(sub, span)); + } + + // Move the "dot" past the repetition in `up` + new_pos.match_cur = item.match_hi; + new_pos.idx += 1; + cur_items.push(new_pos); + } + + // Check if we need a separator. + if idx == len && item.sep.is_some() { + // We have a separator, and it is the current token. We can advance past the + // separator token. + if item.sep + .as_ref() + .map(|sep| token_name_eq(token, sep)) + .unwrap_or(false) + { + item.idx += 1; + next_items.push(item); + } + } + // We don't need a separator. Move the "dot" back to the beginning of the matcher + // and try to match again UNLESS we are only allowed to have _one_ repetition. + else if item.seq_op != Some(mbe::KleeneOp::ZeroOrOne) { + item.match_cur = item.match_lo; + item.idx = 0; + cur_items.push(item); + } + } + // If we are not in a repetition, then being at the end of a matcher means that we have + // reached the potential end of the input. + else { + eof_items.push(item); + } + } + // We are in the middle of a matcher. + else { + // Look at what token in the matcher we are trying to match the current token (`token`) + // against. Depending on that, we may generate new items. + match item.top_elts.get_tt(idx) { + // Need to descend into a sequence + TokenTree::Sequence(sp, seq) => { + // Examine the case where there are 0 matches of this sequence. We are + // implicitly disallowing OneOrMore from having 0 matches here. Thus, that will + // result in a "no rules expected token" error by virtue of this matcher not + // working. + if seq.kleene.op == mbe::KleeneOp::ZeroOrMore + || seq.kleene.op == mbe::KleeneOp::ZeroOrOne + { + let mut new_item = item.clone(); + new_item.match_cur += seq.num_captures; + new_item.idx += 1; + for idx in item.match_cur..item.match_cur + seq.num_captures { + new_item.push_match(idx, MatchedSeq(Lrc::new(smallvec![]), sp)); + } + cur_items.push(new_item); + } + + let matches = create_matches(item.matches.len()); + cur_items.push(MatcherPosHandle::Box(Box::new(MatcherPos { + stack: smallvec![], + sep: seq.separator.clone(), + seq_op: Some(seq.kleene.op), + idx: 0, + matches, + match_lo: item.match_cur, + match_cur: item.match_cur, + match_hi: item.match_cur + seq.num_captures, + up: Some(item), + sp_open: sp.open, + top_elts: Tt(TokenTree::Sequence(sp, seq)), + }))); + } + + // We need to match a metavar (but the identifier is invalid)... this is an error + TokenTree::MetaVarDecl(span, _, id) if id.name == kw::Invalid => { + if sess.missing_fragment_specifiers.borrow_mut().remove(&span) { + return Error(span, "missing fragment specifier".to_string()); + } + } + + // We need to match a metavar with a valid ident... call out to the black-box + // parser by adding an item to `bb_items`. + TokenTree::MetaVarDecl(_, _, id) => { + // Built-in nonterminals never start with these tokens, + // so we can eliminate them from consideration. + if may_begin_with(token, id.name) { + bb_items.push(item); + } + } + + // We need to descend into a delimited submatcher or a doc comment. To do this, we + // push the current matcher onto a stack and push a new item containing the + // submatcher onto `cur_items`. + // + // At the beginning of the loop, if we reach the end of the delimited submatcher, + // we pop the stack to backtrack out of the descent. + seq @ TokenTree::Delimited(..) | + seq @ TokenTree::Token(Token { kind: DocComment(..), .. }) => { + let lower_elts = mem::replace(&mut item.top_elts, Tt(seq)); + let idx = item.idx; + item.stack.push(MatcherTtFrame { + elts: lower_elts, + idx, + }); + item.idx = 0; + cur_items.push(item); + } + + // We just matched a normal token. We can just advance the parser. + TokenTree::Token(t) if token_name_eq(&t, token) => { + item.idx += 1; + next_items.push(item); + } + + // There was another token that was not `token`... This means we can't add any + // rules. NOTE that this is not necessarily an error unless _all_ items in + // `cur_items` end up doing this. There may still be some other matchers that do + // end up working out. + TokenTree::Token(..) | TokenTree::MetaVar(..) => {} + } + } + } + + // Yay a successful parse (so far)! + Success(()) +} + +/// Use the given sequence of token trees (`ms`) as a matcher. Match the given token stream `tts` +/// against it and return the match. +/// +/// # Parameters +/// +/// - `sess`: The session into which errors are emitted +/// - `tts`: The tokenstream we are matching against the pattern `ms` +/// - `ms`: A sequence of token trees representing a pattern against which we are matching +/// - `directory`: Information about the file locations (needed for the black-box parser) +/// - `recurse_into_modules`: Whether or not to recurse into modules (needed for the black-box +/// parser) +pub(super) fn parse( + sess: &ParseSess, + tts: TokenStream, + ms: &[TokenTree], + directory: Option>, + recurse_into_modules: bool, +) -> NamedParseResult { + // Create a parser that can be used for the "black box" parts. + let mut parser = Parser::new( + sess, + tts, + directory, + recurse_into_modules, + true, + syntax::MACRO_ARGUMENTS, + ); + + // A queue of possible matcher positions. We initialize it with the matcher position in which + // the "dot" is before the first token of the first token tree in `ms`. `inner_parse_loop` then + // processes all of these possible matcher positions and produces possible next positions into + // `next_items`. After some post-processing, the contents of `next_items` replenish `cur_items` + // and we start over again. + // + // This MatcherPos instance is allocated on the stack. All others -- and + // there are frequently *no* others! -- are allocated on the heap. + let mut initial = initial_matcher_pos(ms, parser.token.span); + let mut cur_items = smallvec![MatcherPosHandle::Ref(&mut initial)]; + let mut next_items = Vec::new(); + + loop { + // Matcher positions black-box parsed by parser.rs (`parser`) + let mut bb_items = SmallVec::new(); + + // Matcher positions that would be valid if the macro invocation was over now + let mut eof_items = SmallVec::new(); + assert!(next_items.is_empty()); + + // Process `cur_items` until either we have finished the input or we need to get some + // parsing from the black-box parser done. The result is that `next_items` will contain a + // bunch of possible next matcher positions in `next_items`. + match inner_parse_loop( + sess, + &mut cur_items, + &mut next_items, + &mut eof_items, + &mut bb_items, + &parser.token, + ) { + Success(_) => {} + Failure(token, msg) => return Failure(token, msg), + Error(sp, msg) => return Error(sp, msg), + } + + // inner parse loop handled all cur_items, so it's empty + assert!(cur_items.is_empty()); + + // We need to do some post processing after the `inner_parser_loop`. + // + // Error messages here could be improved with links to original rules. + + // If we reached the EOF, check that there is EXACTLY ONE possible matcher. Otherwise, + // either the parse is ambiguous (which should never happen) or there is a syntax error. + if parser.token == token::Eof { + if eof_items.len() == 1 { + let matches = eof_items[0] + .matches + .iter_mut() + .map(|dv| Lrc::make_mut(dv).pop().unwrap()); + return nameize(sess, ms, matches); + } else if eof_items.len() > 1 { + return Error( + parser.token.span, + "ambiguity: multiple successful parses".to_string(), + ); + } else { + return Failure( + Token::new(token::Eof, if parser.token.span.is_dummy() { + parser.token.span + } else { + sess.source_map().next_point(parser.token.span) + }), + "missing tokens in macro arguments", + ); + } + } + // Performance hack: eof_items may share matchers via Rc with other things that we want + // to modify. Dropping eof_items now may drop these refcounts to 1, preventing an + // unnecessary implicit clone later in Rc::make_mut. + drop(eof_items); + + // Another possibility is that we need to call out to parse some rust nonterminal + // (black-box) parser. However, if there is not EXACTLY ONE of these, something is wrong. + if (!bb_items.is_empty() && !next_items.is_empty()) || bb_items.len() > 1 { + let nts = bb_items + .iter() + .map(|item| match item.top_elts.get_tt(item.idx) { + TokenTree::MetaVarDecl(_, bind, name) => format!("{} ('{}')", name, bind), + _ => panic!(), + }) + .collect::>() + .join(" or "); + + return Error( + parser.token.span, + format!( + "local ambiguity: multiple parsing options: {}", + match next_items.len() { + 0 => format!("built-in NTs {}.", nts), + 1 => format!("built-in NTs {} or 1 other option.", nts), + n => format!("built-in NTs {} or {} other options.", nts, n), + } + ), + ); + } + // If there are no possible next positions AND we aren't waiting for the black-box parser, + // then there is a syntax error. + else if bb_items.is_empty() && next_items.is_empty() { + return Failure( + parser.token.take(), + "no rules expected this token in macro call", + ); + } + // Dump all possible `next_items` into `cur_items` for the next iteration. + else if !next_items.is_empty() { + // Now process the next token + cur_items.extend(next_items.drain(..)); + parser.bump(); + } + // Finally, we have the case where we need to call the black-box parser to get some + // nonterminal. + else { + assert_eq!(bb_items.len(), 1); + + let mut item = bb_items.pop().unwrap(); + if let TokenTree::MetaVarDecl(span, _, ident) = item.top_elts.get_tt(item.idx) { + let match_cur = item.match_cur; + item.push_match( + match_cur, + MatchedNonterminal(Lrc::new(parse_nt(&mut parser, span, ident.name))), + ); + item.idx += 1; + item.match_cur += 1; + } else { + unreachable!() + } + cur_items.push(item); + } + + assert!(!cur_items.is_empty()); + } +} + +/// The token is an identifier, but not `_`. +/// We prohibit passing `_` to macros expecting `ident` for now. +fn get_macro_name(token: &Token) -> Option<(Name, bool)> { + match token.kind { + token::Ident(name, is_raw) if name != kw::Underscore => Some((name, is_raw)), + _ => None, + } +} + +/// Checks whether a non-terminal may begin with a particular token. +/// +/// Returning `false` is a *stability guarantee* that such a matcher will *never* begin with that +/// token. Be conservative (return true) if not sure. +fn may_begin_with(token: &Token, name: Name) -> bool { + /// Checks whether the non-terminal may contain a single (non-keyword) identifier. + fn may_be_ident(nt: &token::Nonterminal) -> bool { + match *nt { + token::NtItem(_) | token::NtBlock(_) | token::NtVis(_) => false, + _ => true, + } + } + + match name { + sym::expr => token.can_begin_expr() + // This exception is here for backwards compatibility. + && !token.is_keyword(kw::Let), + sym::ty => token.can_begin_type(), + sym::ident => get_macro_name(token).is_some(), + sym::literal => token.can_begin_literal_or_bool(), + sym::vis => match token.kind { + // The follow-set of :vis + "priv" keyword + interpolated + token::Comma | token::Ident(..) | token::Interpolated(_) => true, + _ => token.can_begin_type(), + }, + sym::block => match token.kind { + token::OpenDelim(token::Brace) => true, + token::Interpolated(ref nt) => match **nt { + token::NtItem(_) + | token::NtPat(_) + | token::NtTy(_) + | token::NtIdent(..) + | token::NtMeta(_) + | token::NtPath(_) + | token::NtVis(_) => false, // none of these may start with '{'. + _ => true, + }, + _ => false, + }, + sym::path | sym::meta => match token.kind { + token::ModSep | token::Ident(..) => true, + token::Interpolated(ref nt) => match **nt { + token::NtPath(_) | token::NtMeta(_) => true, + _ => may_be_ident(&nt), + }, + _ => false, + }, + sym::pat => match token.kind { + token::Ident(..) | // box, ref, mut, and other identifiers (can stricten) + token::OpenDelim(token::Paren) | // tuple pattern + token::OpenDelim(token::Bracket) | // slice pattern + token::BinOp(token::And) | // reference + token::BinOp(token::Minus) | // negative literal + token::AndAnd | // double reference + token::Literal(..) | // literal + token::DotDot | // range pattern (future compat) + token::DotDotDot | // range pattern (future compat) + token::ModSep | // path + token::Lt | // path (UFCS constant) + token::BinOp(token::Shl) => true, // path (double UFCS) + token::Interpolated(ref nt) => may_be_ident(nt), + _ => false, + }, + sym::lifetime => match token.kind { + token::Lifetime(_) => true, + token::Interpolated(ref nt) => match **nt { + token::NtLifetime(_) | token::NtTT(_) => true, + _ => false, + }, + _ => false, + }, + _ => match token.kind { + token::CloseDelim(_) => false, + _ => true, + }, + } +} + +/// A call to the "black-box" parser to parse some Rust non-terminal. +/// +/// # Parameters +/// +/// - `p`: the "black-box" parser to use +/// - `sp`: the `Span` we want to parse +/// - `name`: the name of the metavar _matcher_ we want to match (e.g., `tt`, `ident`, `block`, +/// etc...) +/// +/// # Returns +/// +/// The parsed non-terminal. +fn parse_nt(p: &mut Parser<'_>, sp: Span, name: Symbol) -> Nonterminal { + if name == sym::tt { + return token::NtTT(p.parse_token_tree()); + } + // check at the beginning and the parser checks after each bump + p.process_potential_macro_variable(); + match parse_nt_inner(p, sp, name) { + Ok(nt) => nt, + Err(mut err) => { + err.emit(); + FatalError.raise(); + } + } +} + +fn parse_nt_inner<'a>(p: &mut Parser<'a>, sp: Span, name: Symbol) -> PResult<'a, Nonterminal> { + Ok(match name { + sym::item => match p.parse_item()? { + Some(i) => token::NtItem(i), + None => return Err(p.fatal("expected an item keyword")), + }, + sym::block => token::NtBlock(p.parse_block()?), + sym::stmt => match p.parse_stmt()? { + Some(s) => token::NtStmt(s), + None => return Err(p.fatal("expected a statement")), + }, + sym::pat => token::NtPat(p.parse_pat(None)?), + sym::expr => token::NtExpr(p.parse_expr()?), + sym::literal => token::NtLiteral(p.parse_literal_maybe_minus()?), + sym::ty => token::NtTy(p.parse_ty()?), + // this could be handled like a token, since it is one + sym::ident => if let Some((name, is_raw)) = get_macro_name(&p.token) { + let span = p.token.span; + p.bump(); + token::NtIdent(Ident::new(name, span), is_raw) + } else { + let token_str = pprust::token_to_string(&p.token); + return Err(p.fatal(&format!("expected ident, found {}", &token_str))); + } + sym::path => token::NtPath(p.parse_path(PathStyle::Type)?), + sym::meta => token::NtMeta(p.parse_attr_item()?), + sym::vis => token::NtVis(p.parse_visibility(true)?), + sym::lifetime => if p.check_lifetime() { + token::NtLifetime(p.expect_lifetime().ident) + } else { + let token_str = pprust::token_to_string(&p.token); + return Err(p.fatal(&format!("expected a lifetime, found `{}`", &token_str))); + } + // this is not supposed to happen, since it has been checked + // when compiling the macro. + _ => p.span_bug(sp, "invalid fragment specifier"), + }) +} diff --git a/src/libsyntax_expand/mbe/macro_rules.rs b/src/libsyntax_expand/mbe/macro_rules.rs new file mode 100644 index 00000000000..9a4130b2d8d --- /dev/null +++ b/src/libsyntax_expand/mbe/macro_rules.rs @@ -0,0 +1,1192 @@ +use crate::base::{DummyResult, ExtCtxt, MacResult, TTMacroExpander}; +use crate::base::{SyntaxExtension, SyntaxExtensionKind}; +use crate::expand::{AstFragment, AstFragmentKind, ensure_complete_parse, parse_ast_fragment}; +use crate::mbe; +use crate::mbe::macro_check; +use crate::mbe::macro_parser::parse; +use crate::mbe::macro_parser::{Error, Failure, Success}; +use crate::mbe::macro_parser::{MatchedNonterminal, MatchedSeq, NamedParseResult}; +use crate::mbe::transcribe::transcribe; + +use syntax::ast; +use syntax::attr::{self, TransparencyError}; +use syntax::edition::Edition; +use syntax::feature_gate::Features; +use syntax::parse::parser::Parser; +use syntax::parse::token::TokenKind::*; +use syntax::parse::token::{self, NtTT, Token}; +use syntax::parse::Directory; +use syntax::print::pprust; +use syntax::sess::ParseSess; +use syntax::symbol::{kw, sym, Symbol}; +use syntax::tokenstream::{DelimSpan, TokenStream}; + +use errors::{DiagnosticBuilder, FatalError}; +use log::debug; +use syntax_pos::hygiene::Transparency; +use syntax_pos::Span; + +use rustc_data_structures::fx::FxHashMap; +use std::borrow::Cow; +use std::collections::hash_map::Entry; +use std::slice; + +use errors::Applicability; +use rustc_data_structures::sync::Lrc; + +const VALID_FRAGMENT_NAMES_MSG: &str = "valid fragment specifiers are \ + `ident`, `block`, `stmt`, `expr`, `pat`, `ty`, `lifetime`, \ + `literal`, `path`, `meta`, `tt`, `item` and `vis`"; + +crate struct ParserAnyMacro<'a> { + parser: Parser<'a>, + + /// Span of the expansion site of the macro this parser is for + site_span: Span, + /// The ident of the macro we're parsing + macro_ident: ast::Ident, + arm_span: Span, +} + +crate fn annotate_err_with_kind( + err: &mut DiagnosticBuilder<'_>, + kind: AstFragmentKind, + span: Span, +) { + match kind { + AstFragmentKind::Ty => { + err.span_label(span, "this macro call doesn't expand to a type"); + } + AstFragmentKind::Pat => { + err.span_label(span, "this macro call doesn't expand to a pattern"); + } + _ => {} + }; +} + +impl<'a> ParserAnyMacro<'a> { + crate fn make(mut self: Box>, kind: AstFragmentKind) -> AstFragment { + let ParserAnyMacro { site_span, macro_ident, ref mut parser, arm_span } = *self; + let fragment = panictry!(parse_ast_fragment(parser, kind, true).map_err(|mut e| { + if parser.token == token::Eof && e.message().ends_with(", found ``") { + if !e.span.is_dummy() { + // early end of macro arm (#52866) + e.replace_span_with(parser.sess.source_map().next_point(parser.token.span)); + } + let msg = &e.message[0]; + e.message[0] = ( + format!( + "macro expansion ends with an incomplete expression: {}", + msg.0.replace(", found ``", ""), + ), + msg.1, + ); + } + if e.span.is_dummy() { + // Get around lack of span in error (#30128) + e.replace_span_with(site_span); + if parser.sess.source_map().span_to_filename(arm_span).is_real() { + e.span_label(arm_span, "in this macro arm"); + } + } else if !parser.sess.source_map().span_to_filename(parser.token.span).is_real() { + e.span_label(site_span, "in this macro invocation"); + } + match kind { + AstFragmentKind::Pat if macro_ident.name == sym::vec => { + let mut suggestion = None; + if let Ok(code) = parser.sess.source_map().span_to_snippet(site_span) { + if let Some(bang) = code.find('!') { + suggestion = Some(code[bang + 1..].to_string()); + } + } + if let Some(suggestion) = suggestion { + e.span_suggestion( + site_span, + "use a slice pattern here instead", + suggestion, + Applicability::MachineApplicable, + ); + } else { + e.span_label( + site_span, + "use a slice pattern here instead", + ); + } + e.help("for more information, see https://doc.rust-lang.org/edition-guide/\ + rust-2018/slice-patterns.html"); + } + _ => annotate_err_with_kind(&mut e, kind, site_span), + }; + e + })); + + // We allow semicolons at the end of expressions -- e.g., the semicolon in + // `macro_rules! m { () => { panic!(); } }` isn't parsed by `.parse_expr()`, + // but `m!()` is allowed in expression positions (cf. issue #34706). + if kind == AstFragmentKind::Expr && parser.token == token::Semi { + parser.bump(); + } + + // Make sure we don't have any tokens left to parse so we don't silently drop anything. + let path = ast::Path::from_ident(macro_ident.with_span_pos(site_span)); + ensure_complete_parse(parser, &path, kind.name(), site_span); + fragment + } +} + +struct MacroRulesMacroExpander { + name: ast::Ident, + span: Span, + transparency: Transparency, + lhses: Vec, + rhses: Vec, + valid: bool, +} + +impl TTMacroExpander for MacroRulesMacroExpander { + fn expand<'cx>( + &self, + cx: &'cx mut ExtCtxt<'_>, + sp: Span, + input: TokenStream, + ) -> Box { + if !self.valid { + return DummyResult::any(sp); + } + generic_extension( + cx, sp, self.span, self.name, self.transparency, input, &self.lhses, &self.rhses + ) + } +} + +fn trace_macros_note(cx: &mut ExtCtxt<'_>, sp: Span, message: String) { + let sp = sp.macro_backtrace().last().map(|trace| trace.call_site).unwrap_or(sp); + cx.expansions.entry(sp).or_default().push(message); +} + +/// Given `lhses` and `rhses`, this is the new macro we create +fn generic_extension<'cx>( + cx: &'cx mut ExtCtxt<'_>, + sp: Span, + def_span: Span, + name: ast::Ident, + transparency: Transparency, + arg: TokenStream, + lhses: &[mbe::TokenTree], + rhses: &[mbe::TokenTree], +) -> Box { + if cx.trace_macros() { + let msg = format!("expanding `{}! {{ {} }}`", name, pprust::tts_to_string(arg.clone())); + trace_macros_note(cx, sp, msg); + } + + // Which arm's failure should we report? (the one furthest along) + let mut best_failure: Option<(Token, &str)> = None; + + for (i, lhs) in lhses.iter().enumerate() { + // try each arm's matchers + let lhs_tt = match *lhs { + mbe::TokenTree::Delimited(_, ref delim) => &delim.tts[..], + _ => cx.span_bug(sp, "malformed macro lhs"), + }; + + match parse_tt(cx, lhs_tt, arg.clone()) { + Success(named_matches) => { + let rhs = match rhses[i] { + // ignore delimiters + mbe::TokenTree::Delimited(_, ref delimed) => delimed.tts.clone(), + _ => cx.span_bug(sp, "malformed macro rhs"), + }; + let arm_span = rhses[i].span(); + + let rhs_spans = rhs.iter().map(|t| t.span()).collect::>(); + // rhs has holes ( `$id` and `$(...)` that need filled) + let mut tts = transcribe(cx, &named_matches, rhs, transparency); + + // Replace all the tokens for the corresponding positions in the macro, to maintain + // proper positions in error reporting, while maintaining the macro_backtrace. + if rhs_spans.len() == tts.len() { + tts = tts.map_enumerated(|i, mut tt| { + let mut sp = rhs_spans[i]; + sp = sp.with_ctxt(tt.span().ctxt()); + tt.set_span(sp); + tt + }); + } + + if cx.trace_macros() { + let msg = format!("to `{}`", pprust::tts_to_string(tts.clone())); + trace_macros_note(cx, sp, msg); + } + + let directory = Directory { + path: Cow::from(cx.current_expansion.module.directory.as_path()), + ownership: cx.current_expansion.directory_ownership, + }; + let mut p = Parser::new(cx.parse_sess(), tts, Some(directory), true, false, None); + p.root_module_name = + cx.current_expansion.module.mod_path.last().map(|id| id.as_str().to_string()); + p.last_type_ascription = cx.current_expansion.prior_type_ascription; + + p.process_potential_macro_variable(); + // Let the context choose how to interpret the result. + // Weird, but useful for X-macros. + return Box::new(ParserAnyMacro { + parser: p, + + // Pass along the original expansion site and the name of the macro + // so we can print a useful error message if the parse of the expanded + // macro leaves unparsed tokens. + site_span: sp, + macro_ident: name, + arm_span, + }); + } + Failure(token, msg) => match best_failure { + Some((ref best_token, _)) if best_token.span.lo() >= token.span.lo() => {} + _ => best_failure = Some((token, msg)), + }, + Error(err_sp, ref msg) => cx.span_fatal(err_sp.substitute_dummy(sp), &msg[..]), + } + } + + let (token, label) = best_failure.expect("ran no matchers"); + let span = token.span.substitute_dummy(sp); + let mut err = cx.struct_span_err(span, &parse_failure_msg(&token)); + err.span_label(span, label); + if !def_span.is_dummy() && cx.source_map().span_to_filename(def_span).is_real() { + err.span_label(cx.source_map().def_span(def_span), "when calling this macro"); + } + + // Check whether there's a missing comma in this macro call, like `println!("{}" a);` + if let Some((arg, comma_span)) = arg.add_comma() { + for lhs in lhses { + // try each arm's matchers + let lhs_tt = match *lhs { + mbe::TokenTree::Delimited(_, ref delim) => &delim.tts[..], + _ => continue, + }; + match parse_tt(cx, lhs_tt, arg.clone()) { + Success(_) => { + if comma_span.is_dummy() { + err.note("you might be missing a comma"); + } else { + err.span_suggestion_short( + comma_span, + "missing comma here", + ", ".to_string(), + Applicability::MachineApplicable, + ); + } + } + _ => {} + } + } + } + err.emit(); + cx.trace_macros_diag(); + DummyResult::any(sp) +} + +// Note that macro-by-example's input is also matched against a token tree: +// $( $lhs:tt => $rhs:tt );+ +// +// Holy self-referential! + +/// Converts a macro item into a syntax extension. +pub fn compile_declarative_macro( + sess: &ParseSess, + features: &Features, + def: &ast::Item, + edition: Edition, +) -> SyntaxExtension { + let diag = &sess.span_diagnostic; + let lhs_nm = ast::Ident::new(sym::lhs, def.span); + let rhs_nm = ast::Ident::new(sym::rhs, def.span); + let tt_spec = ast::Ident::new(sym::tt, def.span); + + // Parse the macro_rules! invocation + let body = match def.kind { + ast::ItemKind::MacroDef(ref body) => body, + _ => unreachable!(), + }; + + // The pattern that macro_rules matches. + // The grammar for macro_rules! is: + // $( $lhs:tt => $rhs:tt );+ + // ...quasiquoting this would be nice. + // These spans won't matter, anyways + let argument_gram = vec![ + mbe::TokenTree::Sequence( + DelimSpan::dummy(), + Lrc::new(mbe::SequenceRepetition { + tts: vec![ + mbe::TokenTree::MetaVarDecl(def.span, lhs_nm, tt_spec), + mbe::TokenTree::token(token::FatArrow, def.span), + mbe::TokenTree::MetaVarDecl(def.span, rhs_nm, tt_spec), + ], + separator: Some(Token::new( + if body.legacy { token::Semi } else { token::Comma }, + def.span, + )), + kleene: mbe::KleeneToken::new(mbe::KleeneOp::OneOrMore, def.span), + num_captures: 2, + }), + ), + // to phase into semicolon-termination instead of semicolon-separation + mbe::TokenTree::Sequence( + DelimSpan::dummy(), + Lrc::new(mbe::SequenceRepetition { + tts: vec![mbe::TokenTree::token( + if body.legacy { token::Semi } else { token::Comma }, + def.span, + )], + separator: None, + kleene: mbe::KleeneToken::new(mbe::KleeneOp::ZeroOrMore, def.span), + num_captures: 0, + }), + ), + ]; + + let argument_map = match parse(sess, body.stream(), &argument_gram, None, true) { + Success(m) => m, + Failure(token, msg) => { + let s = parse_failure_msg(&token); + let sp = token.span.substitute_dummy(def.span); + let mut err = sess.span_diagnostic.struct_span_fatal(sp, &s); + err.span_label(sp, msg); + err.emit(); + FatalError.raise(); + } + Error(sp, s) => { + sess.span_diagnostic.span_fatal(sp.substitute_dummy(def.span), &s).raise(); + } + }; + + let mut valid = true; + + // Extract the arguments: + let lhses = match argument_map[&lhs_nm] { + MatchedSeq(ref s, _) => s + .iter() + .map(|m| { + if let MatchedNonterminal(ref nt) = *m { + if let NtTT(ref tt) = **nt { + let tt = mbe::quoted::parse( + tt.clone().into(), + true, + sess, + ) + .pop() + .unwrap(); + valid &= check_lhs_nt_follows(sess, features, &def.attrs, &tt); + return tt; + } + } + sess.span_diagnostic.span_bug(def.span, "wrong-structured lhs") + }) + .collect::>(), + _ => sess.span_diagnostic.span_bug(def.span, "wrong-structured lhs"), + }; + + let rhses = match argument_map[&rhs_nm] { + MatchedSeq(ref s, _) => s + .iter() + .map(|m| { + if let MatchedNonterminal(ref nt) = *m { + if let NtTT(ref tt) = **nt { + return mbe::quoted::parse( + tt.clone().into(), + false, + sess, + ) + .pop() + .unwrap(); + } + } + sess.span_diagnostic.span_bug(def.span, "wrong-structured lhs") + }) + .collect::>(), + _ => sess.span_diagnostic.span_bug(def.span, "wrong-structured rhs"), + }; + + for rhs in &rhses { + valid &= check_rhs(sess, rhs); + } + + // don't abort iteration early, so that errors for multiple lhses can be reported + for lhs in &lhses { + valid &= check_lhs_no_empty_seq(sess, slice::from_ref(lhs)); + } + + // We use CRATE_NODE_ID instead of `def.id` otherwise we may emit buffered lints for a node id + // that is not lint-checked and trigger the "failed to process buffered lint here" bug. + valid &= macro_check::check_meta_variables(sess, ast::CRATE_NODE_ID, def.span, &lhses, &rhses); + + let (transparency, transparency_error) = attr::find_transparency(&def.attrs, body.legacy); + match transparency_error { + Some(TransparencyError::UnknownTransparency(value, span)) => + diag.span_err(span, &format!("unknown macro transparency: `{}`", value)), + Some(TransparencyError::MultipleTransparencyAttrs(old_span, new_span)) => + diag.span_err(vec![old_span, new_span], "multiple macro transparency attributes"), + None => {} + } + + let expander: Box<_> = Box::new(MacroRulesMacroExpander { + name: def.ident, span: def.span, transparency, lhses, rhses, valid + }); + + SyntaxExtension::new( + sess, + SyntaxExtensionKind::LegacyBang(expander), + def.span, + Vec::new(), + edition, + def.ident.name, + &def.attrs, + ) +} + +fn check_lhs_nt_follows( + sess: &ParseSess, + features: &Features, + attrs: &[ast::Attribute], + lhs: &mbe::TokenTree, +) -> bool { + // lhs is going to be like TokenTree::Delimited(...), where the + // entire lhs is those tts. Or, it can be a "bare sequence", not wrapped in parens. + if let mbe::TokenTree::Delimited(_, ref tts) = *lhs { + check_matcher(sess, features, attrs, &tts.tts) + } else { + let msg = "invalid macro matcher; matchers must be contained in balanced delimiters"; + sess.span_diagnostic.span_err(lhs.span(), msg); + false + } + // we don't abort on errors on rejection, the driver will do that for us + // after parsing/expansion. we can report every error in every macro this way. +} + +/// Checks that the lhs contains no repetition which could match an empty token +/// tree, because then the matcher would hang indefinitely. +fn check_lhs_no_empty_seq(sess: &ParseSess, tts: &[mbe::TokenTree]) -> bool { + use mbe::TokenTree; + for tt in tts { + match *tt { + TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => (), + TokenTree::Delimited(_, ref del) => { + if !check_lhs_no_empty_seq(sess, &del.tts) { + return false; + } + } + TokenTree::Sequence(span, ref seq) => { + if seq.separator.is_none() + && seq.tts.iter().all(|seq_tt| match *seq_tt { + TokenTree::MetaVarDecl(_, _, id) => id.name == sym::vis, + TokenTree::Sequence(_, ref sub_seq) => { + sub_seq.kleene.op == mbe::KleeneOp::ZeroOrMore + || sub_seq.kleene.op == mbe::KleeneOp::ZeroOrOne + } + _ => false, + }) + { + let sp = span.entire(); + sess.span_diagnostic.span_err(sp, "repetition matches empty token tree"); + return false; + } + if !check_lhs_no_empty_seq(sess, &seq.tts) { + return false; + } + } + } + } + + true +} + +fn check_rhs(sess: &ParseSess, rhs: &mbe::TokenTree) -> bool { + match *rhs { + mbe::TokenTree::Delimited(..) => return true, + _ => sess.span_diagnostic.span_err(rhs.span(), "macro rhs must be delimited"), + } + false +} + +fn check_matcher( + sess: &ParseSess, + features: &Features, + attrs: &[ast::Attribute], + matcher: &[mbe::TokenTree], +) -> bool { + let first_sets = FirstSets::new(matcher); + let empty_suffix = TokenSet::empty(); + let err = sess.span_diagnostic.err_count(); + check_matcher_core(sess, features, attrs, &first_sets, matcher, &empty_suffix); + err == sess.span_diagnostic.err_count() +} + +// `The FirstSets` for a matcher is a mapping from subsequences in the +// matcher to the FIRST set for that subsequence. +// +// This mapping is partially precomputed via a backwards scan over the +// token trees of the matcher, which provides a mapping from each +// repetition sequence to its *first* set. +// +// (Hypothetically, sequences should be uniquely identifiable via their +// spans, though perhaps that is false, e.g., for macro-generated macros +// that do not try to inject artificial span information. My plan is +// to try to catch such cases ahead of time and not include them in +// the precomputed mapping.) +struct FirstSets { + // this maps each TokenTree::Sequence `$(tt ...) SEP OP` that is uniquely identified by its + // span in the original matcher to the First set for the inner sequence `tt ...`. + // + // If two sequences have the same span in a matcher, then map that + // span to None (invalidating the mapping here and forcing the code to + // use a slow path). + first: FxHashMap>, +} + +impl FirstSets { + fn new(tts: &[mbe::TokenTree]) -> FirstSets { + use mbe::TokenTree; + + let mut sets = FirstSets { first: FxHashMap::default() }; + build_recur(&mut sets, tts); + return sets; + + // walks backward over `tts`, returning the FIRST for `tts` + // and updating `sets` at the same time for all sequence + // substructure we find within `tts`. + fn build_recur(sets: &mut FirstSets, tts: &[TokenTree]) -> TokenSet { + let mut first = TokenSet::empty(); + for tt in tts.iter().rev() { + match *tt { + TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => { + first.replace_with(tt.clone()); + } + TokenTree::Delimited(span, ref delimited) => { + build_recur(sets, &delimited.tts[..]); + first.replace_with(delimited.open_tt(span.open)); + } + TokenTree::Sequence(sp, ref seq_rep) => { + let subfirst = build_recur(sets, &seq_rep.tts[..]); + + match sets.first.entry(sp.entire()) { + Entry::Vacant(vac) => { + vac.insert(Some(subfirst.clone())); + } + Entry::Occupied(mut occ) => { + // if there is already an entry, then a span must have collided. + // This should not happen with typical macro_rules macros, + // but syntax extensions need not maintain distinct spans, + // so distinct syntax trees can be assigned the same span. + // In such a case, the map cannot be trusted; so mark this + // entry as unusable. + occ.insert(None); + } + } + + // If the sequence contents can be empty, then the first + // token could be the separator token itself. + + if let (Some(sep), true) = (&seq_rep.separator, subfirst.maybe_empty) { + first.add_one_maybe(TokenTree::Token(sep.clone())); + } + + // Reverse scan: Sequence comes before `first`. + if subfirst.maybe_empty + || seq_rep.kleene.op == mbe::KleeneOp::ZeroOrMore + || seq_rep.kleene.op == mbe::KleeneOp::ZeroOrOne + { + // If sequence is potentially empty, then + // union them (preserving first emptiness). + first.add_all(&TokenSet { maybe_empty: true, ..subfirst }); + } else { + // Otherwise, sequence guaranteed + // non-empty; replace first. + first = subfirst; + } + } + } + } + + first + } + } + + // walks forward over `tts` until all potential FIRST tokens are + // identified. + fn first(&self, tts: &[mbe::TokenTree]) -> TokenSet { + use mbe::TokenTree; + + let mut first = TokenSet::empty(); + for tt in tts.iter() { + assert!(first.maybe_empty); + match *tt { + TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => { + first.add_one(tt.clone()); + return first; + } + TokenTree::Delimited(span, ref delimited) => { + first.add_one(delimited.open_tt(span.open)); + return first; + } + TokenTree::Sequence(sp, ref seq_rep) => { + let subfirst_owned; + let subfirst = match self.first.get(&sp.entire()) { + Some(&Some(ref subfirst)) => subfirst, + Some(&None) => { + subfirst_owned = self.first(&seq_rep.tts[..]); + &subfirst_owned + } + None => { + panic!("We missed a sequence during FirstSets construction"); + } + }; + + // If the sequence contents can be empty, then the first + // token could be the separator token itself. + if let (Some(sep), true) = (&seq_rep.separator, subfirst.maybe_empty) { + first.add_one_maybe(TokenTree::Token(sep.clone())); + } + + assert!(first.maybe_empty); + first.add_all(subfirst); + if subfirst.maybe_empty + || seq_rep.kleene.op == mbe::KleeneOp::ZeroOrMore + || seq_rep.kleene.op == mbe::KleeneOp::ZeroOrOne + { + // Continue scanning for more first + // tokens, but also make sure we + // restore empty-tracking state. + first.maybe_empty = true; + continue; + } else { + return first; + } + } + } + } + + // we only exit the loop if `tts` was empty or if every + // element of `tts` matches the empty sequence. + assert!(first.maybe_empty); + first + } +} + +// A set of `mbe::TokenTree`s, which may include `TokenTree::Match`s +// (for macro-by-example syntactic variables). It also carries the +// `maybe_empty` flag; that is true if and only if the matcher can +// match an empty token sequence. +// +// The First set is computed on submatchers like `$($a:expr b),* $(c)* d`, +// which has corresponding FIRST = {$a:expr, c, d}. +// Likewise, `$($a:expr b),* $(c)+ d` has FIRST = {$a:expr, c}. +// +// (Notably, we must allow for *-op to occur zero times.) +#[derive(Clone, Debug)] +struct TokenSet { + tokens: Vec, + maybe_empty: bool, +} + +impl TokenSet { + // Returns a set for the empty sequence. + fn empty() -> Self { + TokenSet { tokens: Vec::new(), maybe_empty: true } + } + + // Returns the set `{ tok }` for the single-token (and thus + // non-empty) sequence [tok]. + fn singleton(tok: mbe::TokenTree) -> Self { + TokenSet { tokens: vec![tok], maybe_empty: false } + } + + // Changes self to be the set `{ tok }`. + // Since `tok` is always present, marks self as non-empty. + fn replace_with(&mut self, tok: mbe::TokenTree) { + self.tokens.clear(); + self.tokens.push(tok); + self.maybe_empty = false; + } + + // Changes self to be the empty set `{}`; meant for use when + // the particular token does not matter, but we want to + // record that it occurs. + fn replace_with_irrelevant(&mut self) { + self.tokens.clear(); + self.maybe_empty = false; + } + + // Adds `tok` to the set for `self`, marking sequence as non-empy. + fn add_one(&mut self, tok: mbe::TokenTree) { + if !self.tokens.contains(&tok) { + self.tokens.push(tok); + } + self.maybe_empty = false; + } + + // Adds `tok` to the set for `self`. (Leaves `maybe_empty` flag alone.) + fn add_one_maybe(&mut self, tok: mbe::TokenTree) { + if !self.tokens.contains(&tok) { + self.tokens.push(tok); + } + } + + // Adds all elements of `other` to this. + // + // (Since this is a set, we filter out duplicates.) + // + // If `other` is potentially empty, then preserves the previous + // setting of the empty flag of `self`. If `other` is guaranteed + // non-empty, then `self` is marked non-empty. + fn add_all(&mut self, other: &Self) { + for tok in &other.tokens { + if !self.tokens.contains(tok) { + self.tokens.push(tok.clone()); + } + } + if !other.maybe_empty { + self.maybe_empty = false; + } + } +} + +// Checks that `matcher` is internally consistent and that it +// can legally be followed by a token `N`, for all `N` in `follow`. +// (If `follow` is empty, then it imposes no constraint on +// the `matcher`.) +// +// Returns the set of NT tokens that could possibly come last in +// `matcher`. (If `matcher` matches the empty sequence, then +// `maybe_empty` will be set to true.) +// +// Requires that `first_sets` is pre-computed for `matcher`; +// see `FirstSets::new`. +fn check_matcher_core( + sess: &ParseSess, + features: &Features, + attrs: &[ast::Attribute], + first_sets: &FirstSets, + matcher: &[mbe::TokenTree], + follow: &TokenSet, +) -> TokenSet { + use mbe::TokenTree; + + let mut last = TokenSet::empty(); + + // 2. For each token and suffix [T, SUFFIX] in M: + // ensure that T can be followed by SUFFIX, and if SUFFIX may be empty, + // then ensure T can also be followed by any element of FOLLOW. + 'each_token: for i in 0..matcher.len() { + let token = &matcher[i]; + let suffix = &matcher[i + 1..]; + + let build_suffix_first = || { + let mut s = first_sets.first(suffix); + if s.maybe_empty { + s.add_all(follow); + } + s + }; + + // (we build `suffix_first` on demand below; you can tell + // which cases are supposed to fall through by looking for the + // initialization of this variable.) + let suffix_first; + + // First, update `last` so that it corresponds to the set + // of NT tokens that might end the sequence `... token`. + match *token { + TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => { + let can_be_followed_by_any; + if let Err(bad_frag) = has_legal_fragment_specifier(sess, features, attrs, token) { + let msg = format!("invalid fragment specifier `{}`", bad_frag); + sess.span_diagnostic + .struct_span_err(token.span(), &msg) + .help(VALID_FRAGMENT_NAMES_MSG) + .emit(); + // (This eliminates false positives and duplicates + // from error messages.) + can_be_followed_by_any = true; + } else { + can_be_followed_by_any = token_can_be_followed_by_any(token); + } + + if can_be_followed_by_any { + // don't need to track tokens that work with any, + last.replace_with_irrelevant(); + // ... and don't need to check tokens that can be + // followed by anything against SUFFIX. + continue 'each_token; + } else { + last.replace_with(token.clone()); + suffix_first = build_suffix_first(); + } + } + TokenTree::Delimited(span, ref d) => { + let my_suffix = TokenSet::singleton(d.close_tt(span.close)); + check_matcher_core(sess, features, attrs, first_sets, &d.tts, &my_suffix); + // don't track non NT tokens + last.replace_with_irrelevant(); + + // also, we don't need to check delimited sequences + // against SUFFIX + continue 'each_token; + } + TokenTree::Sequence(_, ref seq_rep) => { + suffix_first = build_suffix_first(); + // The trick here: when we check the interior, we want + // to include the separator (if any) as a potential + // (but not guaranteed) element of FOLLOW. So in that + // case, we make a temp copy of suffix and stuff + // delimiter in there. + // + // FIXME: Should I first scan suffix_first to see if + // delimiter is already in it before I go through the + // work of cloning it? But then again, this way I may + // get a "tighter" span? + let mut new; + let my_suffix = if let Some(sep) = &seq_rep.separator { + new = suffix_first.clone(); + new.add_one_maybe(TokenTree::Token(sep.clone())); + &new + } else { + &suffix_first + }; + + // At this point, `suffix_first` is built, and + // `my_suffix` is some TokenSet that we can use + // for checking the interior of `seq_rep`. + let next = + check_matcher_core(sess, features, attrs, first_sets, &seq_rep.tts, my_suffix); + if next.maybe_empty { + last.add_all(&next); + } else { + last = next; + } + + // the recursive call to check_matcher_core already ran the 'each_last + // check below, so we can just keep going forward here. + continue 'each_token; + } + } + + // (`suffix_first` guaranteed initialized once reaching here.) + + // Now `last` holds the complete set of NT tokens that could + // end the sequence before SUFFIX. Check that every one works with `suffix`. + 'each_last: for token in &last.tokens { + if let TokenTree::MetaVarDecl(_, name, frag_spec) = *token { + for next_token in &suffix_first.tokens { + match is_in_follow(next_token, frag_spec.name) { + IsInFollow::Invalid(msg, help) => { + sess.span_diagnostic + .struct_span_err(next_token.span(), &msg) + .help(help) + .emit(); + // don't bother reporting every source of + // conflict for a particular element of `last`. + continue 'each_last; + } + IsInFollow::Yes => {} + IsInFollow::No(possible) => { + let may_be = if last.tokens.len() == 1 && suffix_first.tokens.len() == 1 + { + "is" + } else { + "may be" + }; + + let sp = next_token.span(); + let mut err = sess.span_diagnostic.struct_span_err( + sp, + &format!( + "`${name}:{frag}` {may_be} followed by `{next}`, which \ + is not allowed for `{frag}` fragments", + name = name, + frag = frag_spec, + next = quoted_tt_to_string(next_token), + may_be = may_be + ), + ); + err.span_label( + sp, + format!("not allowed after `{}` fragments", frag_spec), + ); + let msg = "allowed there are: "; + match possible { + &[] => {} + &[t] => { + err.note(&format!( + "only {} is allowed after `{}` fragments", + t, frag_spec, + )); + } + ts => { + err.note(&format!( + "{}{} or {}", + msg, + ts[..ts.len() - 1] + .iter() + .map(|s| *s) + .collect::>() + .join(", "), + ts[ts.len() - 1], + )); + } + } + err.emit(); + } + } + } + } + } + } + last +} + +fn token_can_be_followed_by_any(tok: &mbe::TokenTree) -> bool { + if let mbe::TokenTree::MetaVarDecl(_, _, frag_spec) = *tok { + frag_can_be_followed_by_any(frag_spec.name) + } else { + // (Non NT's can always be followed by anthing in matchers.) + true + } +} + +/// Returns `true` if a fragment of type `frag` can be followed by any sort of +/// token. We use this (among other things) as a useful approximation +/// for when `frag` can be followed by a repetition like `$(...)*` or +/// `$(...)+`. In general, these can be a bit tricky to reason about, +/// so we adopt a conservative position that says that any fragment +/// specifier which consumes at most one token tree can be followed by +/// a fragment specifier (indeed, these fragments can be followed by +/// ANYTHING without fear of future compatibility hazards). +fn frag_can_be_followed_by_any(frag: Symbol) -> bool { + match frag { + sym::item | // always terminated by `}` or `;` + sym::block | // exactly one token tree + sym::ident | // exactly one token tree + sym::literal | // exactly one token tree + sym::meta | // exactly one token tree + sym::lifetime | // exactly one token tree + sym::tt => // exactly one token tree + true, + + _ => + false, + } +} + +enum IsInFollow { + Yes, + No(&'static [&'static str]), + Invalid(String, &'static str), +} + +/// Returns `true` if `frag` can legally be followed by the token `tok`. For +/// fragments that can consume an unbounded number of tokens, `tok` +/// must be within a well-defined follow set. This is intended to +/// guarantee future compatibility: for example, without this rule, if +/// we expanded `expr` to include a new binary operator, we might +/// break macros that were relying on that binary operator as a +/// separator. +// when changing this do not forget to update doc/book/macros.md! +fn is_in_follow(tok: &mbe::TokenTree, frag: Symbol) -> IsInFollow { + use mbe::TokenTree; + + if let TokenTree::Token(Token { kind: token::CloseDelim(_), .. }) = *tok { + // closing a token tree can never be matched by any fragment; + // iow, we always require that `(` and `)` match, etc. + IsInFollow::Yes + } else { + match frag { + sym::item => { + // since items *must* be followed by either a `;` or a `}`, we can + // accept anything after them + IsInFollow::Yes + } + sym::block => { + // anything can follow block, the braces provide an easy boundary to + // maintain + IsInFollow::Yes + } + sym::stmt | sym::expr => { + const TOKENS: &[&str] = &["`=>`", "`,`", "`;`"]; + match tok { + TokenTree::Token(token) => match token.kind { + FatArrow | Comma | Semi => IsInFollow::Yes, + _ => IsInFollow::No(TOKENS), + }, + _ => IsInFollow::No(TOKENS), + } + } + sym::pat => { + const TOKENS: &[&str] = &["`=>`", "`,`", "`=`", "`|`", "`if`", "`in`"]; + match tok { + TokenTree::Token(token) => match token.kind { + FatArrow | Comma | Eq | BinOp(token::Or) => IsInFollow::Yes, + Ident(name, false) if name == kw::If || name == kw::In => IsInFollow::Yes, + _ => IsInFollow::No(TOKENS), + }, + _ => IsInFollow::No(TOKENS), + } + } + sym::path | sym::ty => { + const TOKENS: &[&str] = &[ + "`{`", "`[`", "`=>`", "`,`", "`>`", "`=`", "`:`", "`;`", "`|`", "`as`", + "`where`", + ]; + match tok { + TokenTree::Token(token) => match token.kind { + OpenDelim(token::DelimToken::Brace) + | OpenDelim(token::DelimToken::Bracket) + | Comma + | FatArrow + | Colon + | Eq + | Gt + | BinOp(token::Shr) + | Semi + | BinOp(token::Or) => IsInFollow::Yes, + Ident(name, false) if name == kw::As || name == kw::Where => { + IsInFollow::Yes + } + _ => IsInFollow::No(TOKENS), + }, + TokenTree::MetaVarDecl(_, _, frag) if frag.name == sym::block => { + IsInFollow::Yes + } + _ => IsInFollow::No(TOKENS), + } + } + sym::ident | sym::lifetime => { + // being a single token, idents and lifetimes are harmless + IsInFollow::Yes + } + sym::literal => { + // literals may be of a single token, or two tokens (negative numbers) + IsInFollow::Yes + } + sym::meta | sym::tt => { + // being either a single token or a delimited sequence, tt is + // harmless + IsInFollow::Yes + } + sym::vis => { + // Explicitly disallow `priv`, on the off chance it comes back. + const TOKENS: &[&str] = &["`,`", "an ident", "a type"]; + match tok { + TokenTree::Token(token) => match token.kind { + Comma => IsInFollow::Yes, + Ident(name, is_raw) if is_raw || name != kw::Priv => IsInFollow::Yes, + _ => { + if token.can_begin_type() { + IsInFollow::Yes + } else { + IsInFollow::No(TOKENS) + } + } + }, + TokenTree::MetaVarDecl(_, _, frag) + if frag.name == sym::ident + || frag.name == sym::ty + || frag.name == sym::path => + { + IsInFollow::Yes + } + _ => IsInFollow::No(TOKENS), + } + } + kw::Invalid => IsInFollow::Yes, + _ => IsInFollow::Invalid( + format!("invalid fragment specifier `{}`", frag), + VALID_FRAGMENT_NAMES_MSG, + ), + } + } +} + +fn has_legal_fragment_specifier( + sess: &ParseSess, + features: &Features, + attrs: &[ast::Attribute], + tok: &mbe::TokenTree, +) -> Result<(), String> { + debug!("has_legal_fragment_specifier({:?})", tok); + if let mbe::TokenTree::MetaVarDecl(_, _, ref frag_spec) = *tok { + let frag_span = tok.span(); + if !is_legal_fragment_specifier(sess, features, attrs, frag_spec.name, frag_span) { + return Err(frag_spec.to_string()); + } + } + Ok(()) +} + +fn is_legal_fragment_specifier( + _sess: &ParseSess, + _features: &Features, + _attrs: &[ast::Attribute], + frag_name: Symbol, + _frag_span: Span, +) -> bool { + /* + * If new fragment specifiers are invented in nightly, `_sess`, + * `_features`, `_attrs`, and `_frag_span` will be useful here + * for checking against feature gates. See past versions of + * this function. + */ + match frag_name { + sym::item + | sym::block + | sym::stmt + | sym::expr + | sym::pat + | sym::lifetime + | sym::path + | sym::ty + | sym::ident + | sym::meta + | sym::tt + | sym::vis + | sym::literal + | kw::Invalid => true, + _ => false, + } +} + +fn quoted_tt_to_string(tt: &mbe::TokenTree) -> String { + match *tt { + mbe::TokenTree::Token(ref token) => pprust::token_to_string(&token), + mbe::TokenTree::MetaVar(_, name) => format!("${}", name), + mbe::TokenTree::MetaVarDecl(_, name, kind) => format!("${}:{}", name, kind), + _ => panic!( + "unexpected mbe::TokenTree::{{Sequence or Delimited}} \ + in follow set checker" + ), + } +} + +/// Use this token tree as a matcher to parse given tts. +fn parse_tt(cx: &ExtCtxt<'_>, mtch: &[mbe::TokenTree], tts: TokenStream) -> NamedParseResult { + // `None` is because we're not interpolating + let directory = Directory { + path: Cow::from(cx.current_expansion.module.directory.as_path()), + ownership: cx.current_expansion.directory_ownership, + }; + parse(cx.parse_sess(), tts, mtch, Some(directory), true) +} + +/// Generates an appropriate parsing failure message. For EOF, this is "unexpected end...". For +/// other tokens, this is "unexpected token...". +fn parse_failure_msg(tok: &Token) -> String { + match tok.kind { + token::Eof => "unexpected end of macro invocation".to_string(), + _ => format!( + "no rules expected the token `{}`", + pprust::token_to_string(tok), + ), + } +} diff --git a/src/libsyntax_expand/mbe/quoted.rs b/src/libsyntax_expand/mbe/quoted.rs new file mode 100644 index 00000000000..cedd59233ad --- /dev/null +++ b/src/libsyntax_expand/mbe/quoted.rs @@ -0,0 +1,264 @@ +use crate::mbe::macro_parser; +use crate::mbe::{TokenTree, KleeneOp, KleeneToken, SequenceRepetition, Delimited}; + +use syntax::ast; +use syntax::parse::token::{self, Token}; +use syntax::print::pprust; +use syntax::sess::ParseSess; +use syntax::symbol::kw; +use syntax::tokenstream; + +use syntax_pos::Span; + +use rustc_data_structures::sync::Lrc; + +/// Takes a `tokenstream::TokenStream` and returns a `Vec`. Specifically, this +/// takes a generic `TokenStream`, such as is used in the rest of the compiler, and returns a +/// collection of `TokenTree` for use in parsing a macro. +/// +/// # Parameters +/// +/// - `input`: a token stream to read from, the contents of which we are parsing. +/// - `expect_matchers`: `parse` can be used to parse either the "patterns" or the "body" of a +/// macro. Both take roughly the same form _except_ that in a pattern, metavars are declared with +/// their "matcher" type. For example `$var:expr` or `$id:ident`. In this example, `expr` and +/// `ident` are "matchers". They are not present in the body of a macro rule -- just in the +/// pattern, so we pass a parameter to indicate whether to expect them or not. +/// - `sess`: the parsing session. Any errors will be emitted to this session. +/// - `features`, `attrs`: language feature flags and attributes so that we know whether to use +/// unstable features or not. +/// - `edition`: which edition are we in. +/// - `macro_node_id`: the NodeId of the macro we are parsing. +/// +/// # Returns +/// +/// A collection of `self::TokenTree`. There may also be some errors emitted to `sess`. +pub(super) fn parse( + input: tokenstream::TokenStream, + expect_matchers: bool, + sess: &ParseSess, +) -> Vec { + // Will contain the final collection of `self::TokenTree` + let mut result = Vec::new(); + + // For each token tree in `input`, parse the token into a `self::TokenTree`, consuming + // additional trees if need be. + let mut trees = input.trees(); + while let Some(tree) = trees.next() { + // Given the parsed tree, if there is a metavar and we are expecting matchers, actually + // parse out the matcher (i.e., in `$id:ident` this would parse the `:` and `ident`). + let tree = parse_tree( + tree, + &mut trees, + expect_matchers, + sess, + ); + match tree { + TokenTree::MetaVar(start_sp, ident) if expect_matchers => { + let span = match trees.next() { + Some(tokenstream::TokenTree::Token(Token { kind: token::Colon, span })) => { + match trees.next() { + Some(tokenstream::TokenTree::Token(token)) => match token.ident() { + Some((kind, _)) => { + let span = token.span.with_lo(start_sp.lo()); + result.push(TokenTree::MetaVarDecl(span, ident, kind)); + continue; + } + _ => token.span, + }, + tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span), + } + } + tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(start_sp), + }; + sess.missing_fragment_specifiers.borrow_mut().insert(span); + result.push(TokenTree::MetaVarDecl(span, ident, ast::Ident::invalid())); + } + + // Not a metavar or no matchers allowed, so just return the tree + _ => result.push(tree), + } + } + result +} + +/// Takes a `tokenstream::TokenTree` and returns a `self::TokenTree`. Specifically, this takes a +/// generic `TokenTree`, such as is used in the rest of the compiler, and returns a `TokenTree` +/// for use in parsing a macro. +/// +/// Converting the given tree may involve reading more tokens. +/// +/// # Parameters +/// +/// - `tree`: the tree we wish to convert. +/// - `trees`: an iterator over trees. We may need to read more tokens from it in order to finish +/// converting `tree` +/// - `expect_matchers`: same as for `parse` (see above). +/// - `sess`: the parsing session. Any errors will be emitted to this session. +/// - `features`, `attrs`: language feature flags and attributes so that we know whether to use +/// unstable features or not. +fn parse_tree( + tree: tokenstream::TokenTree, + trees: &mut impl Iterator, + expect_matchers: bool, + sess: &ParseSess, +) -> TokenTree { + // Depending on what `tree` is, we could be parsing different parts of a macro + match tree { + // `tree` is a `$` token. Look at the next token in `trees` + tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }) => match trees.next() { + // `tree` is followed by a delimited set of token trees. This indicates the beginning + // of a repetition sequence in the macro (e.g. `$(pat)*`). + Some(tokenstream::TokenTree::Delimited(span, delim, tts)) => { + // Must have `(` not `{` or `[` + if delim != token::Paren { + let tok = pprust::token_kind_to_string(&token::OpenDelim(delim)); + let msg = format!("expected `(`, found `{}`", tok); + sess.span_diagnostic.span_err(span.entire(), &msg); + } + // Parse the contents of the sequence itself + let sequence = parse( + tts.into(), + expect_matchers, + sess, + ); + // Get the Kleene operator and optional separator + let (separator, kleene) = parse_sep_and_kleene_op(trees, span.entire(), sess); + // Count the number of captured "names" (i.e., named metavars) + let name_captures = macro_parser::count_names(&sequence); + TokenTree::Sequence( + span, + Lrc::new(SequenceRepetition { + tts: sequence, + separator, + kleene, + num_captures: name_captures, + }), + ) + } + + // `tree` is followed by an `ident`. This could be `$meta_var` or the `$crate` special + // metavariable that names the crate of the invocation. + Some(tokenstream::TokenTree::Token(token)) if token.is_ident() => { + let (ident, is_raw) = token.ident().unwrap(); + let span = ident.span.with_lo(span.lo()); + if ident.name == kw::Crate && !is_raw { + TokenTree::token(token::Ident(kw::DollarCrate, is_raw), span) + } else { + TokenTree::MetaVar(span, ident) + } + } + + // `tree` is followed by a random token. This is an error. + Some(tokenstream::TokenTree::Token(token)) => { + let msg = + format!("expected identifier, found `{}`", pprust::token_to_string(&token),); + sess.span_diagnostic.span_err(token.span, &msg); + TokenTree::MetaVar(token.span, ast::Ident::invalid()) + } + + // There are no more tokens. Just return the `$` we already have. + None => TokenTree::token(token::Dollar, span), + }, + + // `tree` is an arbitrary token. Keep it. + tokenstream::TokenTree::Token(token) => TokenTree::Token(token), + + // `tree` is the beginning of a delimited set of tokens (e.g., `(` or `{`). We need to + // descend into the delimited set and further parse it. + tokenstream::TokenTree::Delimited(span, delim, tts) => TokenTree::Delimited( + span, + Lrc::new(Delimited { + delim, + tts: parse( + tts.into(), + expect_matchers, + sess, + ), + }), + ), + } +} + +/// Takes a token and returns `Some(KleeneOp)` if the token is `+` `*` or `?`. Otherwise, return +/// `None`. +fn kleene_op(token: &Token) -> Option { + match token.kind { + token::BinOp(token::Star) => Some(KleeneOp::ZeroOrMore), + token::BinOp(token::Plus) => Some(KleeneOp::OneOrMore), + token::Question => Some(KleeneOp::ZeroOrOne), + _ => None, + } +} + +/// Parse the next token tree of the input looking for a KleeneOp. Returns +/// +/// - Ok(Ok((op, span))) if the next token tree is a KleeneOp +/// - Ok(Err(tok, span)) if the next token tree is a token but not a KleeneOp +/// - Err(span) if the next token tree is not a token +fn parse_kleene_op( + input: &mut impl Iterator, + span: Span, +) -> Result, Span> { + match input.next() { + Some(tokenstream::TokenTree::Token(token)) => match kleene_op(&token) { + Some(op) => Ok(Ok((op, token.span))), + None => Ok(Err(token)), + }, + tree => Err(tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span)), + } +} + +/// Attempt to parse a single Kleene star, possibly with a separator. +/// +/// For example, in a pattern such as `$(a),*`, `a` is the pattern to be repeated, `,` is the +/// separator, and `*` is the Kleene operator. This function is specifically concerned with parsing +/// the last two tokens of such a pattern: namely, the optional separator and the Kleene operator +/// itself. Note that here we are parsing the _macro_ itself, rather than trying to match some +/// stream of tokens in an invocation of a macro. +/// +/// This function will take some input iterator `input` corresponding to `span` and a parsing +/// session `sess`. If the next one (or possibly two) tokens in `input` correspond to a Kleene +/// operator and separator, then a tuple with `(separator, KleeneOp)` is returned. Otherwise, an +/// error with the appropriate span is emitted to `sess` and a dummy value is returned. +fn parse_sep_and_kleene_op( + input: &mut impl Iterator, + span: Span, + sess: &ParseSess, +) -> (Option, KleeneToken) { + // We basically look at two token trees here, denoted as #1 and #2 below + let span = match parse_kleene_op(input, span) { + // #1 is a `?`, `+`, or `*` KleeneOp + Ok(Ok((op, span))) => return (None, KleeneToken::new(op, span)), + + // #1 is a separator followed by #2, a KleeneOp + Ok(Err(token)) => match parse_kleene_op(input, token.span) { + // #2 is the `?` Kleene op, which does not take a separator (error) + Ok(Ok((KleeneOp::ZeroOrOne, span))) => { + // Error! + sess.span_diagnostic.span_err( + token.span, + "the `?` macro repetition operator does not take a separator", + ); + + // Return a dummy + return (None, KleeneToken::new(KleeneOp::ZeroOrMore, span)); + } + + // #2 is a KleeneOp :D + Ok(Ok((op, span))) => return (Some(token), KleeneToken::new(op, span)), + + // #2 is a random token or not a token at all :( + Ok(Err(Token { span, .. })) | Err(span) => span, + }, + + // #1 is not a token + Err(span) => span, + }; + + // If we ever get to this point, we have experienced an "unexpected token" error + sess.span_diagnostic.span_err(span, "expected one of: `*`, `+`, or `?`"); + + // Return a dummy + (None, KleeneToken::new(KleeneOp::ZeroOrMore, span)) +} diff --git a/src/libsyntax_expand/mbe/transcribe.rs b/src/libsyntax_expand/mbe/transcribe.rs new file mode 100644 index 00000000000..94523bbf91b --- /dev/null +++ b/src/libsyntax_expand/mbe/transcribe.rs @@ -0,0 +1,399 @@ +use crate::base::ExtCtxt; +use crate::mbe; +use crate::mbe::macro_parser::{MatchedNonterminal, MatchedSeq, NamedMatch}; + +use syntax::ast::{Ident, Mac}; +use syntax::mut_visit::{self, MutVisitor}; +use syntax::parse::token::{self, NtTT, Token}; +use syntax::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndJoint}; + +use smallvec::{smallvec, SmallVec}; + +use errors::pluralise; +use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::sync::Lrc; +use syntax_pos::hygiene::{ExpnId, Transparency}; +use syntax_pos::Span; + +use std::mem; + +// A Marker adds the given mark to the syntax context. +struct Marker(ExpnId, Transparency); + +impl MutVisitor for Marker { + fn visit_span(&mut self, span: &mut Span) { + *span = span.apply_mark(self.0, self.1) + } + + fn visit_mac(&mut self, mac: &mut Mac) { + mut_visit::noop_visit_mac(mac, self) + } +} + +impl Marker { + fn visit_delim_span(&mut self, dspan: &mut DelimSpan) { + self.visit_span(&mut dspan.open); + self.visit_span(&mut dspan.close); + } +} + +/// An iterator over the token trees in a delimited token tree (`{ ... }`) or a sequence (`$(...)`). +enum Frame { + Delimited { forest: Lrc, idx: usize, span: DelimSpan }, + Sequence { forest: Lrc, idx: usize, sep: Option }, +} + +impl Frame { + /// Construct a new frame around the delimited set of tokens. + fn new(tts: Vec) -> Frame { + let forest = Lrc::new(mbe::Delimited { delim: token::NoDelim, tts }); + Frame::Delimited { forest, idx: 0, span: DelimSpan::dummy() } + } +} + +impl Iterator for Frame { + type Item = mbe::TokenTree; + + fn next(&mut self) -> Option { + match *self { + Frame::Delimited { ref forest, ref mut idx, .. } => { + *idx += 1; + forest.tts.get(*idx - 1).cloned() + } + Frame::Sequence { ref forest, ref mut idx, .. } => { + *idx += 1; + forest.tts.get(*idx - 1).cloned() + } + } + } +} + +/// This can do Macro-By-Example transcription. +/// - `interp` is a map of meta-variables to the tokens (non-terminals) they matched in the +/// invocation. We are assuming we already know there is a match. +/// - `src` is the RHS of the MBE, that is, the "example" we are filling in. +/// +/// For example, +/// +/// ```rust +/// macro_rules! foo { +/// ($id:ident) => { println!("{}", stringify!($id)); } +/// } +/// +/// foo!(bar); +/// ``` +/// +/// `interp` would contain `$id => bar` and `src` would contain `println!("{}", stringify!($id));`. +/// +/// `transcribe` would return a `TokenStream` containing `println!("{}", stringify!(bar));`. +/// +/// Along the way, we do some additional error checking. +pub(super) fn transcribe( + cx: &ExtCtxt<'_>, + interp: &FxHashMap, + src: Vec, + transparency: Transparency, +) -> TokenStream { + // Nothing for us to transcribe... + if src.is_empty() { + return TokenStream::default(); + } + + // We descend into the RHS (`src`), expanding things as we go. This stack contains the things + // we have yet to expand/are still expanding. We start the stack off with the whole RHS. + let mut stack: SmallVec<[Frame; 1]> = smallvec![Frame::new(src)]; + + // As we descend in the RHS, we will need to be able to match nested sequences of matchers. + // `repeats` keeps track of where we are in matching at each level, with the last element being + // the most deeply nested sequence. This is used as a stack. + let mut repeats = Vec::new(); + + // `result` contains resulting token stream from the TokenTree we just finished processing. At + // the end, this will contain the full result of transcription, but at arbitrary points during + // `transcribe`, `result` will contain subsets of the final result. + // + // Specifically, as we descend into each TokenTree, we will push the existing results onto the + // `result_stack` and clear `results`. We will then produce the results of transcribing the + // TokenTree into `results`. Then, as we unwind back out of the `TokenTree`, we will pop the + // `result_stack` and append `results` too it to produce the new `results` up to that point. + // + // Thus, if we try to pop the `result_stack` and it is empty, we have reached the top-level + // again, and we are done transcribing. + let mut result: Vec = Vec::new(); + let mut result_stack = Vec::new(); + let mut marker = Marker(cx.current_expansion.id, transparency); + + loop { + // Look at the last frame on the stack. + let tree = if let Some(tree) = stack.last_mut().unwrap().next() { + // If it still has a TokenTree we have not looked at yet, use that tree. + tree + } + // The else-case never produces a value for `tree` (it `continue`s or `return`s). + else { + // Otherwise, if we have just reached the end of a sequence and we can keep repeating, + // go back to the beginning of the sequence. + if let Frame::Sequence { idx, sep, .. } = stack.last_mut().unwrap() { + let (repeat_idx, repeat_len) = repeats.last_mut().unwrap(); + *repeat_idx += 1; + if repeat_idx < repeat_len { + *idx = 0; + if let Some(sep) = sep { + result.push(TokenTree::Token(sep.clone()).into()); + } + continue; + } + } + + // We are done with the top of the stack. Pop it. Depending on what it was, we do + // different things. Note that the outermost item must be the delimited, wrapped RHS + // that was passed in originally to `transcribe`. + match stack.pop().unwrap() { + // Done with a sequence. Pop from repeats. + Frame::Sequence { .. } => { + repeats.pop(); + } + + // We are done processing a Delimited. If this is the top-level delimited, we are + // done. Otherwise, we unwind the result_stack to append what we have produced to + // any previous results. + Frame::Delimited { forest, span, .. } => { + if result_stack.is_empty() { + // No results left to compute! We are back at the top-level. + return TokenStream::new(result); + } + + // Step back into the parent Delimited. + let tree = + TokenTree::Delimited(span, forest.delim, TokenStream::new(result).into()); + result = result_stack.pop().unwrap(); + result.push(tree.into()); + } + } + continue; + }; + + // At this point, we know we are in the middle of a TokenTree (the last one on `stack`). + // `tree` contains the next `TokenTree` to be processed. + match tree { + // We are descending into a sequence. We first make sure that the matchers in the RHS + // and the matches in `interp` have the same shape. Otherwise, either the caller or the + // macro writer has made a mistake. + seq @ mbe::TokenTree::Sequence(..) => { + match lockstep_iter_size(&seq, interp, &repeats) { + LockstepIterSize::Unconstrained => { + cx.span_fatal( + seq.span(), /* blame macro writer */ + "attempted to repeat an expression containing no syntax variables \ + matched as repeating at this depth", + ); + } + + LockstepIterSize::Contradiction(ref msg) => { + // FIXME: this really ought to be caught at macro definition time... It + // happens when two meta-variables are used in the same repetition in a + // sequence, but they come from different sequence matchers and repeat + // different amounts. + cx.span_fatal(seq.span(), &msg[..]); + } + + LockstepIterSize::Constraint(len, _) => { + // We do this to avoid an extra clone above. We know that this is a + // sequence already. + let (sp, seq) = if let mbe::TokenTree::Sequence(sp, seq) = seq { + (sp, seq) + } else { + unreachable!() + }; + + // Is the repetition empty? + if len == 0 { + if seq.kleene.op == mbe::KleeneOp::OneOrMore { + // FIXME: this really ought to be caught at macro definition + // time... It happens when the Kleene operator in the matcher and + // the body for the same meta-variable do not match. + cx.span_fatal(sp.entire(), "this must repeat at least once"); + } + } else { + // 0 is the initial counter (we have done 0 repretitions so far). `len` + // is the total number of reptitions we should generate. + repeats.push((0, len)); + + // The first time we encounter the sequence we push it to the stack. It + // then gets reused (see the beginning of the loop) until we are done + // repeating. + stack.push(Frame::Sequence { + idx: 0, + sep: seq.separator.clone(), + forest: seq, + }); + } + } + } + } + + // Replace the meta-var with the matched token tree from the invocation. + mbe::TokenTree::MetaVar(mut sp, mut ident) => { + // Find the matched nonterminal from the macro invocation, and use it to replace + // the meta-var. + if let Some(cur_matched) = lookup_cur_matched(ident, interp, &repeats) { + if let MatchedNonterminal(ref nt) = cur_matched { + // FIXME #2887: why do we apply a mark when matching a token tree meta-var + // (e.g. `$x:tt`), but not when we are matching any other type of token + // tree? + if let NtTT(ref tt) = **nt { + result.push(tt.clone().into()); + } else { + marker.visit_span(&mut sp); + let token = TokenTree::token(token::Interpolated(nt.clone()), sp); + result.push(token.into()); + } + } else { + // We were unable to descend far enough. This is an error. + cx.span_fatal( + sp, /* blame the macro writer */ + &format!("variable '{}' is still repeating at this depth", ident), + ); + } + } else { + // If we aren't able to match the meta-var, we push it back into the result but + // with modified syntax context. (I believe this supports nested macros). + marker.visit_span(&mut sp); + marker.visit_ident(&mut ident); + result.push(TokenTree::token(token::Dollar, sp).into()); + result.push(TokenTree::Token(Token::from_ast_ident(ident)).into()); + } + } + + // If we are entering a new delimiter, we push its contents to the `stack` to be + // processed, and we push all of the currently produced results to the `result_stack`. + // We will produce all of the results of the inside of the `Delimited` and then we will + // jump back out of the Delimited, pop the result_stack and add the new results back to + // the previous results (from outside the Delimited). + mbe::TokenTree::Delimited(mut span, delimited) => { + marker.visit_delim_span(&mut span); + stack.push(Frame::Delimited { forest: delimited, idx: 0, span }); + result_stack.push(mem::take(&mut result)); + } + + // Nothing much to do here. Just push the token to the result, being careful to + // preserve syntax context. + mbe::TokenTree::Token(token) => { + let mut tt = TokenTree::Token(token); + marker.visit_tt(&mut tt); + result.push(tt.into()); + } + + // There should be no meta-var declarations in the invocation of a macro. + mbe::TokenTree::MetaVarDecl(..) => panic!("unexpected `TokenTree::MetaVarDecl"), + } + } +} + +/// Lookup the meta-var named `ident` and return the matched token tree from the invocation using +/// the set of matches `interpolations`. +/// +/// See the definition of `repeats` in the `transcribe` function. `repeats` is used to descend +/// into the right place in nested matchers. If we attempt to descend too far, the macro writer has +/// made a mistake, and we return `None`. +fn lookup_cur_matched<'a>( + ident: Ident, + interpolations: &'a FxHashMap, + repeats: &[(usize, usize)], +) -> Option<&'a NamedMatch> { + interpolations.get(&ident).map(|matched| { + let mut matched = matched; + for &(idx, _) in repeats { + match matched { + MatchedNonterminal(_) => break, + MatchedSeq(ref ads, _) => matched = ads.get(idx).unwrap(), + } + } + + matched + }) +} + +/// An accumulator over a TokenTree to be used with `fold`. During transcription, we need to make +/// sure that the size of each sequence and all of its nested sequences are the same as the sizes +/// of all the matched (nested) sequences in the macro invocation. If they don't match, somebody +/// has made a mistake (either the macro writer or caller). +#[derive(Clone)] +enum LockstepIterSize { + /// No constraints on length of matcher. This is true for any TokenTree variants except a + /// `MetaVar` with an actual `MatchedSeq` (as opposed to a `MatchedNonterminal`). + Unconstrained, + + /// A `MetaVar` with an actual `MatchedSeq`. The length of the match and the name of the + /// meta-var are returned. + Constraint(usize, Ident), + + /// Two `Constraint`s on the same sequence had different lengths. This is an error. + Contradiction(String), +} + +impl LockstepIterSize { + /// Find incompatibilities in matcher/invocation sizes. + /// - `Unconstrained` is compatible with everything. + /// - `Contradiction` is incompatible with everything. + /// - `Constraint(len)` is only compatible with other constraints of the same length. + fn with(self, other: LockstepIterSize) -> LockstepIterSize { + match self { + LockstepIterSize::Unconstrained => other, + LockstepIterSize::Contradiction(_) => self, + LockstepIterSize::Constraint(l_len, ref l_id) => match other { + LockstepIterSize::Unconstrained => self, + LockstepIterSize::Contradiction(_) => other, + LockstepIterSize::Constraint(r_len, _) if l_len == r_len => self, + LockstepIterSize::Constraint(r_len, r_id) => { + let msg = format!( + "meta-variable `{}` repeats {} time{}, but `{}` repeats {} time{}", + l_id, + l_len, + pluralise!(l_len), + r_id, + r_len, + pluralise!(r_len), + ); + LockstepIterSize::Contradiction(msg) + } + }, + } + } +} + +/// Given a `tree`, make sure that all sequences have the same length as the matches for the +/// appropriate meta-vars in `interpolations`. +/// +/// Note that if `repeats` does not match the exact correct depth of a meta-var, +/// `lookup_cur_matched` will return `None`, which is why this still works even in the presnece of +/// multiple nested matcher sequences. +fn lockstep_iter_size( + tree: &mbe::TokenTree, + interpolations: &FxHashMap, + repeats: &[(usize, usize)], +) -> LockstepIterSize { + use mbe::TokenTree; + match *tree { + TokenTree::Delimited(_, ref delimed) => { + delimed.tts.iter().fold(LockstepIterSize::Unconstrained, |size, tt| { + size.with(lockstep_iter_size(tt, interpolations, repeats)) + }) + } + TokenTree::Sequence(_, ref seq) => { + seq.tts.iter().fold(LockstepIterSize::Unconstrained, |size, tt| { + size.with(lockstep_iter_size(tt, interpolations, repeats)) + }) + } + TokenTree::MetaVar(_, name) | TokenTree::MetaVarDecl(_, name, _) => { + match lookup_cur_matched(name, interpolations, repeats) { + Some(matched) => match matched { + MatchedNonterminal(_) => LockstepIterSize::Unconstrained, + MatchedSeq(ref ads, _) => LockstepIterSize::Constraint(ads.len(), name), + }, + _ => LockstepIterSize::Unconstrained, + } + } + TokenTree::Token(..) => LockstepIterSize::Unconstrained, + } +} diff --git a/src/libsyntax_expand/placeholders.rs b/src/libsyntax_expand/placeholders.rs new file mode 100644 index 00000000000..f2c89e14b53 --- /dev/null +++ b/src/libsyntax_expand/placeholders.rs @@ -0,0 +1,350 @@ +use crate::base::ExtCtxt; +use crate::expand::{AstFragment, AstFragmentKind}; + +use syntax::ast::{self, NodeId}; +use syntax::source_map::{DUMMY_SP, dummy_spanned}; +use syntax::tokenstream::TokenStream; +use syntax::mut_visit::*; +use syntax::ptr::P; +use syntax::ThinVec; + +use smallvec::{smallvec, SmallVec}; + +use rustc_data_structures::fx::FxHashMap; + +pub fn placeholder(kind: AstFragmentKind, id: ast::NodeId) -> AstFragment { + fn mac_placeholder() -> ast::Mac { + ast::Mac { + path: ast::Path { span: DUMMY_SP, segments: Vec::new() }, + tts: TokenStream::default().into(), + delim: ast::MacDelimiter::Brace, + span: DUMMY_SP, + prior_type_ascription: None, + } + } + + let ident = ast::Ident::invalid(); + let attrs = Vec::new(); + let generics = ast::Generics::default(); + let vis = dummy_spanned(ast::VisibilityKind::Inherited); + let span = DUMMY_SP; + let expr_placeholder = || P(ast::Expr { + id, span, + attrs: ThinVec::new(), + kind: ast::ExprKind::Mac(mac_placeholder()), + }); + let ty = || P(ast::Ty { + id, + kind: ast::TyKind::Mac(mac_placeholder()), + span, + }); + let pat = || P(ast::Pat { + id, + kind: ast::PatKind::Mac(mac_placeholder()), + span, + }); + + match kind { + AstFragmentKind::Expr => AstFragment::Expr(expr_placeholder()), + AstFragmentKind::OptExpr => AstFragment::OptExpr(Some(expr_placeholder())), + AstFragmentKind::Items => AstFragment::Items(smallvec![P(ast::Item { + id, span, ident, vis, attrs, + kind: ast::ItemKind::Mac(mac_placeholder()), + tokens: None, + })]), + AstFragmentKind::TraitItems => AstFragment::TraitItems(smallvec![ast::TraitItem { + id, span, ident, attrs, generics, + kind: ast::TraitItemKind::Macro(mac_placeholder()), + tokens: None, + }]), + AstFragmentKind::ImplItems => AstFragment::ImplItems(smallvec![ast::ImplItem { + id, span, ident, vis, attrs, generics, + kind: ast::ImplItemKind::Macro(mac_placeholder()), + defaultness: ast::Defaultness::Final, + tokens: None, + }]), + AstFragmentKind::ForeignItems => + AstFragment::ForeignItems(smallvec![ast::ForeignItem { + id, span, ident, vis, attrs, + kind: ast::ForeignItemKind::Macro(mac_placeholder()), + }]), + AstFragmentKind::Pat => AstFragment::Pat(P(ast::Pat { + id, span, kind: ast::PatKind::Mac(mac_placeholder()), + })), + AstFragmentKind::Ty => AstFragment::Ty(P(ast::Ty { + id, span, kind: ast::TyKind::Mac(mac_placeholder()), + })), + AstFragmentKind::Stmts => AstFragment::Stmts(smallvec![{ + let mac = P((mac_placeholder(), ast::MacStmtStyle::Braces, ThinVec::new())); + ast::Stmt { id, span, kind: ast::StmtKind::Mac(mac) } + }]), + AstFragmentKind::Arms => AstFragment::Arms(smallvec![ + ast::Arm { + attrs: Default::default(), + body: expr_placeholder(), + guard: None, + id, + pat: pat(), + span, + is_placeholder: true, + } + ]), + AstFragmentKind::Fields => AstFragment::Fields(smallvec![ + ast::Field { + attrs: Default::default(), + expr: expr_placeholder(), + id, + ident, + is_shorthand: false, + span, + is_placeholder: true, + } + ]), + AstFragmentKind::FieldPats => AstFragment::FieldPats(smallvec![ + ast::FieldPat { + attrs: Default::default(), + id, + ident, + is_shorthand: false, + pat: pat(), + span, + is_placeholder: true, + } + ]), + AstFragmentKind::GenericParams => AstFragment::GenericParams(smallvec![{ + ast::GenericParam { + attrs: Default::default(), + bounds: Default::default(), + id, + ident, + is_placeholder: true, + kind: ast::GenericParamKind::Lifetime, + } + }]), + AstFragmentKind::Params => AstFragment::Params(smallvec![ + ast::Param { + attrs: Default::default(), + id, + pat: pat(), + span, + ty: ty(), + is_placeholder: true, + } + ]), + AstFragmentKind::StructFields => AstFragment::StructFields(smallvec![ + ast::StructField { + attrs: Default::default(), + id, + ident: None, + span, + ty: ty(), + vis, + is_placeholder: true, + } + ]), + AstFragmentKind::Variants => AstFragment::Variants(smallvec![ + ast::Variant { + attrs: Default::default(), + data: ast::VariantData::Struct(Default::default(), false), + disr_expr: None, + id, + ident, + span, + is_placeholder: true, + } + ]) + } +} + +pub struct PlaceholderExpander<'a, 'b> { + expanded_fragments: FxHashMap, + cx: &'a mut ExtCtxt<'b>, + monotonic: bool, +} + +impl<'a, 'b> PlaceholderExpander<'a, 'b> { + pub fn new(cx: &'a mut ExtCtxt<'b>, monotonic: bool) -> Self { + PlaceholderExpander { + cx, + expanded_fragments: FxHashMap::default(), + monotonic, + } + } + + pub fn add(&mut self, id: ast::NodeId, mut fragment: AstFragment, placeholders: Vec) { + fragment.mut_visit_with(self); + if let AstFragment::Items(mut items) = fragment { + for placeholder in placeholders { + match self.remove(placeholder) { + AstFragment::Items(derived_items) => items.extend(derived_items), + _ => unreachable!(), + } + } + fragment = AstFragment::Items(items); + } + self.expanded_fragments.insert(id, fragment); + } + + fn remove(&mut self, id: ast::NodeId) -> AstFragment { + self.expanded_fragments.remove(&id).unwrap() + } +} + +impl<'a, 'b> MutVisitor for PlaceholderExpander<'a, 'b> { + fn flat_map_arm(&mut self, arm: ast::Arm) -> SmallVec<[ast::Arm; 1]> { + if arm.is_placeholder { + self.remove(arm.id).make_arms() + } else { + noop_flat_map_arm(arm, self) + } + } + + fn flat_map_field(&mut self, field: ast::Field) -> SmallVec<[ast::Field; 1]> { + if field.is_placeholder { + self.remove(field.id).make_fields() + } else { + noop_flat_map_field(field, self) + } + } + + fn flat_map_field_pattern(&mut self, fp: ast::FieldPat) -> SmallVec<[ast::FieldPat; 1]> { + if fp.is_placeholder { + self.remove(fp.id).make_field_patterns() + } else { + noop_flat_map_field_pattern(fp, self) + } + } + + fn flat_map_generic_param( + &mut self, + param: ast::GenericParam + ) -> SmallVec<[ast::GenericParam; 1]> + { + if param.is_placeholder { + self.remove(param.id).make_generic_params() + } else { + noop_flat_map_generic_param(param, self) + } + } + + fn flat_map_param(&mut self, p: ast::Param) -> SmallVec<[ast::Param; 1]> { + if p.is_placeholder { + self.remove(p.id).make_params() + } else { + noop_flat_map_param(p, self) + } + } + + fn flat_map_struct_field(&mut self, sf: ast::StructField) -> SmallVec<[ast::StructField; 1]> { + if sf.is_placeholder { + self.remove(sf.id).make_struct_fields() + } else { + noop_flat_map_struct_field(sf, self) + } + } + + fn flat_map_variant(&mut self, variant: ast::Variant) -> SmallVec<[ast::Variant; 1]> { + if variant.is_placeholder { + self.remove(variant.id).make_variants() + } else { + noop_flat_map_variant(variant, self) + } + } + + fn flat_map_item(&mut self, item: P) -> SmallVec<[P; 1]> { + match item.kind { + ast::ItemKind::Mac(_) => return self.remove(item.id).make_items(), + ast::ItemKind::MacroDef(_) => return smallvec![item], + _ => {} + } + + noop_flat_map_item(item, self) + } + + fn flat_map_trait_item(&mut self, item: ast::TraitItem) -> SmallVec<[ast::TraitItem; 1]> { + match item.kind { + ast::TraitItemKind::Macro(_) => self.remove(item.id).make_trait_items(), + _ => noop_flat_map_trait_item(item, self), + } + } + + fn flat_map_impl_item(&mut self, item: ast::ImplItem) -> SmallVec<[ast::ImplItem; 1]> { + match item.kind { + ast::ImplItemKind::Macro(_) => self.remove(item.id).make_impl_items(), + _ => noop_flat_map_impl_item(item, self), + } + } + + fn flat_map_foreign_item(&mut self, item: ast::ForeignItem) -> SmallVec<[ast::ForeignItem; 1]> { + match item.kind { + ast::ForeignItemKind::Macro(_) => self.remove(item.id).make_foreign_items(), + _ => noop_flat_map_foreign_item(item, self), + } + } + + fn visit_expr(&mut self, expr: &mut P) { + match expr.kind { + ast::ExprKind::Mac(_) => *expr = self.remove(expr.id).make_expr(), + _ => noop_visit_expr(expr, self), + } + } + + fn filter_map_expr(&mut self, expr: P) -> Option> { + match expr.kind { + ast::ExprKind::Mac(_) => self.remove(expr.id).make_opt_expr(), + _ => noop_filter_map_expr(expr, self), + } + } + + fn flat_map_stmt(&mut self, stmt: ast::Stmt) -> SmallVec<[ast::Stmt; 1]> { + let (style, mut stmts) = match stmt.kind { + ast::StmtKind::Mac(mac) => (mac.1, self.remove(stmt.id).make_stmts()), + _ => return noop_flat_map_stmt(stmt, self), + }; + + if style == ast::MacStmtStyle::Semicolon { + if let Some(stmt) = stmts.pop() { + stmts.push(stmt.add_trailing_semicolon()); + } + } + + stmts + } + + fn visit_pat(&mut self, pat: &mut P) { + match pat.kind { + ast::PatKind::Mac(_) => *pat = self.remove(pat.id).make_pat(), + _ => noop_visit_pat(pat, self), + } + } + + fn visit_ty(&mut self, ty: &mut P) { + match ty.kind { + ast::TyKind::Mac(_) => *ty = self.remove(ty.id).make_ty(), + _ => noop_visit_ty(ty, self), + } + } + + fn visit_block(&mut self, block: &mut P) { + noop_visit_block(block, self); + + for stmt in block.stmts.iter_mut() { + if self.monotonic { + assert_eq!(stmt.id, ast::DUMMY_NODE_ID); + stmt.id = self.cx.resolver.next_node_id(); + } + } + } + + fn visit_mod(&mut self, module: &mut ast::Mod) { + noop_visit_mod(module, self); + module.items.retain(|item| match item.kind { + ast::ItemKind::Mac(_) if !self.cx.ecfg.keep_macs => false, // remove macro definitions + _ => true, + }); + } + + fn visit_mac(&mut self, _mac: &mut ast::Mac) { + // Do nothing. + } +} diff --git a/src/libsyntax_expand/proc_macro.rs b/src/libsyntax_expand/proc_macro.rs new file mode 100644 index 00000000000..07b618c99a5 --- /dev/null +++ b/src/libsyntax_expand/proc_macro.rs @@ -0,0 +1,215 @@ +use crate::base::{self, *}; +use crate::proc_macro_server; + +use syntax::ast::{self, ItemKind, Attribute, Mac}; +use syntax::attr::{mark_used, mark_known}; +use syntax::errors::{Applicability, FatalError}; +use syntax::parse::{self, token}; +use syntax::symbol::sym; +use syntax::tokenstream::{self, TokenStream}; +use syntax::visit::Visitor; + +use rustc_data_structures::sync::Lrc; +use syntax_pos::{Span, DUMMY_SP}; + +const EXEC_STRATEGY: pm::bridge::server::SameThread = pm::bridge::server::SameThread; + +pub struct BangProcMacro { + pub client: pm::bridge::client::Client< + fn(pm::TokenStream) -> pm::TokenStream, + >, +} + +impl base::ProcMacro for BangProcMacro { + fn expand<'cx>(&self, + ecx: &'cx mut ExtCtxt<'_>, + span: Span, + input: TokenStream) + -> TokenStream { + let server = proc_macro_server::Rustc::new(ecx); + match self.client.run(&EXEC_STRATEGY, server, input) { + Ok(stream) => stream, + Err(e) => { + let msg = "proc macro panicked"; + let mut err = ecx.struct_span_fatal(span, msg); + if let Some(s) = e.as_str() { + err.help(&format!("message: {}", s)); + } + + err.emit(); + FatalError.raise(); + } + } + } +} + +pub struct AttrProcMacro { + pub client: pm::bridge::client::Client pm::TokenStream>, +} + +impl base::AttrProcMacro for AttrProcMacro { + fn expand<'cx>(&self, + ecx: &'cx mut ExtCtxt<'_>, + span: Span, + annotation: TokenStream, + annotated: TokenStream) + -> TokenStream { + let server = proc_macro_server::Rustc::new(ecx); + match self.client.run(&EXEC_STRATEGY, server, annotation, annotated) { + Ok(stream) => stream, + Err(e) => { + let msg = "custom attribute panicked"; + let mut err = ecx.struct_span_fatal(span, msg); + if let Some(s) = e.as_str() { + err.help(&format!("message: {}", s)); + } + + err.emit(); + FatalError.raise(); + } + } + } +} + +pub struct ProcMacroDerive { + pub client: pm::bridge::client::Client pm::TokenStream>, +} + +impl MultiItemModifier for ProcMacroDerive { + fn expand(&self, + ecx: &mut ExtCtxt<'_>, + span: Span, + _meta_item: &ast::MetaItem, + item: Annotatable) + -> Vec { + let item = match item { + Annotatable::Arm(..) | + Annotatable::Field(..) | + Annotatable::FieldPat(..) | + Annotatable::GenericParam(..) | + Annotatable::Param(..) | + Annotatable::StructField(..) | + Annotatable::Variant(..) + => panic!("unexpected annotatable"), + Annotatable::Item(item) => item, + Annotatable::ImplItem(_) | + Annotatable::TraitItem(_) | + Annotatable::ForeignItem(_) | + Annotatable::Stmt(_) | + Annotatable::Expr(_) => { + ecx.span_err(span, "proc-macro derives may only be \ + applied to a struct, enum, or union"); + return Vec::new() + } + }; + match item.kind { + ItemKind::Struct(..) | + ItemKind::Enum(..) | + ItemKind::Union(..) => {}, + _ => { + ecx.span_err(span, "proc-macro derives may only be \ + applied to a struct, enum, or union"); + return Vec::new() + } + } + + let token = token::Interpolated(Lrc::new(token::NtItem(item))); + let input = tokenstream::TokenTree::token(token, DUMMY_SP).into(); + + let server = proc_macro_server::Rustc::new(ecx); + let stream = match self.client.run(&EXEC_STRATEGY, server, input) { + Ok(stream) => stream, + Err(e) => { + let msg = "proc-macro derive panicked"; + let mut err = ecx.struct_span_fatal(span, msg); + if let Some(s) = e.as_str() { + err.help(&format!("message: {}", s)); + } + + err.emit(); + FatalError.raise(); + } + }; + + let error_count_before = ecx.parse_sess.span_diagnostic.err_count(); + let msg = "proc-macro derive produced unparseable tokens"; + + let mut parser = parse::stream_to_parser(ecx.parse_sess, stream, Some("proc-macro derive")); + let mut items = vec![]; + + loop { + match parser.parse_item() { + Ok(None) => break, + Ok(Some(item)) => { + items.push(Annotatable::Item(item)) + } + Err(mut err) => { + // FIXME: handle this better + err.cancel(); + ecx.struct_span_fatal(span, msg).emit(); + FatalError.raise(); + } + } + } + + + // fail if there have been errors emitted + if ecx.parse_sess.span_diagnostic.err_count() > error_count_before { + ecx.struct_span_fatal(span, msg).emit(); + FatalError.raise(); + } + + items + } +} + +crate struct MarkAttrs<'a>(crate &'a [ast::Name]); + +impl<'a> Visitor<'a> for MarkAttrs<'a> { + fn visit_attribute(&mut self, attr: &Attribute) { + if let Some(ident) = attr.ident() { + if self.0.contains(&ident.name) { + mark_used(attr); + mark_known(attr); + } + } + } + + fn visit_mac(&mut self, _mac: &Mac) {} +} + +pub fn is_proc_macro_attr(attr: &Attribute) -> bool { + [sym::proc_macro, sym::proc_macro_attribute, sym::proc_macro_derive] + .iter().any(|kind| attr.check_name(*kind)) +} + +crate fn collect_derives(cx: &mut ExtCtxt<'_>, attrs: &mut Vec) -> Vec { + let mut result = Vec::new(); + attrs.retain(|attr| { + if attr.path != sym::derive { + return true; + } + if !attr.is_meta_item_list() { + cx.struct_span_err(attr.span, "malformed `derive` attribute input") + .span_suggestion( + attr.span, + "missing traits to be derived", + "#[derive(Trait1, Trait2, ...)]".to_owned(), + Applicability::HasPlaceholders, + ).emit(); + return false; + } + + match attr.parse_derive_paths(cx.parse_sess) { + Ok(traits) => { + result.extend(traits); + true + } + Err(mut e) => { + e.emit(); + false + } + } + }); + result +} diff --git a/src/libsyntax_expand/proc_macro_server.rs b/src/libsyntax_expand/proc_macro_server.rs new file mode 100644 index 00000000000..4ce99cfe73b --- /dev/null +++ b/src/libsyntax_expand/proc_macro_server.rs @@ -0,0 +1,713 @@ +use crate::base::ExtCtxt; + +use syntax::ast; +use syntax::parse::{self, token}; +use syntax::parse::lexer::comments; +use syntax::print::pprust; +use syntax::sess::ParseSess; +use syntax::tokenstream::{self, DelimSpan, IsJoint::*, TokenStream, TreeAndJoint}; + +use errors::Diagnostic; +use rustc_data_structures::sync::Lrc; +use syntax_pos::{BytePos, FileName, MultiSpan, Pos, SourceFile, Span}; +use syntax_pos::symbol::{kw, sym, Symbol}; + +use pm::{Delimiter, Level, LineColumn, Spacing}; +use pm::bridge::{server, TokenTree}; +use std::{ascii, panic}; +use std::ops::Bound; + +trait FromInternal { + fn from_internal(x: T) -> Self; +} + +trait ToInternal { + fn to_internal(self) -> T; +} + +impl FromInternal for Delimiter { + fn from_internal(delim: token::DelimToken) -> Delimiter { + match delim { + token::Paren => Delimiter::Parenthesis, + token::Brace => Delimiter::Brace, + token::Bracket => Delimiter::Bracket, + token::NoDelim => Delimiter::None, + } + } +} + +impl ToInternal for Delimiter { + fn to_internal(self) -> token::DelimToken { + match self { + Delimiter::Parenthesis => token::Paren, + Delimiter::Brace => token::Brace, + Delimiter::Bracket => token::Bracket, + Delimiter::None => token::NoDelim, + } + } +} + +impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec)> + for TokenTree +{ + fn from_internal(((tree, is_joint), sess, stack): (TreeAndJoint, &ParseSess, &mut Vec)) + -> Self { + use syntax::parse::token::*; + + let joint = is_joint == Joint; + let Token { kind, span } = match tree { + tokenstream::TokenTree::Delimited(span, delim, tts) => { + let delimiter = Delimiter::from_internal(delim); + return TokenTree::Group(Group { + delimiter, + stream: tts.into(), + span, + }); + } + tokenstream::TokenTree::Token(token) => token, + }; + + macro_rules! tt { + ($ty:ident { $($field:ident $(: $value:expr)*),+ $(,)? }) => ( + TokenTree::$ty(self::$ty { + $($field $(: $value)*,)+ + span, + }) + ); + ($ty:ident::$method:ident($($value:expr),*)) => ( + TokenTree::$ty(self::$ty::$method($($value,)* span)) + ); + } + macro_rules! op { + ($a:expr) => { + tt!(Punct::new($a, joint)) + }; + ($a:expr, $b:expr) => {{ + stack.push(tt!(Punct::new($b, joint))); + tt!(Punct::new($a, true)) + }}; + ($a:expr, $b:expr, $c:expr) => {{ + stack.push(tt!(Punct::new($c, joint))); + stack.push(tt!(Punct::new($b, true))); + tt!(Punct::new($a, true)) + }}; + } + + match kind { + Eq => op!('='), + Lt => op!('<'), + Le => op!('<', '='), + EqEq => op!('=', '='), + Ne => op!('!', '='), + Ge => op!('>', '='), + Gt => op!('>'), + AndAnd => op!('&', '&'), + OrOr => op!('|', '|'), + Not => op!('!'), + Tilde => op!('~'), + BinOp(Plus) => op!('+'), + BinOp(Minus) => op!('-'), + BinOp(Star) => op!('*'), + BinOp(Slash) => op!('/'), + BinOp(Percent) => op!('%'), + BinOp(Caret) => op!('^'), + BinOp(And) => op!('&'), + BinOp(Or) => op!('|'), + BinOp(Shl) => op!('<', '<'), + BinOp(Shr) => op!('>', '>'), + BinOpEq(Plus) => op!('+', '='), + BinOpEq(Minus) => op!('-', '='), + BinOpEq(Star) => op!('*', '='), + BinOpEq(Slash) => op!('/', '='), + BinOpEq(Percent) => op!('%', '='), + BinOpEq(Caret) => op!('^', '='), + BinOpEq(And) => op!('&', '='), + BinOpEq(Or) => op!('|', '='), + BinOpEq(Shl) => op!('<', '<', '='), + BinOpEq(Shr) => op!('>', '>', '='), + At => op!('@'), + Dot => op!('.'), + DotDot => op!('.', '.'), + DotDotDot => op!('.', '.', '.'), + DotDotEq => op!('.', '.', '='), + Comma => op!(','), + Semi => op!(';'), + Colon => op!(':'), + ModSep => op!(':', ':'), + RArrow => op!('-', '>'), + LArrow => op!('<', '-'), + FatArrow => op!('=', '>'), + Pound => op!('#'), + Dollar => op!('$'), + Question => op!('?'), + SingleQuote => op!('\''), + + Ident(name, false) if name == kw::DollarCrate => tt!(Ident::dollar_crate()), + Ident(name, is_raw) => tt!(Ident::new(name, is_raw)), + Lifetime(name) => { + let ident = ast::Ident::new(name, span).without_first_quote(); + stack.push(tt!(Ident::new(ident.name, false))); + tt!(Punct::new('\'', true)) + } + Literal(lit) => tt!(Literal { lit }), + DocComment(c) => { + let style = comments::doc_comment_style(&c.as_str()); + let stripped = comments::strip_doc_comment_decoration(&c.as_str()); + let mut escaped = String::new(); + for ch in stripped.chars() { + escaped.extend(ch.escape_debug()); + } + let stream = vec![ + Ident(sym::doc, false), + Eq, + TokenKind::lit(token::Str, Symbol::intern(&escaped), None), + ] + .into_iter() + .map(|kind| tokenstream::TokenTree::token(kind, span)) + .collect(); + stack.push(TokenTree::Group(Group { + delimiter: Delimiter::Bracket, + stream, + span: DelimSpan::from_single(span), + })); + if style == ast::AttrStyle::Inner { + stack.push(tt!(Punct::new('!', false))); + } + tt!(Punct::new('#', false)) + } + + Interpolated(nt) => { + let stream = parse::nt_to_tokenstream(&nt, sess, span); + TokenTree::Group(Group { + delimiter: Delimiter::None, + stream, + span: DelimSpan::from_single(span), + }) + } + + OpenDelim(..) | CloseDelim(..) => unreachable!(), + Whitespace | Comment | Shebang(..) | Unknown(..) | Eof => unreachable!(), + } + } +} + +impl ToInternal for TokenTree { + fn to_internal(self) -> TokenStream { + use syntax::parse::token::*; + + let (ch, joint, span) = match self { + TokenTree::Punct(Punct { ch, joint, span }) => (ch, joint, span), + TokenTree::Group(Group { + delimiter, + stream, + span, + }) => { + return tokenstream::TokenTree::Delimited( + span, + delimiter.to_internal(), + stream.into(), + ) + .into(); + } + TokenTree::Ident(self::Ident { sym, is_raw, span }) => { + return tokenstream::TokenTree::token(Ident(sym, is_raw), span).into(); + } + TokenTree::Literal(self::Literal { + lit: token::Lit { kind: token::Integer, symbol, suffix }, + span, + }) if symbol.as_str().starts_with("-") => { + let minus = BinOp(BinOpToken::Minus); + let symbol = Symbol::intern(&symbol.as_str()[1..]); + let integer = TokenKind::lit(token::Integer, symbol, suffix); + let a = tokenstream::TokenTree::token(minus, span); + let b = tokenstream::TokenTree::token(integer, span); + return vec![a, b].into_iter().collect(); + } + TokenTree::Literal(self::Literal { + lit: token::Lit { kind: token::Float, symbol, suffix }, + span, + }) if symbol.as_str().starts_with("-") => { + let minus = BinOp(BinOpToken::Minus); + let symbol = Symbol::intern(&symbol.as_str()[1..]); + let float = TokenKind::lit(token::Float, symbol, suffix); + let a = tokenstream::TokenTree::token(minus, span); + let b = tokenstream::TokenTree::token(float, span); + return vec![a, b].into_iter().collect(); + } + TokenTree::Literal(self::Literal { lit, span }) => { + return tokenstream::TokenTree::token(Literal(lit), span).into() + } + }; + + let kind = match ch { + '=' => Eq, + '<' => Lt, + '>' => Gt, + '!' => Not, + '~' => Tilde, + '+' => BinOp(Plus), + '-' => BinOp(Minus), + '*' => BinOp(Star), + '/' => BinOp(Slash), + '%' => BinOp(Percent), + '^' => BinOp(Caret), + '&' => BinOp(And), + '|' => BinOp(Or), + '@' => At, + '.' => Dot, + ',' => Comma, + ';' => Semi, + ':' => Colon, + '#' => Pound, + '$' => Dollar, + '?' => Question, + '\'' => SingleQuote, + _ => unreachable!(), + }; + + let tree = tokenstream::TokenTree::token(kind, span); + TokenStream::new(vec![(tree, if joint { Joint } else { NonJoint })]) + } +} + +impl ToInternal for Level { + fn to_internal(self) -> errors::Level { + match self { + Level::Error => errors::Level::Error, + Level::Warning => errors::Level::Warning, + Level::Note => errors::Level::Note, + Level::Help => errors::Level::Help, + _ => unreachable!("unknown proc_macro::Level variant: {:?}", self), + } + } +} + +#[derive(Clone)] +pub struct TokenStreamIter { + cursor: tokenstream::Cursor, + stack: Vec>, +} + +#[derive(Clone)] +pub struct Group { + delimiter: Delimiter, + stream: TokenStream, + span: DelimSpan, +} + +#[derive(Copy, Clone, PartialEq, Eq, Hash)] +pub struct Punct { + ch: char, + // NB. not using `Spacing` here because it doesn't implement `Hash`. + joint: bool, + span: Span, +} + +impl Punct { + fn new(ch: char, joint: bool, span: Span) -> Punct { + const LEGAL_CHARS: &[char] = &['=', '<', '>', '!', '~', '+', '-', '*', '/', '%', '^', + '&', '|', '@', '.', ',', ';', ':', '#', '$', '?', '\'']; + if !LEGAL_CHARS.contains(&ch) { + panic!("unsupported character `{:?}`", ch) + } + Punct { ch, joint, span } + } +} + +#[derive(Copy, Clone, PartialEq, Eq, Hash)] +pub struct Ident { + sym: Symbol, + is_raw: bool, + span: Span, +} + +impl Ident { + fn is_valid(string: &str) -> bool { + let mut chars = string.chars(); + if let Some(start) = chars.next() { + rustc_lexer::is_id_start(start) && chars.all(rustc_lexer::is_id_continue) + } else { + false + } + } + fn new(sym: Symbol, is_raw: bool, span: Span) -> Ident { + let string = sym.as_str(); + if !Self::is_valid(&string) { + panic!("`{:?}` is not a valid identifier", string) + } + if is_raw && !sym.can_be_raw() { + panic!("`{}` cannot be a raw identifier", string); + } + Ident { sym, is_raw, span } + } + fn dollar_crate(span: Span) -> Ident { + // `$crate` is accepted as an ident only if it comes from the compiler. + Ident { sym: kw::DollarCrate, is_raw: false, span } + } +} + +// FIXME(eddyb) `Literal` should not expose internal `Debug` impls. +#[derive(Clone, Debug)] +pub struct Literal { + lit: token::Lit, + span: Span, +} + +pub(crate) struct Rustc<'a> { + sess: &'a ParseSess, + def_site: Span, + call_site: Span, + mixed_site: Span, +} + +impl<'a> Rustc<'a> { + pub fn new(cx: &'a ExtCtxt<'_>) -> Self { + let expn_data = cx.current_expansion.id.expn_data(); + Rustc { + sess: cx.parse_sess, + def_site: cx.with_def_site_ctxt(expn_data.def_site), + call_site: cx.with_call_site_ctxt(expn_data.call_site), + mixed_site: cx.with_mixed_site_ctxt(expn_data.call_site), + } + } + + fn lit(&mut self, kind: token::LitKind, symbol: Symbol, suffix: Option) -> Literal { + Literal { + lit: token::Lit::new(kind, symbol, suffix), + span: server::Span::call_site(self), + } + } +} + +impl server::Types for Rustc<'_> { + type TokenStream = TokenStream; + type TokenStreamBuilder = tokenstream::TokenStreamBuilder; + type TokenStreamIter = TokenStreamIter; + type Group = Group; + type Punct = Punct; + type Ident = Ident; + type Literal = Literal; + type SourceFile = Lrc; + type MultiSpan = Vec; + type Diagnostic = Diagnostic; + type Span = Span; +} + +impl server::TokenStream for Rustc<'_> { + fn new(&mut self) -> Self::TokenStream { + TokenStream::default() + } + fn is_empty(&mut self, stream: &Self::TokenStream) -> bool { + stream.is_empty() + } + fn from_str(&mut self, src: &str) -> Self::TokenStream { + parse::parse_stream_from_source_str( + FileName::proc_macro_source_code(src), + src.to_string(), + self.sess, + Some(self.call_site), + ) + } + fn to_string(&mut self, stream: &Self::TokenStream) -> String { + pprust::tts_to_string(stream.clone()) + } + fn from_token_tree( + &mut self, + tree: TokenTree, + ) -> Self::TokenStream { + tree.to_internal() + } + fn into_iter(&mut self, stream: Self::TokenStream) -> Self::TokenStreamIter { + TokenStreamIter { + cursor: stream.trees(), + stack: vec![], + } + } +} + +impl server::TokenStreamBuilder for Rustc<'_> { + fn new(&mut self) -> Self::TokenStreamBuilder { + tokenstream::TokenStreamBuilder::new() + } + fn push(&mut self, builder: &mut Self::TokenStreamBuilder, stream: Self::TokenStream) { + builder.push(stream); + } + fn build(&mut self, builder: Self::TokenStreamBuilder) -> Self::TokenStream { + builder.build() + } +} + +impl server::TokenStreamIter for Rustc<'_> { + fn next( + &mut self, + iter: &mut Self::TokenStreamIter, + ) -> Option> { + loop { + let tree = iter.stack.pop().or_else(|| { + let next = iter.cursor.next_with_joint()?; + Some(TokenTree::from_internal((next, self.sess, &mut iter.stack))) + })?; + // HACK: The condition "dummy span + group with empty delimiter" represents an AST + // fragment approximately converted into a token stream. This may happen, for + // example, with inputs to proc macro attributes, including derives. Such "groups" + // need to flattened during iteration over stream's token trees. + // Eventually this needs to be removed in favor of keeping original token trees + // and not doing the roundtrip through AST. + if let TokenTree::Group(ref group) = tree { + if group.delimiter == Delimiter::None && group.span.entire().is_dummy() { + iter.cursor.append(group.stream.clone()); + continue; + } + } + return Some(tree); + } + } +} + +impl server::Group for Rustc<'_> { + fn new(&mut self, delimiter: Delimiter, stream: Self::TokenStream) -> Self::Group { + Group { + delimiter, + stream, + span: DelimSpan::from_single(server::Span::call_site(self)), + } + } + fn delimiter(&mut self, group: &Self::Group) -> Delimiter { + group.delimiter + } + fn stream(&mut self, group: &Self::Group) -> Self::TokenStream { + group.stream.clone() + } + fn span(&mut self, group: &Self::Group) -> Self::Span { + group.span.entire() + } + fn span_open(&mut self, group: &Self::Group) -> Self::Span { + group.span.open + } + fn span_close(&mut self, group: &Self::Group) -> Self::Span { + group.span.close + } + fn set_span(&mut self, group: &mut Self::Group, span: Self::Span) { + group.span = DelimSpan::from_single(span); + } +} + +impl server::Punct for Rustc<'_> { + fn new(&mut self, ch: char, spacing: Spacing) -> Self::Punct { + Punct::new(ch, spacing == Spacing::Joint, server::Span::call_site(self)) + } + fn as_char(&mut self, punct: Self::Punct) -> char { + punct.ch + } + fn spacing(&mut self, punct: Self::Punct) -> Spacing { + if punct.joint { + Spacing::Joint + } else { + Spacing::Alone + } + } + fn span(&mut self, punct: Self::Punct) -> Self::Span { + punct.span + } + fn with_span(&mut self, punct: Self::Punct, span: Self::Span) -> Self::Punct { + Punct { span, ..punct } + } +} + +impl server::Ident for Rustc<'_> { + fn new(&mut self, string: &str, span: Self::Span, is_raw: bool) -> Self::Ident { + Ident::new(Symbol::intern(string), is_raw, span) + } + fn span(&mut self, ident: Self::Ident) -> Self::Span { + ident.span + } + fn with_span(&mut self, ident: Self::Ident, span: Self::Span) -> Self::Ident { + Ident { span, ..ident } + } +} + +impl server::Literal for Rustc<'_> { + // FIXME(eddyb) `Literal` should not expose internal `Debug` impls. + fn debug(&mut self, literal: &Self::Literal) -> String { + format!("{:?}", literal) + } + fn integer(&mut self, n: &str) -> Self::Literal { + self.lit(token::Integer, Symbol::intern(n), None) + } + fn typed_integer(&mut self, n: &str, kind: &str) -> Self::Literal { + self.lit(token::Integer, Symbol::intern(n), Some(Symbol::intern(kind))) + } + fn float(&mut self, n: &str) -> Self::Literal { + self.lit(token::Float, Symbol::intern(n), None) + } + fn f32(&mut self, n: &str) -> Self::Literal { + self.lit(token::Float, Symbol::intern(n), Some(sym::f32)) + } + fn f64(&mut self, n: &str) -> Self::Literal { + self.lit(token::Float, Symbol::intern(n), Some(sym::f64)) + } + fn string(&mut self, string: &str) -> Self::Literal { + let mut escaped = String::new(); + for ch in string.chars() { + escaped.extend(ch.escape_debug()); + } + self.lit(token::Str, Symbol::intern(&escaped), None) + } + fn character(&mut self, ch: char) -> Self::Literal { + let mut escaped = String::new(); + escaped.extend(ch.escape_unicode()); + self.lit(token::Char, Symbol::intern(&escaped), None) + } + fn byte_string(&mut self, bytes: &[u8]) -> Self::Literal { + let string = bytes + .iter() + .cloned() + .flat_map(ascii::escape_default) + .map(Into::::into) + .collect::(); + self.lit(token::ByteStr, Symbol::intern(&string), None) + } + fn span(&mut self, literal: &Self::Literal) -> Self::Span { + literal.span + } + fn set_span(&mut self, literal: &mut Self::Literal, span: Self::Span) { + literal.span = span; + } + fn subspan( + &mut self, + literal: &Self::Literal, + start: Bound, + end: Bound, + ) -> Option { + let span = literal.span; + let length = span.hi().to_usize() - span.lo().to_usize(); + + let start = match start { + Bound::Included(lo) => lo, + Bound::Excluded(lo) => lo + 1, + Bound::Unbounded => 0, + }; + + let end = match end { + Bound::Included(hi) => hi + 1, + Bound::Excluded(hi) => hi, + Bound::Unbounded => length, + }; + + // Bounds check the values, preventing addition overflow and OOB spans. + if start > u32::max_value() as usize + || end > u32::max_value() as usize + || (u32::max_value() - start as u32) < span.lo().to_u32() + || (u32::max_value() - end as u32) < span.lo().to_u32() + || start >= end + || end > length + { + return None; + } + + let new_lo = span.lo() + BytePos::from_usize(start); + let new_hi = span.lo() + BytePos::from_usize(end); + Some(span.with_lo(new_lo).with_hi(new_hi)) + } +} + +impl server::SourceFile for Rustc<'_> { + fn eq(&mut self, file1: &Self::SourceFile, file2: &Self::SourceFile) -> bool { + Lrc::ptr_eq(file1, file2) + } + fn path(&mut self, file: &Self::SourceFile) -> String { + match file.name { + FileName::Real(ref path) => path + .to_str() + .expect("non-UTF8 file path in `proc_macro::SourceFile::path`") + .to_string(), + _ => file.name.to_string(), + } + } + fn is_real(&mut self, file: &Self::SourceFile) -> bool { + file.is_real_file() + } +} + +impl server::MultiSpan for Rustc<'_> { + fn new(&mut self) -> Self::MultiSpan { + vec![] + } + fn push(&mut self, spans: &mut Self::MultiSpan, span: Self::Span) { + spans.push(span) + } +} + +impl server::Diagnostic for Rustc<'_> { + fn new(&mut self, level: Level, msg: &str, spans: Self::MultiSpan) -> Self::Diagnostic { + let mut diag = Diagnostic::new(level.to_internal(), msg); + diag.set_span(MultiSpan::from_spans(spans)); + diag + } + fn sub( + &mut self, + diag: &mut Self::Diagnostic, + level: Level, + msg: &str, + spans: Self::MultiSpan, + ) { + diag.sub(level.to_internal(), msg, MultiSpan::from_spans(spans), None); + } + fn emit(&mut self, diag: Self::Diagnostic) { + self.sess.span_diagnostic.emit_diagnostic(&diag); + } +} + +impl server::Span for Rustc<'_> { + fn debug(&mut self, span: Self::Span) -> String { + format!("{:?} bytes({}..{})", span.ctxt(), span.lo().0, span.hi().0) + } + fn def_site(&mut self) -> Self::Span { + self.def_site + } + fn call_site(&mut self) -> Self::Span { + self.call_site + } + fn mixed_site(&mut self) -> Self::Span { + self.mixed_site + } + fn source_file(&mut self, span: Self::Span) -> Self::SourceFile { + self.sess.source_map().lookup_char_pos(span.lo()).file + } + fn parent(&mut self, span: Self::Span) -> Option { + span.parent() + } + fn source(&mut self, span: Self::Span) -> Self::Span { + span.source_callsite() + } + fn start(&mut self, span: Self::Span) -> LineColumn { + let loc = self.sess.source_map().lookup_char_pos(span.lo()); + LineColumn { + line: loc.line, + column: loc.col.to_usize(), + } + } + fn end(&mut self, span: Self::Span) -> LineColumn { + let loc = self.sess.source_map().lookup_char_pos(span.hi()); + LineColumn { + line: loc.line, + column: loc.col.to_usize(), + } + } + fn join(&mut self, first: Self::Span, second: Self::Span) -> Option { + let self_loc = self.sess.source_map().lookup_char_pos(first.lo()); + let other_loc = self.sess.source_map().lookup_char_pos(second.lo()); + + if self_loc.file.name != other_loc.file.name { + return None; + } + + Some(first.to(second)) + } + fn resolved_at(&mut self, span: Self::Span, at: Self::Span) -> Self::Span { + span.with_ctxt(at.ctxt()) + } + fn source_text(&mut self, span: Self::Span) -> Option { + self.sess.source_map().span_to_snippet(span).ok() + } +} diff --git a/src/libsyntax_ext/Cargo.toml b/src/libsyntax_ext/Cargo.toml index 73310df305b..440873f3c2b 100644 --- a/src/libsyntax_ext/Cargo.toml +++ b/src/libsyntax_ext/Cargo.toml @@ -17,4 +17,5 @@ rustc_data_structures = { path = "../librustc_data_structures" } rustc_target = { path = "../librustc_target" } smallvec = { version = "0.6.7", features = ["union", "may_dangle"] } syntax = { path = "../libsyntax" } +syntax_expand = { path = "../libsyntax_expand" } syntax_pos = { path = "../libsyntax_pos" } diff --git a/src/libsyntax_ext/asm.rs b/src/libsyntax_ext/asm.rs index becbf6d60a0..8c9a34713ea 100644 --- a/src/libsyntax_ext/asm.rs +++ b/src/libsyntax_ext/asm.rs @@ -7,7 +7,7 @@ use rustc_data_structures::thin_vec::ThinVec; use errors::DiagnosticBuilder; use syntax::ast; -use syntax::ext::base::{self, *}; +use syntax_expand::base::{self, *}; use syntax::parse::token::{self, Token}; use syntax::ptr::P; use syntax::symbol::{kw, sym, Symbol}; diff --git a/src/libsyntax_ext/assert.rs b/src/libsyntax_ext/assert.rs index cbfe14fa439..f4d1f7fb09c 100644 --- a/src/libsyntax_ext/assert.rs +++ b/src/libsyntax_ext/assert.rs @@ -1,7 +1,7 @@ use errors::{Applicability, DiagnosticBuilder}; use syntax::ast::{self, *}; -use syntax::ext::base::*; +use syntax_expand::base::*; use syntax::parse::token::{self, TokenKind}; use syntax::parse::parser::Parser; use syntax::print::pprust; diff --git a/src/libsyntax_ext/cfg.rs b/src/libsyntax_ext/cfg.rs index 3c33baf95a5..9e693f29c5a 100644 --- a/src/libsyntax_ext/cfg.rs +++ b/src/libsyntax_ext/cfg.rs @@ -5,7 +5,7 @@ use errors::DiagnosticBuilder; use syntax::ast; -use syntax::ext::base::{self, *}; +use syntax_expand::base::{self, *}; use syntax::attr; use syntax::tokenstream::TokenStream; use syntax::parse::token; diff --git a/src/libsyntax_ext/cmdline_attrs.rs b/src/libsyntax_ext/cmdline_attrs.rs index 2c3159739e2..2d981526a39 100644 --- a/src/libsyntax_ext/cmdline_attrs.rs +++ b/src/libsyntax_ext/cmdline_attrs.rs @@ -2,9 +2,9 @@ use syntax::ast::{self, AttrItem, AttrStyle}; use syntax::attr::mk_attr; -use syntax::panictry; use syntax::parse::{self, token}; use syntax::sess::ParseSess; +use syntax_expand::panictry; use syntax_pos::FileName; pub fn inject(mut krate: ast::Crate, parse_sess: &ParseSess, attrs: &[String]) -> ast::Crate { diff --git a/src/libsyntax_ext/compile_error.rs b/src/libsyntax_ext/compile_error.rs index 24f3a66d4ae..cd7f78e9e34 100644 --- a/src/libsyntax_ext/compile_error.rs +++ b/src/libsyntax_ext/compile_error.rs @@ -1,6 +1,6 @@ // The compiler code necessary to support the compile_error! extension. -use syntax::ext::base::{self, *}; +use syntax_expand::base::{self, *}; use syntax_pos::Span; use syntax::tokenstream::TokenStream; diff --git a/src/libsyntax_ext/concat.rs b/src/libsyntax_ext/concat.rs index 790fdad5b3f..47bade698a8 100644 --- a/src/libsyntax_ext/concat.rs +++ b/src/libsyntax_ext/concat.rs @@ -1,5 +1,5 @@ use syntax::ast; -use syntax::ext::base::{self, DummyResult}; +use syntax_expand::base::{self, DummyResult}; use syntax::symbol::Symbol; use syntax::tokenstream::TokenStream; diff --git a/src/libsyntax_ext/concat_idents.rs b/src/libsyntax_ext/concat_idents.rs index f6747658c07..a132a4136ea 100644 --- a/src/libsyntax_ext/concat_idents.rs +++ b/src/libsyntax_ext/concat_idents.rs @@ -1,7 +1,7 @@ use rustc_data_structures::thin_vec::ThinVec; use syntax::ast; -use syntax::ext::base::{self, *}; +use syntax_expand::base::{self, *}; use syntax::parse::token::{self, Token}; use syntax::ptr::P; use syntax_pos::Span; diff --git a/src/libsyntax_ext/deriving/bounds.rs b/src/libsyntax_ext/deriving/bounds.rs index d5b8a00c75b..6a9b7092024 100644 --- a/src/libsyntax_ext/deriving/bounds.rs +++ b/src/libsyntax_ext/deriving/bounds.rs @@ -3,7 +3,7 @@ use crate::deriving::generic::*; use crate::deriving::generic::ty::*; use syntax::ast::MetaItem; -use syntax::ext::base::{Annotatable, ExtCtxt}; +use syntax_expand::base::{Annotatable, ExtCtxt}; use syntax_pos::Span; pub fn expand_deriving_copy(cx: &mut ExtCtxt<'_>, diff --git a/src/libsyntax_ext/deriving/clone.rs b/src/libsyntax_ext/deriving/clone.rs index 9ef2c033b07..eb7d480aa98 100644 --- a/src/libsyntax_ext/deriving/clone.rs +++ b/src/libsyntax_ext/deriving/clone.rs @@ -3,7 +3,7 @@ use crate::deriving::generic::*; use crate::deriving::generic::ty::*; use syntax::ast::{self, Expr, GenericArg, Generics, ItemKind, MetaItem, VariantData}; -use syntax::ext::base::{Annotatable, ExtCtxt, SpecialDerives}; +use syntax_expand::base::{Annotatable, ExtCtxt, SpecialDerives}; use syntax::ptr::P; use syntax::symbol::{kw, sym, Symbol}; use syntax_pos::Span; diff --git a/src/libsyntax_ext/deriving/cmp/eq.rs b/src/libsyntax_ext/deriving/cmp/eq.rs index c92339dd2fb..92721dab878 100644 --- a/src/libsyntax_ext/deriving/cmp/eq.rs +++ b/src/libsyntax_ext/deriving/cmp/eq.rs @@ -3,7 +3,7 @@ use crate::deriving::generic::*; use crate::deriving::generic::ty::*; use syntax::ast::{self, Ident, Expr, MetaItem, GenericArg}; -use syntax::ext::base::{Annotatable, ExtCtxt, SpecialDerives}; +use syntax_expand::base::{Annotatable, ExtCtxt, SpecialDerives}; use syntax::ptr::P; use syntax::symbol::{sym, Symbol}; use syntax_pos::Span; diff --git a/src/libsyntax_ext/deriving/cmp/ord.rs b/src/libsyntax_ext/deriving/cmp/ord.rs index 1f4f5aa3709..3eeed95aff7 100644 --- a/src/libsyntax_ext/deriving/cmp/ord.rs +++ b/src/libsyntax_ext/deriving/cmp/ord.rs @@ -3,7 +3,7 @@ use crate::deriving::generic::*; use crate::deriving::generic::ty::*; use syntax::ast::{self, Expr, MetaItem}; -use syntax::ext::base::{Annotatable, ExtCtxt}; +use syntax_expand::base::{Annotatable, ExtCtxt}; use syntax::ptr::P; use syntax::symbol::sym; use syntax_pos::Span; diff --git a/src/libsyntax_ext/deriving/cmp/partial_eq.rs b/src/libsyntax_ext/deriving/cmp/partial_eq.rs index 91e1e80e4fb..1615d991792 100644 --- a/src/libsyntax_ext/deriving/cmp/partial_eq.rs +++ b/src/libsyntax_ext/deriving/cmp/partial_eq.rs @@ -3,7 +3,7 @@ use crate::deriving::generic::*; use crate::deriving::generic::ty::*; use syntax::ast::{BinOpKind, Expr, MetaItem}; -use syntax::ext::base::{Annotatable, ExtCtxt, SpecialDerives}; +use syntax_expand::base::{Annotatable, ExtCtxt, SpecialDerives}; use syntax::ptr::P; use syntax::symbol::sym; use syntax_pos::Span; diff --git a/src/libsyntax_ext/deriving/cmp/partial_ord.rs b/src/libsyntax_ext/deriving/cmp/partial_ord.rs index 13d63aaf2a8..af8aacc6eb9 100644 --- a/src/libsyntax_ext/deriving/cmp/partial_ord.rs +++ b/src/libsyntax_ext/deriving/cmp/partial_ord.rs @@ -5,7 +5,7 @@ use crate::deriving::generic::*; use crate::deriving::generic::ty::*; use syntax::ast::{self, BinOpKind, Expr, MetaItem}; -use syntax::ext::base::{Annotatable, ExtCtxt}; +use syntax_expand::base::{Annotatable, ExtCtxt}; use syntax::ptr::P; use syntax::symbol::{sym, Symbol}; use syntax_pos::Span; diff --git a/src/libsyntax_ext/deriving/debug.rs b/src/libsyntax_ext/deriving/debug.rs index 003c2423576..35298211e4d 100644 --- a/src/libsyntax_ext/deriving/debug.rs +++ b/src/libsyntax_ext/deriving/debug.rs @@ -6,7 +6,7 @@ use rustc_data_structures::thin_vec::ThinVec; use syntax::ast::{self, Ident}; use syntax::ast::{Expr, MetaItem}; -use syntax::ext::base::{Annotatable, ExtCtxt}; +use syntax_expand::base::{Annotatable, ExtCtxt}; use syntax::ptr::P; use syntax::symbol::sym; use syntax_pos::{DUMMY_SP, Span}; diff --git a/src/libsyntax_ext/deriving/decodable.rs b/src/libsyntax_ext/deriving/decodable.rs index cde72abbdef..3a0379a0eb0 100644 --- a/src/libsyntax_ext/deriving/decodable.rs +++ b/src/libsyntax_ext/deriving/decodable.rs @@ -6,7 +6,7 @@ use crate::deriving::generic::ty::*; use syntax::ast; use syntax::ast::{Expr, MetaItem, Mutability}; -use syntax::ext::base::{Annotatable, ExtCtxt}; +use syntax_expand::base::{Annotatable, ExtCtxt}; use syntax::ptr::P; use syntax::symbol::Symbol; use syntax_pos::Span; diff --git a/src/libsyntax_ext/deriving/default.rs b/src/libsyntax_ext/deriving/default.rs index 2fdea10b76f..6176791c31b 100644 --- a/src/libsyntax_ext/deriving/default.rs +++ b/src/libsyntax_ext/deriving/default.rs @@ -3,7 +3,7 @@ use crate::deriving::generic::*; use crate::deriving::generic::ty::*; use syntax::ast::{Expr, MetaItem}; -use syntax::ext::base::{Annotatable, DummyResult, ExtCtxt}; +use syntax_expand::base::{Annotatable, DummyResult, ExtCtxt}; use syntax::ptr::P; use syntax::symbol::{kw, sym}; use syntax::span_err; diff --git a/src/libsyntax_ext/deriving/encodable.rs b/src/libsyntax_ext/deriving/encodable.rs index 655d3bb7c4a..2105946b666 100644 --- a/src/libsyntax_ext/deriving/encodable.rs +++ b/src/libsyntax_ext/deriving/encodable.rs @@ -90,7 +90,7 @@ use crate::deriving::generic::*; use crate::deriving::generic::ty::*; use syntax::ast::{Expr, ExprKind, MetaItem, Mutability}; -use syntax::ext::base::{Annotatable, ExtCtxt}; +use syntax_expand::base::{Annotatable, ExtCtxt}; use syntax::ptr::P; use syntax::symbol::Symbol; use syntax_pos::Span; diff --git a/src/libsyntax_ext/deriving/generic/mod.rs b/src/libsyntax_ext/deriving/generic/mod.rs index 9c3ec06d59a..1886a5154b7 100644 --- a/src/libsyntax_ext/deriving/generic/mod.rs +++ b/src/libsyntax_ext/deriving/generic/mod.rs @@ -186,12 +186,12 @@ use rustc_target::spec::abi::Abi; use syntax::ast::{self, BinOpKind, EnumDef, Expr, Generics, Ident, PatKind}; use syntax::ast::{VariantData, GenericParamKind, GenericArg}; use syntax::attr; -use syntax::ext::base::{Annotatable, ExtCtxt, SpecialDerives}; use syntax::source_map::respan; use syntax::util::map_in_place::MapInPlace; use syntax::ptr::P; use syntax::sess::ParseSess; use syntax::symbol::{Symbol, kw, sym}; +use syntax_expand::base::{Annotatable, ExtCtxt, SpecialDerives}; use syntax_pos::{Span}; use ty::{LifetimeBounds, Path, Ptr, PtrTy, Self_, Ty}; diff --git a/src/libsyntax_ext/deriving/generic/ty.rs b/src/libsyntax_ext/deriving/generic/ty.rs index 6ae02a5cab1..607746597a5 100644 --- a/src/libsyntax_ext/deriving/generic/ty.rs +++ b/src/libsyntax_ext/deriving/generic/ty.rs @@ -5,7 +5,7 @@ pub use PtrTy::*; pub use Ty::*; use syntax::ast::{self, Expr, GenericParamKind, Generics, Ident, SelfKind, GenericArg}; -use syntax::ext::base::ExtCtxt; +use syntax_expand::base::ExtCtxt; use syntax::source_map::{respan, DUMMY_SP}; use syntax::ptr::P; use syntax_pos::Span; diff --git a/src/libsyntax_ext/deriving/hash.rs b/src/libsyntax_ext/deriving/hash.rs index 2fc594abd70..fe9ef78bb1b 100644 --- a/src/libsyntax_ext/deriving/hash.rs +++ b/src/libsyntax_ext/deriving/hash.rs @@ -3,7 +3,7 @@ use crate::deriving::generic::*; use crate::deriving::generic::ty::*; use syntax::ast::{Expr, MetaItem, Mutability}; -use syntax::ext::base::{Annotatable, ExtCtxt}; +use syntax_expand::base::{Annotatable, ExtCtxt}; use syntax::ptr::P; use syntax::symbol::sym; use syntax_pos::Span; diff --git a/src/libsyntax_ext/deriving/mod.rs b/src/libsyntax_ext/deriving/mod.rs index 60b6eba7a4b..f0471a857dc 100644 --- a/src/libsyntax_ext/deriving/mod.rs +++ b/src/libsyntax_ext/deriving/mod.rs @@ -1,7 +1,7 @@ //! The compiler code necessary to implement the `#[derive]` extensions. use syntax::ast::{self, MetaItem}; -use syntax::ext::base::{Annotatable, ExtCtxt, MultiItemModifier}; +use syntax_expand::base::{Annotatable, ExtCtxt, MultiItemModifier}; use syntax::ptr::P; use syntax::symbol::{Symbol, sym}; use syntax_pos::Span; diff --git a/src/libsyntax_ext/env.rs b/src/libsyntax_ext/env.rs index 02757bf6b16..58fe56bd235 100644 --- a/src/libsyntax_ext/env.rs +++ b/src/libsyntax_ext/env.rs @@ -4,7 +4,7 @@ // use syntax::ast::{self, Ident, GenericArg}; -use syntax::ext::base::{self, *}; +use syntax_expand::base::{self, *}; use syntax::symbol::{kw, sym, Symbol}; use syntax_pos::Span; use syntax::tokenstream::TokenStream; diff --git a/src/libsyntax_ext/format.rs b/src/libsyntax_ext/format.rs index 8fc64021b51..45d9f79c28f 100644 --- a/src/libsyntax_ext/format.rs +++ b/src/libsyntax_ext/format.rs @@ -8,7 +8,7 @@ use errors::Applicability; use errors::pluralise; use syntax::ast; -use syntax::ext::base::{self, *}; +use syntax_expand::base::{self, *}; use syntax::parse::token; use syntax::ptr::P; use syntax::symbol::{Symbol, sym}; diff --git a/src/libsyntax_ext/global_allocator.rs b/src/libsyntax_ext/global_allocator.rs index f014b4f4f9f..75dda9535b3 100644 --- a/src/libsyntax_ext/global_allocator.rs +++ b/src/libsyntax_ext/global_allocator.rs @@ -2,8 +2,8 @@ use crate::util::check_builtin_macro_attribute; use syntax::ast::{ItemKind, Mutability, Stmt, Ty, TyKind, Unsafety}; use syntax::ast::{self, Param, Attribute, Expr, FnHeader, Generics, Ident}; -use syntax::ext::allocator::{AllocatorKind, AllocatorMethod, AllocatorTy, ALLOCATOR_METHODS}; -use syntax::ext::base::{Annotatable, ExtCtxt}; +use syntax_expand::allocator::{AllocatorKind, AllocatorMethod, AllocatorTy, ALLOCATOR_METHODS}; +use syntax_expand::base::{Annotatable, ExtCtxt}; use syntax::ptr::P; use syntax::symbol::{kw, sym, Symbol}; use syntax_pos::Span; diff --git a/src/libsyntax_ext/global_asm.rs b/src/libsyntax_ext/global_asm.rs index 72fb5b47c21..879ae1e4215 100644 --- a/src/libsyntax_ext/global_asm.rs +++ b/src/libsyntax_ext/global_asm.rs @@ -12,7 +12,7 @@ use errors::DiagnosticBuilder; use syntax::ast; use syntax::source_map::respan; -use syntax::ext::base::{self, *}; +use syntax_expand::base::{self, *}; use syntax::parse::token; use syntax::ptr::P; use syntax_pos::Span; diff --git a/src/libsyntax_ext/lib.rs b/src/libsyntax_ext/lib.rs index 5ba76a52350..5516f276422 100644 --- a/src/libsyntax_ext/lib.rs +++ b/src/libsyntax_ext/lib.rs @@ -15,9 +15,9 @@ use crate::deriving::*; use syntax::ast::Ident; use syntax::edition::Edition; -use syntax::ext::base::{SyntaxExtension, SyntaxExtensionKind, MacroExpanderFn}; -use syntax::ext::proc_macro::BangProcMacro; use syntax::symbol::sym; +use syntax_expand::base::{Resolver, SyntaxExtension, SyntaxExtensionKind, MacroExpanderFn}; +use syntax_expand::proc_macro::BangProcMacro; mod error_codes; @@ -45,7 +45,7 @@ pub mod proc_macro_harness; pub mod standard_library_imports; pub mod test_harness; -pub fn register_builtin_macros(resolver: &mut dyn syntax::ext::base::Resolver, edition: Edition) { +pub fn register_builtin_macros(resolver: &mut dyn Resolver, edition: Edition) { let mut register = |name, kind| resolver.register_builtin_macro( Ident::with_dummy_span(name), SyntaxExtension { is_builtin: true, ..SyntaxExtension::default(kind, edition) diff --git a/src/libsyntax_ext/log_syntax.rs b/src/libsyntax_ext/log_syntax.rs index 92130bfaf68..2202375e5e7 100644 --- a/src/libsyntax_ext/log_syntax.rs +++ b/src/libsyntax_ext/log_syntax.rs @@ -1,4 +1,4 @@ -use syntax::ext::base; +use syntax_expand::base; use syntax::print; use syntax::tokenstream::TokenStream; use syntax_pos; diff --git a/src/libsyntax_ext/plugin_macro_defs.rs b/src/libsyntax_ext/plugin_macro_defs.rs index 62c7e188eba..1ca9422eb9d 100644 --- a/src/libsyntax_ext/plugin_macro_defs.rs +++ b/src/libsyntax_ext/plugin_macro_defs.rs @@ -4,7 +4,7 @@ use syntax::ast::*; use syntax::attr; use syntax::edition::Edition; -use syntax::ext::base::{Resolver, NamedSyntaxExtension}; +use syntax_expand::base::{Resolver, NamedSyntaxExtension}; use syntax::parse::token; use syntax::ptr::P; use syntax::source_map::respan; diff --git a/src/libsyntax_ext/proc_macro_harness.rs b/src/libsyntax_ext/proc_macro_harness.rs index 1b7068818b0..96d0c3fcab1 100644 --- a/src/libsyntax_ext/proc_macro_harness.rs +++ b/src/libsyntax_ext/proc_macro_harness.rs @@ -3,14 +3,14 @@ use std::mem; use smallvec::smallvec; use syntax::ast::{self, Ident}; use syntax::attr; -use syntax::ext::base::ExtCtxt; -use syntax::ext::expand::{AstFragment, ExpansionConfig}; -use syntax::ext::proc_macro::is_proc_macro_attr; use syntax::print::pprust; use syntax::ptr::P; use syntax::sess::ParseSess; use syntax::symbol::{kw, sym}; use syntax::visit::{self, Visitor}; +use syntax_expand::base::{ExtCtxt, Resolver}; +use syntax_expand::expand::{AstFragment, ExpansionConfig}; +use syntax_expand::proc_macro::is_proc_macro_attr; use syntax_pos::{Span, DUMMY_SP}; use syntax_pos::hygiene::AstPass; @@ -46,7 +46,7 @@ struct CollectProcMacros<'a> { } pub fn inject(sess: &ParseSess, - resolver: &mut dyn (::syntax::ext::base::Resolver), + resolver: &mut dyn Resolver, mut krate: ast::Crate, is_proc_macro_crate: bool, has_proc_macro_decls: bool, diff --git a/src/libsyntax_ext/source_util.rs b/src/libsyntax_ext/source_util.rs index f74507dcc21..438e199ebdb 100644 --- a/src/libsyntax_ext/source_util.rs +++ b/src/libsyntax_ext/source_util.rs @@ -1,5 +1,6 @@ -use syntax::{ast, panictry}; -use syntax::ext::base::{self, *}; +use syntax_expand::panictry; +use syntax_expand::base::{self, *}; +use syntax::ast; use syntax::parse::{self, token, DirectoryOwnership}; use syntax::print::pprust; use syntax::ptr::P; diff --git a/src/libsyntax_ext/standard_library_imports.rs b/src/libsyntax_ext/standard_library_imports.rs index 4f17acf2d0f..fd27a218906 100644 --- a/src/libsyntax_ext/standard_library_imports.rs +++ b/src/libsyntax_ext/standard_library_imports.rs @@ -1,11 +1,11 @@ use syntax::{ast, attr}; use syntax::edition::Edition; -use syntax::ext::expand::ExpansionConfig; -use syntax::ext::hygiene::AstPass; -use syntax::ext::base::{ExtCtxt, Resolver}; use syntax::ptr::P; use syntax::sess::ParseSess; use syntax::symbol::{Ident, Symbol, kw, sym}; +use syntax_expand::expand::ExpansionConfig; +use syntax_expand::hygiene::AstPass; +use syntax_expand::base::{ExtCtxt, Resolver}; use syntax_pos::DUMMY_SP; pub fn inject( diff --git a/src/libsyntax_ext/test.rs b/src/libsyntax_ext/test.rs index 4d7441ca0b6..b0da413d63a 100644 --- a/src/libsyntax_ext/test.rs +++ b/src/libsyntax_ext/test.rs @@ -5,7 +5,7 @@ use crate::util::check_builtin_macro_attribute; use syntax::ast; use syntax::attr; -use syntax::ext::base::*; +use syntax_expand::base::*; use syntax::print::pprust; use syntax::source_map::respan; use syntax::symbol::{Symbol, sym}; diff --git a/src/libsyntax_ext/test_harness.rs b/src/libsyntax_ext/test_harness.rs index 0bb279c0cb0..33d41a7f53e 100644 --- a/src/libsyntax_ext/test_harness.rs +++ b/src/libsyntax_ext/test_harness.rs @@ -6,8 +6,8 @@ use rustc_target::spec::PanicStrategy; use syntax::ast::{self, Ident}; use syntax::attr; use syntax::entry::{self, EntryPointType}; -use syntax::ext::base::{ExtCtxt, Resolver}; -use syntax::ext::expand::{AstFragment, ExpansionConfig}; +use syntax_expand::base::{ExtCtxt, Resolver}; +use syntax_expand::expand::{AstFragment, ExpansionConfig}; use syntax::feature_gate::Features; use syntax::mut_visit::{*, ExpectOne}; use syntax::ptr::P; diff --git a/src/libsyntax_ext/trace_macros.rs b/src/libsyntax_ext/trace_macros.rs index d83c24046d9..dbf96d3b561 100644 --- a/src/libsyntax_ext/trace_macros.rs +++ b/src/libsyntax_ext/trace_macros.rs @@ -1,4 +1,4 @@ -use syntax::ext::base::{self, ExtCtxt}; +use syntax_expand::base::{self, ExtCtxt}; use syntax::symbol::kw; use syntax_pos::Span; use syntax::tokenstream::{TokenTree, TokenStream}; diff --git a/src/libsyntax_ext/util.rs b/src/libsyntax_ext/util.rs index fbae68057c5..d84fe19b3ea 100644 --- a/src/libsyntax_ext/util.rs +++ b/src/libsyntax_ext/util.rs @@ -1,7 +1,7 @@ use syntax_pos::Symbol; use syntax::ast::MetaItem; use syntax::attr::{check_builtin_attribute, AttributeTemplate}; -use syntax::ext::base::ExtCtxt; +use syntax_expand::base::ExtCtxt; pub fn check_builtin_macro_attribute(ecx: &ExtCtxt<'_>, meta_item: &MetaItem, name: Symbol) { // All the built-in macro attributes are "words" at the moment. -- cgit 1.4.1-3-g733a5 From a6eef299d3b0ca24f8ffc0c3dc03283c09ec7945 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 14 Oct 2019 10:37:21 +1100 Subject: Make `TokenStream::from_iter` less general and more efficient. The current code has this impl: ``` impl> iter::FromIterator for TokenStream ``` If given an `IntoIterator`, it will convert each individual `TokenTree` to a `TokenStream` (at the cost of two allocations: a `Vec` and an `Lrc`). It will then merge those `TokenStream`s into a single `TokenStream`. This is inefficient. This commit changes the impl to this less general one: ``` impl iter::FromIterator for TokenStream ``` It collects the `TokenTree`s into a single `Vec` first and then converts that to a `TokenStream` by wrapping it in a single `Lrc`. The previous generality was unnecessary; no other code needs changing. This change speeds up several benchmarks by up to 4%. --- src/libsyntax/tokenstream.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src/libsyntax/tokenstream.rs') diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index db6832d6423..3d89e73d729 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -202,9 +202,9 @@ impl From for TreeAndJoint { } } -impl> iter::FromIterator for TokenStream { - fn from_iter>(iter: I) -> Self { - TokenStream::from_streams(iter.into_iter().map(Into::into).collect::>()) +impl iter::FromIterator for TokenStream { + fn from_iter>(iter: I) -> Self { + TokenStream::new(iter.into_iter().map(Into::into).collect::>()) } } -- cgit 1.4.1-3-g733a5 From e4ec4a6da350ae2564971ae826a1bc3ec9a41988 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 14 Oct 2019 14:06:00 +1100 Subject: Change `MetaItem::tokens()` to `MetaItem::token_trees_and_joints()`. Likewise for `NestedMetaItem::tokens()`. Also, add `MetaItemKind::token_trees_and_joints()`, which `MetaItemKind::tokens()` now calls. This avoids some unnecessary `TokenTree` to `TokenStream` conversions, and removes the need for the clumsy `TokenStream::append_to_tree_and_joint_vec()`. --- src/libsyntax/attr/mod.rs | 43 ++++++++++++++++++++++++++----------------- src/libsyntax/tokenstream.rs | 4 ---- 2 files changed, 26 insertions(+), 21 deletions(-) (limited to 'src/libsyntax/tokenstream.rs') diff --git a/src/libsyntax/attr/mod.rs b/src/libsyntax/attr/mod.rs index 8786c850bfd..4aec5040881 100644 --- a/src/libsyntax/attr/mod.rs +++ b/src/libsyntax/attr/mod.rs @@ -22,7 +22,7 @@ use crate::ptr::P; use crate::sess::ParseSess; use crate::symbol::{sym, Symbol}; use crate::ThinVec; -use crate::tokenstream::{TokenStream, TokenTree, DelimSpan}; +use crate::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndJoint}; use crate::GLOBALS; use log::debug; @@ -463,7 +463,7 @@ pub fn first_attr_value_str_by_name(attrs: &[Attribute], name: Symbol) -> Option } impl MetaItem { - fn tokens(&self) -> TokenStream { + fn token_trees_and_joints(&self) -> Vec { let mut idents = vec![]; let mut last_pos = BytePos(0 as u32); for (i, segment) in self.path.segments.iter().enumerate() { @@ -477,8 +477,8 @@ impl MetaItem { idents.push(TokenTree::Token(Token::from_ast_ident(segment.ident)).into()); last_pos = segment.ident.span.hi(); } - self.kind.tokens(self.span).append_to_tree_and_joint_vec(&mut idents); - TokenStream::new(idents) + idents.extend(self.kind.token_trees_and_joints(self.span)); + idents } fn from_tokens(tokens: &mut iter::Peekable) -> Option @@ -537,14 +537,14 @@ impl MetaItem { } impl MetaItemKind { - pub fn tokens(&self, span: Span) -> TokenStream { + pub fn token_trees_and_joints(&self, span: Span) -> Vec { match *self { - MetaItemKind::Word => TokenStream::default(), + MetaItemKind::Word => vec![], MetaItemKind::NameValue(ref lit) => { - TokenStream::new(vec![ + vec![ TokenTree::token(token::Eq, span).into(), lit.token_tree().into(), - ]) + ] } MetaItemKind::List(ref list) => { let mut tokens = Vec::new(); @@ -552,17 +552,26 @@ impl MetaItemKind { if i > 0 { tokens.push(TokenTree::token(token::Comma, span).into()); } - item.tokens().append_to_tree_and_joint_vec(&mut tokens); + tokens.extend(item.token_trees_and_joints()) } - TokenTree::Delimited( - DelimSpan::from_single(span), - token::Paren, - TokenStream::new(tokens).into(), - ).into() + vec![ + TokenTree::Delimited( + DelimSpan::from_single(span), + token::Paren, + TokenStream::new(tokens).into(), + ).into() + ] } } } + // Premature conversions of `TokenTree`s to `TokenStream`s can hurt + // performance. Do not use this function if `token_trees_and_joints()` can + // be used instead. + pub fn tokens(&self, span: Span) -> TokenStream { + TokenStream::new(self.token_trees_and_joints(span)) + } + fn from_tokens(tokens: &mut iter::Peekable) -> Option where I: Iterator, { @@ -604,10 +613,10 @@ impl NestedMetaItem { } } - fn tokens(&self) -> TokenStream { + fn token_trees_and_joints(&self) -> Vec { match *self { - NestedMetaItem::MetaItem(ref item) => item.tokens(), - NestedMetaItem::Literal(ref lit) => lit.token_tree().into(), + NestedMetaItem::MetaItem(ref item) => item.token_trees_and_joints(), + NestedMetaItem::Literal(ref lit) => vec![lit.token_tree().into()], } } diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index 3d89e73d729..ac155556cda 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -271,10 +271,6 @@ impl TokenStream { } } - pub fn append_to_tree_and_joint_vec(self, vec: &mut Vec) { - vec.extend(self.0.iter().cloned()); - } - pub fn trees(&self) -> Cursor { self.clone().into_trees() } -- cgit 1.4.1-3-g733a5