diff options
| -rw-r--r-- | src/libproc_macro/diagnostic.rs | 44 | ||||
| -rw-r--r-- | src/libproc_macro/lib.rs | 348 | ||||
| -rw-r--r-- | src/libproc_macro/quote.rs | 344 | ||||
| -rw-r--r-- | src/libproc_macro/rustc.rs | 284 | ||||
| -rw-r--r-- | src/librustc_errors/diagnostic.rs | 2 | ||||
| -rw-r--r-- | src/librustc_metadata/cstore_impl.rs | 6 | ||||
| -rw-r--r-- | src/librustc_metadata/lib.rs | 1 | ||||
| -rw-r--r-- | src/libsyntax/parse/lexer/mod.rs | 6 |
8 files changed, 465 insertions, 570 deletions
diff --git a/src/libproc_macro/diagnostic.rs b/src/libproc_macro/diagnostic.rs index 06939a9d1e1..51e7647f36c 100644 --- a/src/libproc_macro/diagnostic.rs +++ b/src/libproc_macro/diagnostic.rs @@ -10,7 +10,8 @@ use Span; -use rustc_errors as rustc; +use rustc_errors as errors; +use syntax_pos::MultiSpan; /// An enum representing a diagnostic level. #[unstable(feature = "proc_macro_diagnostic", issue = "38356")] @@ -97,38 +98,21 @@ impl Diagnostic { /// Emit the diagnostic. #[unstable(feature = "proc_macro_diagnostic", issue = "38356")] pub fn emit(self) { - ::__internal::with_sess(move |sess, _| { - let handler = &sess.span_diagnostic; - let level = __internal::level_to_internal_level(self.level); - let mut diag = rustc::DiagnosticBuilder::new(handler, level, &*self.message); + let level = self.level.to_internal(); + let mut diag = errors::Diagnostic::new(level, &*self.message); - if let Some(span) = self.span { - diag.set_span(span.0); - } + if let Some(span) = self.span { + diag.set_span(span.0); + } - for child in self.children { - let span = child.span.map(|s| s.0); - let level = __internal::level_to_internal_level(child.level); - diag.sub(level, &*child.message, span); - } + for child in self.children { + let span = child.span.map_or(MultiSpan::new(), |s| s.0.into()); + let level = child.level.to_internal(); + diag.sub(level, &*child.message, span, None); + } - diag.emit(); + ::__internal::with_sess(move |sess, _| { + errors::DiagnosticBuilder::new_diagnostic(&sess.span_diagnostic, diag).emit(); }); } } - -#[unstable(feature = "proc_macro_internals", issue = "27812")] -#[doc(hidden)] -pub mod __internal { - use super::{Level, rustc}; - - pub fn level_to_internal_level(level: Level) -> rustc::Level { - match level { - Level::Error => rustc::Level::Error, - Level::Warning => rustc::Level::Warning, - Level::Note => rustc::Level::Note, - Level::Help => rustc::Level::Help, - Level::__Nonexhaustive => unreachable!("Level::__Nonexhaustive") - } - } -} diff --git a/src/libproc_macro/lib.rs b/src/libproc_macro/lib.rs index f5a7c88a1b7..61da9db76f6 100644 --- a/src/libproc_macro/lib.rs +++ b/src/libproc_macro/lib.rs @@ -44,21 +44,24 @@ extern crate syntax_pos; extern crate rustc_errors; extern crate rustc_data_structures; +#[unstable(feature = "proc_macro_internals", issue = "27812")] +#[doc(hidden)] +pub mod rustc; + mod diagnostic; #[unstable(feature = "proc_macro_diagnostic", issue = "38356")] pub use diagnostic::{Diagnostic, Level}; use std::{ascii, fmt, iter}; +use std::path::PathBuf; use rustc_data_structures::sync::Lrc; use std::str::FromStr; -use syntax::ast; use syntax::errors::DiagnosticBuilder; use syntax::parse::{self, token}; -use syntax::symbol::{keywords, Symbol}; +use syntax::symbol::Symbol; use syntax::tokenstream; -use syntax::parse::lexer::{self, comments}; use syntax_pos::{FileMap, Pos, FileName}; /// The main type provided by this crate, representing an abstract stream of @@ -145,6 +148,9 @@ impl fmt::Debug for TokenStream { } } +#[unstable(feature = "proc_macro_quote", issue = "38356")] +pub use quote::{quote, quote_span}; + /// Creates a token stream containing a single token tree. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] impl From<TokenTree> for TokenStream { @@ -237,7 +243,7 @@ pub mod token_stream { /// Unquoting is done with `$`, and works by taking the single next ident as the unquoted term. /// To quote `$` itself, use `$$`. /// -/// This is a dummy macro, the actual implementation is in quote::Quoter +/// This is a dummy macro, the actual implementation is in `quote::quote`.` #[unstable(feature = "proc_macro_quote", issue = "38356")] #[macro_export] macro_rules! quote { () => {} } @@ -246,13 +252,6 @@ macro_rules! quote { () => {} } #[doc(hidden)] mod quote; -/// Quote a `Span` into a `TokenStream`. -/// This is needed to implement a custom quoter. -#[unstable(feature = "proc_macro_quote", issue = "38356")] -pub fn quote_span(span: Span) -> TokenStream { - quote::Quote::quote(span) -} - /// A region of source code, along with macro expansion information. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] #[derive(Copy, Clone)] @@ -425,8 +424,11 @@ impl SourceFile { /// /// [`is_real`]: #method.is_real #[unstable(feature = "proc_macro_span", issue = "38356")] - pub fn path(&self) -> &FileName { - &self.filemap.name + pub fn path(&self) -> PathBuf { + match self.filemap.name { + FileName::Real(ref path) => path.clone(), + _ => PathBuf::from(self.filemap.name.to_string()) + } } /// Returns `true` if this source file is a real source file, and not generated by an external @@ -440,18 +442,12 @@ impl SourceFile { } } -#[unstable(feature = "proc_macro_span", issue = "38356")] -impl AsRef<FileName> for SourceFile { - fn as_ref(&self) -> &FileName { - self.path() - } -} #[unstable(feature = "proc_macro_span", issue = "38356")] impl fmt::Debug for SourceFile { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { f.debug_struct("SourceFile") - .field("path", self.path()) + .field("path", &self.path()) .field("is_real", &self.is_real()) .finish() } @@ -467,13 +463,6 @@ impl PartialEq for SourceFile { #[unstable(feature = "proc_macro_span", issue = "38356")] impl Eq for SourceFile {} -#[unstable(feature = "proc_macro_span", issue = "38356")] -impl PartialEq<FileName> for SourceFile { - fn eq(&self, other: &FileName) -> bool { - self.as_ref() == other - } -} - /// A single token or a delimited sequence of token trees (e.g. `[1, (), ..]`). #[stable(feature = "proc_macro_lib2", since = "1.29.0")] #[derive(Clone)] @@ -599,7 +588,7 @@ impl fmt::Display for TokenTree { /// A delimited token stream. /// /// A `Group` internally contains a `TokenStream` which is surrounded by `Delimiter`s. -#[derive(Clone, Debug)] +#[derive(Clone)] #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub struct Group { delimiter: Delimiter, @@ -693,12 +682,23 @@ impl fmt::Display for Group { } } +#[stable(feature = "proc_macro_lib2", since = "1.29.0")] +impl fmt::Debug for Group { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("Group") + .field("delimiter", &self.delimiter()) + .field("stream", &self.stream()) + .field("span", &self.span()) + .finish() + } +} + /// An `Punct` is an single punctuation character like `+`, `-` or `#`. /// /// Multicharacter operators like `+=` are represented as two instances of `Punct` with different /// forms of `Spacing` returned. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] -#[derive(Clone, Debug)] +#[derive(Clone)] pub struct Punct { ch: char, spacing: Spacing, @@ -782,8 +782,19 @@ impl fmt::Display for Punct { } } +#[stable(feature = "proc_macro_lib2", since = "1.29.0")] +impl fmt::Debug for Punct { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("Punct") + .field("ch", &self.as_char()) + .field("spacing", &self.spacing()) + .field("span", &self.span()) + .finish() + } +} + /// An identifier (`ident`). -#[derive(Clone, Debug)] +#[derive(Clone)] #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub struct Ident { sym: Symbol, @@ -797,6 +808,16 @@ impl !Send for Ident {} impl !Sync for Ident {} impl Ident { + fn is_valid(string: &str) -> bool { + let mut chars = string.chars(); + if let Some(start) = chars.next() { + (start == '_' || start.is_xid_start()) + && chars.all(|cont| cont == '_' || cont.is_xid_continue()) + } else { + false + } + } + /// Creates a new `Ident` with the given `string` as well as the specified /// `span`. /// The `string` argument must be a valid identifier permitted by the @@ -818,26 +839,19 @@ impl Ident { /// tokens, requires a `Span` to be specified at construction. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn new(string: &str, span: Span) -> Ident { - if !lexer::is_valid_ident(string) { + if !Ident::is_valid(string) { panic!("`{:?}` is not a valid identifier", string) } - Ident { - sym: Symbol::intern(string), - span, - is_raw: false, - } + Ident::new_maybe_raw(string, span, false) } /// Same as `Ident::new`, but creates a raw identifier (`r#ident`). #[unstable(feature = "proc_macro_raw_ident", issue = "38356")] pub fn new_raw(string: &str, span: Span) -> Ident { - let mut ident = Ident::new(string, span); - if ident.sym == keywords::Underscore.name() || - ast::Ident::with_empty_ctxt(ident.sym).is_path_segment_keyword() { - panic!("`{:?}` is not a valid raw identifier", string) + if !Ident::is_valid(string) { + panic!("`{:?}` is not a valid identifier", string) } - ident.is_raw = true; - ident + Ident::new_maybe_raw(string, span, true) } /// Returns the span of this `Ident`, encompassing the entire string returned @@ -859,10 +873,17 @@ impl Ident { #[stable(feature = "proc_macro_lib2", since = "1.29.0")] impl fmt::Display for Ident { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - if self.is_raw { - f.write_str("r#")?; - } - self.sym.as_str().fmt(f) + TokenStream::from(TokenTree::from(self.clone())).fmt(f) + } +} + +#[stable(feature = "proc_macro_lib2", since = "1.29.0")] +impl fmt::Debug for Ident { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("Ident") + .field("ident", &self.to_string()) + .field("span", &self.span()) + .finish() } } @@ -870,11 +891,12 @@ impl fmt::Display for Ident { /// character (`'a'`), byte character (`b'a'`), an integer or floating point number /// with or without a suffix (`1`, `1u8`, `2.3`, `2.3f32`). /// Boolean literals like `true` and `false` do not belong here, they are `Ident`s. +// FIXME(eddyb) `Literal` should not expose internal `Debug` impls. #[derive(Clone, Debug)] #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub struct Literal { lit: token::Lit, - suffix: Option<ast::Name>, + suffix: Option<Symbol>, span: Span, } @@ -1122,236 +1144,6 @@ impl fmt::Display for Literal { } } -impl Delimiter { - fn from_internal(delim: token::DelimToken) -> Delimiter { - match delim { - token::Paren => Delimiter::Parenthesis, - token::Brace => Delimiter::Brace, - token::Bracket => Delimiter::Bracket, - token::NoDelim => Delimiter::None, - } - } - - fn to_internal(self) -> token::DelimToken { - match self { - Delimiter::Parenthesis => token::Paren, - Delimiter::Brace => token::Brace, - Delimiter::Bracket => token::Bracket, - Delimiter::None => token::NoDelim, - } - } -} - -impl TokenTree { - fn from_internal(stream: tokenstream::TokenStream, stack: &mut Vec<TokenTree>) - -> TokenTree { - use syntax::parse::token::*; - - let (tree, is_joint) = stream.as_tree(); - let (span, token) = match tree { - tokenstream::TokenTree::Token(span, token) => (span, token), - tokenstream::TokenTree::Delimited(span, delimed) => { - let delimiter = Delimiter::from_internal(delimed.delim); - let mut g = Group::new(delimiter, TokenStream(delimed.tts.into())); - g.set_span(Span(span)); - return g.into() - } - }; - - let op_kind = if is_joint { Spacing::Joint } else { Spacing::Alone }; - macro_rules! tt { - ($e:expr) => ({ - let mut x = TokenTree::from($e); - x.set_span(Span(span)); - x - }) - } - macro_rules! op { - ($a:expr) => (tt!(Punct::new($a, op_kind))); - ($a:expr, $b:expr) => ({ - stack.push(tt!(Punct::new($b, op_kind))); - tt!(Punct::new($a, Spacing::Joint)) - }); - ($a:expr, $b:expr, $c:expr) => ({ - stack.push(tt!(Punct::new($c, op_kind))); - stack.push(tt!(Punct::new($b, Spacing::Joint))); - tt!(Punct::new($a, Spacing::Joint)) - }) - } - - match token { - Eq => op!('='), - Lt => op!('<'), - Le => op!('<', '='), - EqEq => op!('=', '='), - Ne => op!('!', '='), - Ge => op!('>', '='), - Gt => op!('>'), - AndAnd => op!('&', '&'), - OrOr => op!('|', '|'), - Not => op!('!'), - Tilde => op!('~'), - BinOp(Plus) => op!('+'), - BinOp(Minus) => op!('-'), - BinOp(Star) => op!('*'), - BinOp(Slash) => op!('/'), - BinOp(Percent) => op!('%'), - BinOp(Caret) => op!('^'), - BinOp(And) => op!('&'), - BinOp(Or) => op!('|'), - BinOp(Shl) => op!('<', '<'), - BinOp(Shr) => op!('>', '>'), - BinOpEq(Plus) => op!('+', '='), - BinOpEq(Minus) => op!('-', '='), - BinOpEq(Star) => op!('*', '='), - BinOpEq(Slash) => op!('/', '='), - BinOpEq(Percent) => op!('%', '='), - BinOpEq(Caret) => op!('^', '='), - BinOpEq(And) => op!('&', '='), - BinOpEq(Or) => op!('|', '='), - BinOpEq(Shl) => op!('<', '<', '='), - BinOpEq(Shr) => op!('>', '>', '='), - At => op!('@'), - Dot => op!('.'), - DotDot => op!('.', '.'), - DotDotDot => op!('.', '.', '.'), - DotDotEq => op!('.', '.', '='), - Comma => op!(','), - Semi => op!(';'), - Colon => op!(':'), - ModSep => op!(':', ':'), - RArrow => op!('-', '>'), - LArrow => op!('<', '-'), - FatArrow => op!('=', '>'), - Pound => op!('#'), - Dollar => op!('$'), - Question => op!('?'), - SingleQuote => op!('\''), - - Ident(ident, false) => { - tt!(self::Ident::new(&ident.as_str(), Span(span))) - } - Ident(ident, true) => { - tt!(self::Ident::new_raw(&ident.as_str(), Span(span))) - } - Lifetime(ident) => { - let ident = ident.without_first_quote(); - stack.push(tt!(self::Ident::new(&ident.as_str(), Span(span)))); - tt!(Punct::new('\'', Spacing::Joint)) - } - Literal(lit, suffix) => tt!(self::Literal { lit, suffix, span: Span(span) }), - DocComment(c) => { - let style = comments::doc_comment_style(&c.as_str()); - let stripped = comments::strip_doc_comment_decoration(&c.as_str()); - let stream = vec![ - tt!(self::Ident::new("doc", Span(span))), - tt!(Punct::new('=', Spacing::Alone)), - tt!(self::Literal::string(&stripped)), - ].into_iter().collect(); - stack.push(tt!(Group::new(Delimiter::Bracket, stream))); - if style == ast::AttrStyle::Inner { - stack.push(tt!(Punct::new('!', Spacing::Alone))); - } - tt!(Punct::new('#', Spacing::Alone)) - } - - Interpolated(_) => { - __internal::with_sess(|sess, _| { - let tts = token.interpolated_to_tokenstream(sess, span); - tt!(Group::new(Delimiter::None, TokenStream(tts))) - }) - } - - DotEq => op!('.', '='), - OpenDelim(..) | CloseDelim(..) => unreachable!(), - Whitespace | Comment | Shebang(..) | Eof => unreachable!(), - } - } - - fn to_internal(self) -> tokenstream::TokenStream { - use syntax::parse::token::*; - use syntax::tokenstream::{TokenTree, Delimited}; - - let (ch, kind, span) = match self { - self::TokenTree::Punct(tt) => (tt.as_char(), tt.spacing(), tt.span()), - self::TokenTree::Group(tt) => { - return TokenTree::Delimited(tt.span.0, Delimited { - delim: tt.delimiter.to_internal(), - tts: tt.stream.0.into(), - }).into(); - }, - self::TokenTree::Ident(tt) => { - let token = Ident(ast::Ident::new(tt.sym, tt.span.0), tt.is_raw); - return TokenTree::Token(tt.span.0, token).into(); - } - self::TokenTree::Literal(self::Literal { - lit: Lit::Integer(ref a), - suffix, - span, - }) - if a.as_str().starts_with("-") => - { - let minus = BinOp(BinOpToken::Minus); - let integer = Symbol::intern(&a.as_str()[1..]); - let integer = Literal(Lit::Integer(integer), suffix); - let a = TokenTree::Token(span.0, minus); - let b = TokenTree::Token(span.0, integer); - return vec![a, b].into_iter().collect() - } - self::TokenTree::Literal(self::Literal { - lit: Lit::Float(ref a), - suffix, - span, - }) - if a.as_str().starts_with("-") => - { - let minus = BinOp(BinOpToken::Minus); - let float = Symbol::intern(&a.as_str()[1..]); - let float = Literal(Lit::Float(float), suffix); - let a = TokenTree::Token(span.0, minus); - let b = TokenTree::Token(span.0, float); - return vec![a, b].into_iter().collect() - } - self::TokenTree::Literal(tt) => { - let token = Literal(tt.lit, tt.suffix); - return TokenTree::Token(tt.span.0, token).into() - } - }; - - let token = match ch { - '=' => Eq, - '<' => Lt, - '>' => Gt, - '!' => Not, - '~' => Tilde, - '+' => BinOp(Plus), - '-' => BinOp(Minus), - '*' => BinOp(Star), - '/' => BinOp(Slash), - '%' => BinOp(Percent), - '^' => BinOp(Caret), - '&' => BinOp(And), - '|' => BinOp(Or), - '@' => At, - '.' => Dot, - ',' => Comma, - ';' => Semi, - ':' => Colon, - '#' => Pound, - '$' => Dollar, - '?' => Question, - '\'' => SingleQuote, - _ => unreachable!(), - }; - - let tree = TokenTree::Token(span.0, token); - match kind { - Spacing::Alone => tree.into(), - Spacing::Joint => tree.joint(), - } - } -} - /// Permanently unstable internal implementation details of this crate. This /// should not be used. /// @@ -1364,8 +1156,6 @@ impl TokenTree { #[unstable(feature = "proc_macro_internals", issue = "27812")] #[doc(hidden)] pub mod __internal { - pub use quote::{LiteralKind, SpannedSymbol, Quoter, unquote}; - use std::cell::Cell; use std::ptr; diff --git a/src/libproc_macro/quote.rs b/src/libproc_macro/quote.rs index 73a66640c59..7ae7b13a152 100644 --- a/src/libproc_macro/quote.rs +++ b/src/libproc_macro/quote.rs @@ -14,35 +14,26 @@ //! This quasiquoter uses macros 2.0 hygiene to reliably access //! items from `proc_macro`, to build a `proc_macro::TokenStream`. -use {Delimiter, Literal, Spacing, Span, Ident, Punct, Group, TokenStream, TokenTree}; - -use syntax::ext::base::{ExtCtxt, ProcMacro}; -use syntax::parse::token; -use syntax::symbol::Symbol; -use syntax::tokenstream; - -/// This is the actual quote!() proc macro -/// -/// It is manually loaded in CStore::load_macro_untracked -pub struct Quoter; - -pub fn unquote<T: Into<TokenStream> + Clone>(tokens: &T) -> TokenStream { - tokens.clone().into() -} - -pub trait Quote { - fn quote(self) -> TokenStream; -} - -macro_rules! tt2ts { - ($e:expr) => (TokenStream::from(TokenTree::from($e))) -} - -macro_rules! quote_tok { - (,) => { tt2ts!(Punct::new(',', Spacing::Alone)) }; - (.) => { tt2ts!(Punct::new('.', Spacing::Alone)) }; - (:) => { tt2ts!(Punct::new(':', Spacing::Alone)) }; - (|) => { tt2ts!(Punct::new('|', Spacing::Alone)) }; +use {Delimiter, Group, Ident, Literal, Punct, Spacing, Span, TokenStream, TokenTree}; + +macro_rules! quote_tt { + (($($t:tt)*)) => { Group::new(Delimiter::Parenthesis, quote!($($t)*)) }; + ([$($t:tt)*]) => { Group::new(Delimiter::Bracket, quote!($($t)*)) }; + ({$($t:tt)*}) => { Group::new(Delimiter::Brace, quote!($($t)*)) }; + (,) => { Punct::new(',', Spacing::Alone) }; + (.) => { Punct::new('.', Spacing::Alone) }; + (:) => { Punct::new(':', Spacing::Alone) }; + (;) => { Punct::new(';', Spacing::Alone) }; + (!) => { Punct::new('!', Spacing::Alone) }; + (<) => { Punct::new('<', Spacing::Alone) }; + (>) => { Punct::new('>', Spacing::Alone) }; + (&) => { Punct::new('&', Spacing::Alone) }; + (=) => { Punct::new('=', Spacing::Alone) }; + ($i:ident) => { Ident::new(stringify!($i), Span::def_site()) }; +} + +macro_rules! quote_ts { + ((@ $($t:tt)*)) => { $($t)* }; (::) => { [ TokenTree::from(Punct::new(':', Spacing::Joint)), @@ -55,65 +46,45 @@ macro_rules! quote_tok { }) .collect::<TokenStream>() }; - (!) => { tt2ts!(Punct::new('!', Spacing::Alone)) }; - (<) => { tt2ts!(Punct::new('<', Spacing::Alone)) }; - (>) => { tt2ts!(Punct::new('>', Spacing::Alone)) }; - (_) => { tt2ts!(Punct::new('_', Spacing::Alone)) }; - (0) => { tt2ts!(Literal::i8_unsuffixed(0)) }; - (&) => { tt2ts!(Punct::new('&', Spacing::Alone)) }; - ($i:ident) => { tt2ts!(Ident::new(stringify!($i), Span::def_site())) }; -} - -macro_rules! quote_tree { - ((unquote $($t:tt)*)) => { $($t)* }; - ((quote $($t:tt)*)) => { ($($t)*).quote() }; - (($($t:tt)*)) => { tt2ts!(Group::new(Delimiter::Parenthesis, quote!($($t)*))) }; - ([$($t:tt)*]) => { tt2ts!(Group::new(Delimiter::Bracket, quote!($($t)*))) }; - ({$($t:tt)*}) => { tt2ts!(Group::new(Delimiter::Brace, quote!($($t)*))) }; - ($t:tt) => { quote_tok!($t) }; + ($t:tt) => { TokenTree::from(quote_tt!($t)) }; } +/// Simpler version of the real `quote!` macro, implemented solely +/// through `macro_rules`, for bootstrapping the real implementation +/// (see the `quote` function), which does not have access to the +/// real `quote!` macro due to the `proc_macro` crate not being +/// able to depend on itself. +/// +/// Note: supported tokens are a subset of the real `quote!`, but +/// unquoting is different: instead of `$x`, this uses `(@ expr)`. macro_rules! quote { () => { TokenStream::new() }; ($($t:tt)*) => { - [$(quote_tree!($t),)*].iter() - .cloned() - .flat_map(|x| x.into_iter()) - .collect::<TokenStream>() + [ + $(TokenStream::from(quote_ts!($t)),)* + ].iter().cloned().collect::<TokenStream>() }; } -impl ProcMacro for Quoter { - fn expand<'cx>(&self, cx: &'cx mut ExtCtxt, - _: ::syntax_pos::Span, - stream: tokenstream::TokenStream) - -> tokenstream::TokenStream { - ::__internal::set_sess(cx, || TokenStream(stream).quote().0) - } -} - -impl<T: Quote> Quote for Option<T> { - fn quote(self) -> TokenStream { - match self { - Some(t) => quote!(Some((quote t))), - None => quote!(None), - } +/// Quote a `TokenStream` into a `TokenStream`. +/// This is the actual `quote!()` proc macro. +/// +/// It is manually loaded in `CStore::load_macro_untracked`. +#[unstable(feature = "proc_macro_quote", issue = "38356")] +pub fn quote(stream: TokenStream) -> TokenStream { + if stream.is_empty() { + return quote!(::TokenStream::new()); } -} - -impl Quote for TokenStream { - fn quote(self) -> TokenStream { - if self.is_empty() { - return quote!(::TokenStream::new()); - } - let mut after_dollar = false; - let tokens = self.into_iter().filter_map(|tree| { + let mut after_dollar = false; + let tokens = stream + .into_iter() + .filter_map(|tree| { if after_dollar { after_dollar = false; match tree { TokenTree::Ident(_) => { - let tree = TokenStream::from(tree); - return Some(quote!(::__internal::unquote(&(unquote tree)),)); + return Some(quote!(Into::<::TokenStream>::into( + Clone::clone(&(@ tree))),)); } TokenTree::Punct(ref tt) if tt.as_char() == '$' => {} _ => panic!("`$` must be followed by an ident or `$` in `quote!`"), @@ -125,186 +96,55 @@ impl Quote for TokenStream { } } - Some(quote!(::TokenStream::from((quote tree)),)) - }).flat_map(|t| t.into_iter()).collect::<TokenStream>(); - - if after_dollar { - panic!("unexpected trailing `$` in `quote!`"); - } - - quote!( - [(unquote tokens)].iter() - .cloned() - .flat_map(|x| x.into_iter()) - .collect::<::TokenStream>() - ) - } -} - -impl Quote for TokenTree { - fn quote(self) -> TokenStream { - match self { - TokenTree::Punct(tt) => quote!(::TokenTree::Punct( (quote tt) )), - TokenTree::Group(tt) => quote!(::TokenTree::Group( (quote tt) )), - TokenTree::Ident(tt) => quote!(::TokenTree::Ident( (quote tt) )), - TokenTree::Literal(tt) => quote!(::TokenTree::Literal( (quote tt) )), - } - } -} - -impl Quote for char { - fn quote(self) -> TokenStream { - TokenTree::from(Literal::character(self)).into() - } -} - -impl<'a> Quote for &'a str { - fn quote(self) -> TokenStream { - TokenTree::from(Literal::string(self)).into() - } -} - -impl Quote for u16 { - fn quote(self) -> TokenStream { - TokenTree::from(Literal::u16_unsuffixed(self)).into() - } -} - -impl Quote for Group { - fn quote(self) -> TokenStream { - quote!(::Group::new((quote self.delimiter()), (quote self.stream()))) - } -} - -impl Quote for Punct { - fn quote(self) -> TokenStream { - quote!(::Punct::new((quote self.as_char()), (quote self.spacing()))) - } -} - -impl Quote for Ident { - fn quote(self) -> TokenStream { - quote!(::Ident::new((quote self.sym.as_str()), (quote self.span()))) - } -} - -impl Quote for Span { - fn quote(self) -> TokenStream { - quote!(::Span::def_site()) - } -} - -macro_rules! literals { - ($($i:ident),*; $($raw:ident),*) => { - pub struct SpannedSymbol { - sym: Symbol, - span: Span, - } - - impl SpannedSymbol { - pub fn new(string: &str, span: Span) -> SpannedSymbol { - SpannedSymbol { sym: Symbol::intern(string), span } - } - } - - impl Quote for SpannedSymbol { - fn quote(self) -> TokenStream { - quote!(::__internal::SpannedSymbol::new((quote self.sym.as_str()), - (quote self.span))) - } - } - - pub enum LiteralKind { - $($i,)* - $($raw(u16),)* - } - - impl LiteralKind { - pub fn with_contents_and_suffix(self, contents: SpannedSymbol, - suffix: Option<SpannedSymbol>) -> Literal { - let sym = contents.sym; - let suffix = suffix.map(|t| t.sym); - match self { - $(LiteralKind::$i => { - Literal { - lit: token::Lit::$i(sym), - suffix, - span: contents.span, - } - })* - $(LiteralKind::$raw(n) => { - Literal { - lit: token::Lit::$raw(sym, n), - suffix, - span: contents.span, - } - })* - } - } - } - - impl Literal { - fn kind_contents_and_suffix(self) -> (LiteralKind, SpannedSymbol, Option<SpannedSymbol>) - { - let (kind, contents) = match self.lit { - $(token::Lit::$i(contents) => (LiteralKind::$i, contents),)* - $(token::Lit::$raw(contents, n) => (LiteralKind::$raw(n), contents),)* - }; - let suffix = self.suffix.map(|sym| SpannedSymbol::new(&sym.as_str(), self.span())); - (kind, SpannedSymbol::new(&contents.as_str(), self.span()), suffix) - } - } - - impl Quote for LiteralKind { - fn quote(self) -> TokenStream { - match self { - $(LiteralKind::$i => quote! { - ::__internal::LiteralKind::$i - },)* - $(LiteralKind::$raw(n) => quote! { - ::__internal::LiteralKind::$raw((quote n)) - },)* - } - } - } + Some(quote!(::TokenStream::from((@ match tree { + TokenTree::Punct(tt) => quote!(::TokenTree::Punct(::Punct::new( + (@ TokenTree::from(Literal::character(tt.as_char()))), + (@ match tt.spacing() { + Spacing::Alone => quote!(::Spacing::Alone), + Spacing::Joint => quote!(::Spacing::Joint), + }), + ))), + TokenTree::Group(tt) => quote!(::TokenTree::Group(::Group::new( + (@ match tt.delimiter() { + Delimiter::Parenthesis => quote!(::Delimiter::Parenthesis), + Delimiter::Brace => quote!(::Delimiter::Brace), + Delimiter::Bracket => quote!(::Delimiter::Bracket), + Delimiter::None => quote!(::Delimiter::None), + }), + (@ quote(tt.stream())), + ))), + TokenTree::Ident(tt) => quote!(::TokenTree::Ident(::Ident::new( + (@ TokenTree::from(Literal::string(&tt.to_string()))), + (@ quote_span(tt.span())), + ))), + TokenTree::Literal(tt) => quote!(::TokenTree::Literal({ + let mut iter = (@ TokenTree::from(Literal::string(&tt.to_string()))) + .parse::<::TokenStream>() + .unwrap() + .into_iter(); + if let (Some(::TokenTree::Literal(mut lit)), None) = + (iter.next(), iter.next()) + { + lit.set_span((@ quote_span(tt.span()))); + lit + } else { + unreachable!() + } + })) + })),)) + }) + .collect::<TokenStream>(); - impl Quote for Literal { - fn quote(self) -> TokenStream { - let (kind, contents, suffix) = self.kind_contents_and_suffix(); - quote! { - (quote kind).with_contents_and_suffix((quote contents), (quote suffix)) - } - } - } + if after_dollar { + panic!("unexpected trailing `$` in `quote!`"); } -} -literals!(Byte, Char, Float, Str_, Integer, ByteStr; StrRaw, ByteStrRaw); - -impl Quote for Delimiter { - fn quote(self) -> TokenStream { - macro_rules! gen_match { - ($($i:ident),*) => { - match self { - $(Delimiter::$i => { quote!(::Delimiter::$i) })* - } - } - } - - gen_match!(Parenthesis, Brace, Bracket, None) - } + quote!([(@ tokens)].iter().cloned().collect::<::TokenStream>()) } -impl Quote for Spacing { - fn quote(self) -> TokenStream { - macro_rules! gen_match { - ($($i:ident),*) => { - match self { - $(Spacing::$i => { quote!(::Spacing::$i) })* - } - } - } - - gen_match!(Alone, Joint) - } +/// Quote a `Span` into a `TokenStream`. +/// This is needed to implement a custom quoter. +#[unstable(feature = "proc_macro_quote", issue = "38356")] +pub fn quote_span(_: Span) -> TokenStream { + quote!(::Span::def_site()) } diff --git a/src/libproc_macro/rustc.rs b/src/libproc_macro/rustc.rs new file mode 100644 index 00000000000..a54c695f637 --- /dev/null +++ b/src/libproc_macro/rustc.rs @@ -0,0 +1,284 @@ +// Copyright 2018 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license +// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use {Delimiter, Level, Spacing, Span, __internal}; +use {Group, Ident, Literal, Punct, TokenTree}; + +use rustc_errors as errors; +use syntax::ast; +use syntax::parse::lexer::comments; +use syntax::parse::token; +use syntax::tokenstream; +use syntax_pos::symbol::{keywords, Symbol}; + +impl Ident { + pub(crate) fn new_maybe_raw(string: &str, span: Span, is_raw: bool) -> Ident { + let sym = Symbol::intern(string); + if is_raw + && (sym == keywords::Underscore.name() + || ast::Ident::with_empty_ctxt(sym).is_path_segment_keyword()) + { + panic!("`{:?}` is not a valid raw identifier", string) + } + Ident { sym, span, is_raw } + } +} + +impl Delimiter { + pub(crate) fn from_internal(delim: token::DelimToken) -> Delimiter { + match delim { + token::Paren => Delimiter::Parenthesis, + token::Brace => Delimiter::Brace, + token::Bracket => Delimiter::Bracket, + token::NoDelim => Delimiter::None, + } + } + + pub(crate) fn to_internal(self) -> token::DelimToken { + match self { + Delimiter::Parenthesis => token::Paren, + Delimiter::Brace => token::Brace, + Delimiter::Bracket => token::Bracket, + Delimiter::None => token::NoDelim, + } + } +} + +impl TokenTree { + pub(crate) fn from_internal( + stream: tokenstream::TokenStream, + stack: &mut Vec<TokenTree>, + ) -> TokenTree { + use syntax::parse::token::*; + + let (tree, is_joint) = stream.as_tree(); + let (span, token) = match tree { + tokenstream::TokenTree::Token(span, token) => (span, token), + tokenstream::TokenTree::Delimited(span, delimed) => { + let delimiter = Delimiter::from_internal(delimed.delim); + let mut g = Group::new(delimiter, ::TokenStream(delimed.tts.into())); + g.set_span(Span(span)); + return g.into(); + } + }; + + let op_kind = if is_joint { + Spacing::Joint + } else { + Spacing::Alone + }; + macro_rules! tt { + ($e:expr) => {{ + let mut x = TokenTree::from($e); + x.set_span(Span(span)); + x + }}; + } + macro_rules! op { + ($a:expr) => { + tt!(Punct::new($a, op_kind)) + }; + ($a:expr, $b:expr) => {{ + stack.push(tt!(Punct::new($b, op_kind))); + tt!(Punct::new($a, Spacing::Joint)) + }}; + ($a:expr, $b:expr, $c:expr) => {{ + stack.push(tt!(Punct::new($c, op_kind))); + stack.push(tt!(Punct::new($b, Spacing::Joint))); + tt!(Punct::new($a, Spacing::Joint)) + }}; + } + + match token { + Eq => op!('='), + Lt => op!('<'), + Le => op!('<', '='), + EqEq => op!('=', '='), + Ne => op!('!', '='), + Ge => op!('>', '='), + Gt => op!('>'), + AndAnd => op!('&', '&'), + OrOr => op!('|', '|'), + Not => op!('!'), + Tilde => op!('~'), + BinOp(Plus) => op!('+'), + BinOp(Minus) => op!('-'), + BinOp(Star) => op!('*'), + BinOp(Slash) => op!('/'), + BinOp(Percent) => op!('%'), + BinOp(Caret) => op!('^'), + BinOp(And) => op!('&'), + BinOp(Or) => op!('|'), + BinOp(Shl) => op!('<', '<'), + BinOp(Shr) => op!('>', '>'), + BinOpEq(Plus) => op!('+', '='), + BinOpEq(Minus) => op!('-', '='), + BinOpEq(Star) => op!('*', '='), + BinOpEq(Slash) => op!('/', '='), + BinOpEq(Percent) => op!('%', '='), + BinOpEq(Caret) => op!('^', '='), + BinOpEq(And) => op!('&', '='), + BinOpEq(Or) => op!('|', '='), + BinOpEq(Shl) => op!('<', '<', '='), + BinOpEq(Shr) => op!('>', '>', '='), + At => op!('@'), + Dot => op!('.'), + DotDot => op!('.', '.'), + DotDotDot => op!('.', '.', '.'), + DotDotEq => op!('.', '.', '='), + Comma => op!(','), + Semi => op!(';'), + Colon => op!(':'), + ModSep => op!(':', ':'), + RArrow => op!('-', '>'), + LArrow => op!('<', '-'), + FatArrow => op!('=', '>'), + Pound => op!('#'), + Dollar => op!('$'), + Question => op!('?'), + SingleQuote => op!('\''), + + Ident(ident, false) => tt!(self::Ident::new(&ident.as_str(), Span(span))), + Ident(ident, true) => tt!(self::Ident::new_raw(&ident.as_str(), Span(span))), + Lifetime(ident) => { + let ident = ident.without_first_quote(); + stack.push(tt!(self::Ident::new(&ident.as_str(), Span(span)))); + tt!(Punct::new('\'', Spacing::Joint)) + } + Literal(lit, suffix) => tt!(self::Literal { + lit, + suffix, + span: Span(span) + }), + DocComment(c) => { + let style = comments::doc_comment_style(&c.as_str()); + let stripped = comments::strip_doc_comment_decoration(&c.as_str()); + let stream = vec![ + tt!(self::Ident::new("doc", Span(span))), + tt!(Punct::new('=', Spacing::Alone)), + tt!(self::Literal::string(&stripped)), + ].into_iter() + .collect(); + stack.push(tt!(Group::new(Delimiter::Bracket, stream))); + if style == ast::AttrStyle::Inner { + stack.push(tt!(Punct::new('!', Spacing::Alone))); + } + tt!(Punct::new('#', Spacing::Alone)) + } + + Interpolated(_) => __internal::with_sess(|sess, _| { + let tts = token.interpolated_to_tokenstream(sess, span); + tt!(Group::new(Delimiter::None, ::TokenStream(tts))) + }), + + DotEq => op!('.', '='), + OpenDelim(..) | CloseDelim(..) => unreachable!(), + Whitespace | Comment | Shebang(..) | Eof => unreachable!(), + } + } + + pub(crate) fn to_internal(self) -> tokenstream::TokenStream { + use syntax::parse::token::*; + use syntax::tokenstream::{Delimited, TokenTree}; + + let (ch, kind, span) = match self { + self::TokenTree::Punct(tt) => (tt.as_char(), tt.spacing(), tt.span()), + self::TokenTree::Group(tt) => { + return TokenTree::Delimited( + tt.span.0, + Delimited { + delim: tt.delimiter.to_internal(), + tts: tt.stream.0.into(), + }, + ).into(); + } + self::TokenTree::Ident(tt) => { + let token = Ident(ast::Ident::new(tt.sym, tt.span.0), tt.is_raw); + return TokenTree::Token(tt.span.0, token).into(); + } + self::TokenTree::Literal(self::Literal { + lit: Lit::Integer(ref a), + suffix, + span, + }) + if a.as_str().starts_with("-") => + { + let minus = BinOp(BinOpToken::Minus); + let integer = Symbol::intern(&a.as_str()[1..]); + let integer = Literal(Lit::Integer(integer), suffix); + let a = TokenTree::Token(span.0, minus); + let b = TokenTree::Token(span.0, integer); + return vec![a, b].into_iter().collect(); + } + self::TokenTree::Literal(self::Literal { + lit: Lit::Float(ref a), + suffix, + span, + }) + if a.as_str().starts_with("-") => + { + let minus = BinOp(BinOpToken::Minus); + let float = Symbol::intern(&a.as_str()[1..]); + let float = Literal(Lit::Float(float), suffix); + let a = TokenTree::Token(span.0, minus); + let b = TokenTree::Token(span.0, float); + return vec![a, b].into_iter().collect(); + } + self::TokenTree::Literal(tt) => { + let token = Literal(tt.lit, tt.suffix); + return TokenTree::Token(tt.span.0, token).into(); + } + }; + + let token = match ch { + '=' => Eq, + '<' => Lt, + '>' => Gt, + '!' => Not, + '~' => Tilde, + '+' => BinOp(Plus), + '-' => BinOp(Minus), + '*' => BinOp(Star), + '/' => BinOp(Slash), + '%' => BinOp(Percent), + '^' => BinOp(Caret), + '&' => BinOp(And), + '|' => BinOp(Or), + '@' => At, + '.' => Dot, + ',' => Comma, + ';' => Semi, + ':' => Colon, + '#' => Pound, + '$' => Dollar, + '?' => Question, + '\'' => SingleQuote, + _ => unreachable!(), + }; + + let tree = TokenTree::Token(span.0, token); + match kind { + Spacing::Alone => tree.into(), + Spacing::Joint => tree.joint(), + } + } +} + +impl Level { + pub(crate) fn to_internal(self) -> errors::Level { + match self { + Level::Error => errors::Level::Error, + Level::Warning => errors::Level::Warning, + Level::Note => errors::Level::Note, + Level::Help => errors::Level::Help, + Level::__Nonexhaustive => unreachable!("Level::__Nonexhaustive"), + } + } +} diff --git a/src/librustc_errors/diagnostic.rs b/src/librustc_errors/diagnostic.rs index d079102a4ba..b1578b697bb 100644 --- a/src/librustc_errors/diagnostic.rs +++ b/src/librustc_errors/diagnostic.rs @@ -379,7 +379,7 @@ impl Diagnostic { /// Convenience function for internal use, clients should use one of the /// public methods above. - pub(crate) fn sub(&mut self, + pub fn sub(&mut self, level: Level, message: &str, span: MultiSpan, diff --git a/src/librustc_metadata/cstore_impl.rs b/src/librustc_metadata/cstore_impl.rs index d5078642147..e3a7918f8c5 100644 --- a/src/librustc_metadata/cstore_impl.rs +++ b/src/librustc_metadata/cstore_impl.rs @@ -39,7 +39,6 @@ use syntax::ast; use syntax::attr; use syntax::codemap; use syntax::edition::Edition; -use syntax::ext::base::SyntaxExtension; use syntax::parse::filemap_to_stream; use syntax::symbol::Symbol; use syntax_pos::{Span, NO_EXPANSION, FileName}; @@ -517,8 +516,11 @@ impl CrateStore for cstore::CStore { return LoadedMacro::ProcMacro(proc_macros[id.index.to_proc_macro_index()].1.clone()); } else if data.name == "proc_macro" && self.get_crate_data(id.krate).item_name(id.index) == "quote" { + use syntax::ext::base::SyntaxExtension; + use syntax_ext::proc_macro_impl::BangProcMacro; + let ext = SyntaxExtension::ProcMacro { - expander: Box::new(::proc_macro::__internal::Quoter), + expander: Box::new(BangProcMacro { inner: ::proc_macro::quote }), allow_internal_unstable: true, edition: data.root.edition, }; diff --git a/src/librustc_metadata/lib.rs b/src/librustc_metadata/lib.rs index 5c9915e94e5..d535c1ef903 100644 --- a/src/librustc_metadata/lib.rs +++ b/src/librustc_metadata/lib.rs @@ -19,6 +19,7 @@ #![feature(libc)] #![feature(macro_at_most_once_rep)] #![feature(proc_macro_internals)] +#![feature(proc_macro_quote)] #![feature(quote)] #![feature(rustc_diagnostic_macros)] #![feature(slice_sort_by_cached_key)] diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs index bf790e6143a..9748e2947ee 100644 --- a/src/libsyntax/parse/lexer/mod.rs +++ b/src/libsyntax/parse/lexer/mod.rs @@ -1775,12 +1775,6 @@ fn ident_continue(c: Option<char>) -> bool { (c > '\x7f' && c.is_xid_continue()) } -// The string is a valid identifier or a lifetime identifier. -pub fn is_valid_ident(s: &str) -> bool { - let mut chars = s.chars(); - ident_start(chars.next()) && chars.all(|ch| ident_continue(Some(ch))) -} - #[cfg(test)] mod tests { use super::*; |