diff options
Diffstat (limited to 'src/libsyntax_ext/proc_macro_server.rs')
| -rw-r--r-- | src/libsyntax_ext/proc_macro_server.rs | 751 |
1 files changed, 751 insertions, 0 deletions
diff --git a/src/libsyntax_ext/proc_macro_server.rs b/src/libsyntax_ext/proc_macro_server.rs new file mode 100644 index 00000000000..56bd58b28a6 --- /dev/null +++ b/src/libsyntax_ext/proc_macro_server.rs @@ -0,0 +1,751 @@ +// Copyright 2018 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license +// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use errors::{self, Diagnostic, DiagnosticBuilder}; +use std::panic; + +use proc_macro::bridge::{server, TokenTree}; +use proc_macro::{Delimiter, Level, LineColumn, Spacing}; + +use rustc_data_structures::sync::Lrc; +use std::ascii; +use std::ops::Bound; +use syntax::ast; +use syntax::ext::base::ExtCtxt; +use syntax::parse::lexer::comments; +use syntax::parse::{self, token, ParseSess}; +use syntax::tokenstream::{self, DelimSpan, TokenStream}; +use syntax_pos::hygiene::{SyntaxContext, Transparency}; +use syntax_pos::symbol::{keywords, Symbol}; +use syntax_pos::{BytePos, FileName, MultiSpan, Pos, SourceFile, Span}; + +trait FromInternal<T> { + fn from_internal(x: T) -> Self; +} + +trait ToInternal<T> { + fn to_internal(self) -> T; +} + +impl FromInternal<token::DelimToken> for Delimiter { + fn from_internal(delim: token::DelimToken) -> Delimiter { + match delim { + token::Paren => Delimiter::Parenthesis, + token::Brace => Delimiter::Brace, + token::Bracket => Delimiter::Bracket, + token::NoDelim => Delimiter::None, + } + } +} + +impl ToInternal<token::DelimToken> for Delimiter { + fn to_internal(self) -> token::DelimToken { + match self { + Delimiter::Parenthesis => token::Paren, + Delimiter::Brace => token::Brace, + Delimiter::Bracket => token::Bracket, + Delimiter::None => token::NoDelim, + } + } +} + +impl FromInternal<(TokenStream, &'_ ParseSess, &'_ mut Vec<Self>)> + for TokenTree<Group, Punct, Ident, Literal> +{ + fn from_internal((stream, sess, stack): (TokenStream, &ParseSess, &mut Vec<Self>)) -> Self { + use syntax::parse::token::*; + + let (tree, joint) = stream.as_tree(); + let (span, token) = match tree { + tokenstream::TokenTree::Delimited(span, delimed) => { + let delimiter = Delimiter::from_internal(delimed.delim); + return TokenTree::Group(Group { + delimiter, + stream: delimed.tts.into(), + span, + }); + } + tokenstream::TokenTree::Token(span, token) => (span, token), + }; + + macro_rules! tt { + ($ty:ident { $($field:ident $(: $value:expr)*),+ $(,)* }) => ( + TokenTree::$ty(self::$ty { + $($field $(: $value)*,)* + span, + }) + ) + } + macro_rules! op { + ($a:expr) => { + tt!(Punct { ch: $a, joint }) + }; + ($a:expr, $b:expr) => {{ + stack.push(tt!(Punct { ch: $b, joint })); + tt!(Punct { + ch: $a, + joint: true + }) + }}; + ($a:expr, $b:expr, $c:expr) => {{ + stack.push(tt!(Punct { ch: $c, joint })); + stack.push(tt!(Punct { + ch: $b, + joint: true + })); + tt!(Punct { + ch: $a, + joint: true + }) + }}; + } + + match token { + Eq => op!('='), + Lt => op!('<'), + Le => op!('<', '='), + EqEq => op!('=', '='), + Ne => op!('!', '='), + Ge => op!('>', '='), + Gt => op!('>'), + AndAnd => op!('&', '&'), + OrOr => op!('|', '|'), + Not => op!('!'), + Tilde => op!('~'), + BinOp(Plus) => op!('+'), + BinOp(Minus) => op!('-'), + BinOp(Star) => op!('*'), + BinOp(Slash) => op!('/'), + BinOp(Percent) => op!('%'), + BinOp(Caret) => op!('^'), + BinOp(And) => op!('&'), + BinOp(Or) => op!('|'), + BinOp(Shl) => op!('<', '<'), + BinOp(Shr) => op!('>', '>'), + BinOpEq(Plus) => op!('+', '='), + BinOpEq(Minus) => op!('-', '='), + BinOpEq(Star) => op!('*', '='), + BinOpEq(Slash) => op!('/', '='), + BinOpEq(Percent) => op!('%', '='), + BinOpEq(Caret) => op!('^', '='), + BinOpEq(And) => op!('&', '='), + BinOpEq(Or) => op!('|', '='), + BinOpEq(Shl) => op!('<', '<', '='), + BinOpEq(Shr) => op!('>', '>', '='), + At => op!('@'), + Dot => op!('.'), + DotDot => op!('.', '.'), + DotDotDot => op!('.', '.', '.'), + DotDotEq => op!('.', '.', '='), + Comma => op!(','), + Semi => op!(';'), + Colon => op!(':'), + ModSep => op!(':', ':'), + RArrow => op!('-', '>'), + LArrow => op!('<', '-'), + FatArrow => op!('=', '>'), + Pound => op!('#'), + Dollar => op!('$'), + Question => op!('?'), + SingleQuote => op!('\''), + + Ident(ident, is_raw) => tt!(Ident { + sym: ident.name, + is_raw + }), + Lifetime(ident) => { + let ident = ident.without_first_quote(); + stack.push(tt!(Ident { + sym: ident.name, + is_raw: false + })); + tt!(Punct { + ch: '\'', + joint: true + }) + } + Literal(lit, suffix) => tt!(Literal { lit, suffix }), + DocComment(c) => { + let style = comments::doc_comment_style(&c.as_str()); + let stripped = comments::strip_doc_comment_decoration(&c.as_str()); + let mut escaped = String::new(); + for ch in stripped.chars() { + escaped.extend(ch.escape_debug()); + } + let stream = vec![ + Ident(ast::Ident::new(Symbol::intern("doc"), span), false), + Eq, + Literal(Lit::Str_(Symbol::intern(&escaped)), None), + ] + .into_iter() + .map(|token| tokenstream::TokenTree::Token(span, token)) + .collect(); + stack.push(TokenTree::Group(Group { + delimiter: Delimiter::Bracket, + stream, + span: DelimSpan::from_single(span), + })); + if style == ast::AttrStyle::Inner { + stack.push(tt!(Punct { + ch: '!', + joint: false + })); + } + tt!(Punct { + ch: '#', + joint: false + }) + } + + Interpolated(_) => { + let stream = token.interpolated_to_tokenstream(sess, span); + TokenTree::Group(Group { + delimiter: Delimiter::None, + stream, + span: DelimSpan::from_single(span), + }) + } + + DotEq => op!('.', '='), + OpenDelim(..) | CloseDelim(..) => unreachable!(), + Whitespace | Comment | Shebang(..) | Eof => unreachable!(), + } + } +} + +impl ToInternal<TokenStream> for TokenTree<Group, Punct, Ident, Literal> { + fn to_internal(self) -> TokenStream { + use syntax::parse::token::*; + + let (ch, joint, span) = match self { + TokenTree::Punct(Punct { ch, joint, span }) => (ch, joint, span), + TokenTree::Group(Group { + delimiter, + stream, + span, + }) => { + return tokenstream::TokenTree::Delimited( + span, + tokenstream::Delimited { + delim: delimiter.to_internal(), + tts: stream.into(), + }, + ) + .into(); + } + TokenTree::Ident(self::Ident { sym, span, is_raw }) => { + let token = Ident(ast::Ident::new(sym, span), is_raw); + return tokenstream::TokenTree::Token(span, token).into(); + } + TokenTree::Literal(self::Literal { + lit: Lit::Integer(ref a), + suffix, + span, + }) if a.as_str().starts_with("-") => { + let minus = BinOp(BinOpToken::Minus); + let integer = Symbol::intern(&a.as_str()[1..]); + let integer = Literal(Lit::Integer(integer), suffix); + let a = tokenstream::TokenTree::Token(span, minus); + let b = tokenstream::TokenTree::Token(span, integer); + return vec![a, b].into_iter().collect(); + } + TokenTree::Literal(self::Literal { + lit: Lit::Float(ref a), + suffix, + span, + }) if a.as_str().starts_with("-") => { + let minus = BinOp(BinOpToken::Minus); + let float = Symbol::intern(&a.as_str()[1..]); + let float = Literal(Lit::Float(float), suffix); + let a = tokenstream::TokenTree::Token(span, minus); + let b = tokenstream::TokenTree::Token(span, float); + return vec![a, b].into_iter().collect(); + } + TokenTree::Literal(self::Literal { lit, suffix, span }) => { + return tokenstream::TokenTree::Token(span, Literal(lit, suffix)).into() + } + }; + + let token = match ch { + '=' => Eq, + '<' => Lt, + '>' => Gt, + '!' => Not, + '~' => Tilde, + '+' => BinOp(Plus), + '-' => BinOp(Minus), + '*' => BinOp(Star), + '/' => BinOp(Slash), + '%' => BinOp(Percent), + '^' => BinOp(Caret), + '&' => BinOp(And), + '|' => BinOp(Or), + '@' => At, + '.' => Dot, + ',' => Comma, + ';' => Semi, + ':' => Colon, + '#' => Pound, + '$' => Dollar, + '?' => Question, + '\'' => SingleQuote, + _ => unreachable!(), + }; + + let tree = tokenstream::TokenTree::Token(span, token); + if joint { + tree.joint() + } else { + tree.into() + } + } +} + +impl ToInternal<errors::Level> for Level { + fn to_internal(self) -> errors::Level { + match self { + Level::Error => errors::Level::Error, + Level::Warning => errors::Level::Warning, + Level::Note => errors::Level::Note, + Level::Help => errors::Level::Help, + _ => unreachable!("unknown proc_macro::Level variant: {:?}", self), + } + } +} + +#[derive(Clone)] +pub struct TokenStreamIter { + cursor: tokenstream::Cursor, + stack: Vec<TokenTree<Group, Punct, Ident, Literal>>, +} + +#[derive(Clone)] +pub struct Group { + delimiter: Delimiter, + stream: TokenStream, + span: DelimSpan, +} + +#[derive(Copy, Clone, PartialEq, Eq, Hash)] +pub struct Punct { + ch: char, + // NB. not using `Spacing` here because it doesn't implement `Hash`. + joint: bool, + span: Span, +} + +#[derive(Copy, Clone, PartialEq, Eq, Hash)] +pub struct Ident { + sym: Symbol, + span: Span, + is_raw: bool, +} + +// FIXME(eddyb) `Literal` should not expose internal `Debug` impls. +#[derive(Clone, Debug)] +pub struct Literal { + lit: token::Lit, + suffix: Option<Symbol>, + span: Span, +} + +pub(crate) struct Rustc<'a> { + sess: &'a ParseSess, + def_site: Span, + call_site: Span, +} + +impl<'a> Rustc<'a> { + pub fn new(cx: &'a ExtCtxt) -> Self { + // No way to determine def location for a proc macro right now, so use call location. + let location = cx.current_expansion.mark.expn_info().unwrap().call_site; + let to_span = |transparency| { + location.with_ctxt( + SyntaxContext::empty() + .apply_mark_with_transparency(cx.current_expansion.mark, transparency), + ) + }; + Rustc { + sess: cx.parse_sess, + def_site: to_span(Transparency::Opaque), + call_site: to_span(Transparency::Transparent), + } + } +} + +impl server::Types for Rustc<'_> { + type TokenStream = TokenStream; + type TokenStreamBuilder = tokenstream::TokenStreamBuilder; + type TokenStreamIter = TokenStreamIter; + type Group = Group; + type Punct = Punct; + type Ident = Ident; + type Literal = Literal; + type SourceFile = Lrc<SourceFile>; + type MultiSpan = Vec<Span>; + type Diagnostic = Diagnostic; + type Span = Span; +} + +impl server::TokenStream for Rustc<'_> { + fn new(&mut self) -> Self::TokenStream { + TokenStream::empty() + } + fn is_empty(&mut self, stream: &Self::TokenStream) -> bool { + stream.is_empty() + } + fn from_str(&mut self, src: &str) -> Self::TokenStream { + parse::parse_stream_from_source_str( + FileName::ProcMacroSourceCode, + src.to_string(), + self.sess, + Some(self.call_site), + ) + } + fn to_string(&mut self, stream: &Self::TokenStream) -> String { + stream.to_string() + } + fn from_token_tree( + &mut self, + tree: TokenTree<Self::Group, Self::Punct, Self::Ident, Self::Literal>, + ) -> Self::TokenStream { + tree.to_internal() + } + fn into_iter(&mut self, stream: Self::TokenStream) -> Self::TokenStreamIter { + TokenStreamIter { + cursor: stream.trees(), + stack: vec![], + } + } +} + +impl server::TokenStreamBuilder for Rustc<'_> { + fn new(&mut self) -> Self::TokenStreamBuilder { + tokenstream::TokenStreamBuilder::new() + } + fn push(&mut self, builder: &mut Self::TokenStreamBuilder, stream: Self::TokenStream) { + builder.push(stream); + } + fn build(&mut self, builder: Self::TokenStreamBuilder) -> Self::TokenStream { + builder.build() + } +} + +impl server::TokenStreamIter for Rustc<'_> { + fn next( + &mut self, + iter: &mut Self::TokenStreamIter, + ) -> Option<TokenTree<Self::Group, Self::Punct, Self::Ident, Self::Literal>> { + loop { + let tree = iter.stack.pop().or_else(|| { + let next = iter.cursor.next_as_stream()?; + Some(TokenTree::from_internal((next, self.sess, &mut iter.stack))) + })?; + // HACK: The condition "dummy span + group with empty delimiter" represents an AST + // fragment approximately converted into a token stream. This may happen, for + // example, with inputs to proc macro attributes, including derives. Such "groups" + // need to flattened during iteration over stream's token trees. + // Eventually this needs to be removed in favor of keeping original token trees + // and not doing the roundtrip through AST. + if let TokenTree::Group(ref group) = tree { + if group.delimiter == Delimiter::None && group.span.entire().is_dummy() { + iter.cursor.insert(group.stream.clone()); + continue; + } + } + return Some(tree); + } + } +} + +impl server::Group for Rustc<'_> { + fn new(&mut self, delimiter: Delimiter, stream: Self::TokenStream) -> Self::Group { + Group { + delimiter, + stream, + span: DelimSpan::from_single(server::Span::call_site(self)), + } + } + fn delimiter(&mut self, group: &Self::Group) -> Delimiter { + group.delimiter + } + fn stream(&mut self, group: &Self::Group) -> Self::TokenStream { + group.stream.clone() + } + fn span(&mut self, group: &Self::Group) -> Self::Span { + group.span.entire() + } + fn span_open(&mut self, group: &Self::Group) -> Self::Span { + group.span.open + } + fn span_close(&mut self, group: &Self::Group) -> Self::Span { + group.span.close + } + fn set_span(&mut self, group: &mut Self::Group, span: Self::Span) { + group.span = DelimSpan::from_single(span); + } +} + +impl server::Punct for Rustc<'_> { + fn new(&mut self, ch: char, spacing: Spacing) -> Self::Punct { + Punct { + ch, + joint: spacing == Spacing::Joint, + span: server::Span::call_site(self), + } + } + fn as_char(&mut self, punct: Self::Punct) -> char { + punct.ch + } + fn spacing(&mut self, punct: Self::Punct) -> Spacing { + if punct.joint { + Spacing::Joint + } else { + Spacing::Alone + } + } + fn span(&mut self, punct: Self::Punct) -> Self::Span { + punct.span + } + fn with_span(&mut self, punct: Self::Punct, span: Self::Span) -> Self::Punct { + Punct { span, ..punct } + } +} + +impl server::Ident for Rustc<'_> { + fn new(&mut self, string: &str, span: Self::Span, is_raw: bool) -> Self::Ident { + let sym = Symbol::intern(string); + if is_raw + && (sym == keywords::Underscore.name() + || ast::Ident::with_empty_ctxt(sym).is_path_segment_keyword()) + { + panic!("`{:?}` is not a valid raw identifier", string) + } + Ident { sym, span, is_raw } + } + fn span(&mut self, ident: Self::Ident) -> Self::Span { + ident.span + } + fn with_span(&mut self, ident: Self::Ident, span: Self::Span) -> Self::Ident { + Ident { span, ..ident } + } +} + +impl server::Literal for Rustc<'_> { + // FIXME(eddyb) `Literal` should not expose internal `Debug` impls. + fn debug(&mut self, literal: &Self::Literal) -> String { + format!("{:?}", literal) + } + fn integer(&mut self, n: &str) -> Self::Literal { + Literal { + lit: token::Lit::Integer(Symbol::intern(n)), + suffix: None, + span: server::Span::call_site(self), + } + } + fn typed_integer(&mut self, n: &str, kind: &str) -> Self::Literal { + Literal { + lit: token::Lit::Integer(Symbol::intern(n)), + suffix: Some(Symbol::intern(kind)), + span: server::Span::call_site(self), + } + } + fn float(&mut self, n: &str) -> Self::Literal { + Literal { + lit: token::Lit::Float(Symbol::intern(n)), + suffix: None, + span: server::Span::call_site(self), + } + } + fn f32(&mut self, n: &str) -> Self::Literal { + Literal { + lit: token::Lit::Float(Symbol::intern(n)), + suffix: Some(Symbol::intern("f32")), + span: server::Span::call_site(self), + } + } + fn f64(&mut self, n: &str) -> Self::Literal { + Literal { + lit: token::Lit::Float(Symbol::intern(n)), + suffix: Some(Symbol::intern("f64")), + span: server::Span::call_site(self), + } + } + fn string(&mut self, string: &str) -> Self::Literal { + let mut escaped = String::new(); + for ch in string.chars() { + escaped.extend(ch.escape_debug()); + } + Literal { + lit: token::Lit::Str_(Symbol::intern(&escaped)), + suffix: None, + span: server::Span::call_site(self), + } + } + fn character(&mut self, ch: char) -> Self::Literal { + let mut escaped = String::new(); + escaped.extend(ch.escape_unicode()); + Literal { + lit: token::Lit::Char(Symbol::intern(&escaped)), + suffix: None, + span: server::Span::call_site(self), + } + } + fn byte_string(&mut self, bytes: &[u8]) -> Self::Literal { + let string = bytes + .iter() + .cloned() + .flat_map(ascii::escape_default) + .map(Into::<char>::into) + .collect::<String>(); + Literal { + lit: token::Lit::ByteStr(Symbol::intern(&string)), + suffix: None, + span: server::Span::call_site(self), + } + } + fn span(&mut self, literal: &Self::Literal) -> Self::Span { + literal.span + } + fn set_span(&mut self, literal: &mut Self::Literal, span: Self::Span) { + literal.span = span; + } + fn subspan( + &mut self, + literal: &Self::Literal, + start: Bound<usize>, + end: Bound<usize>, + ) -> Option<Self::Span> { + let span = literal.span; + let length = span.hi().to_usize() - span.lo().to_usize(); + + let start = match start { + Bound::Included(lo) => lo, + Bound::Excluded(lo) => lo + 1, + Bound::Unbounded => 0, + }; + + let end = match end { + Bound::Included(hi) => hi + 1, + Bound::Excluded(hi) => hi, + Bound::Unbounded => length, + }; + + // Bounds check the values, preventing addition overflow and OOB spans. + if start > u32::max_value() as usize + || end > u32::max_value() as usize + || (u32::max_value() - start as u32) < span.lo().to_u32() + || (u32::max_value() - end as u32) < span.lo().to_u32() + || start >= end + || end > length + { + return None; + } + + let new_lo = span.lo() + BytePos::from_usize(start); + let new_hi = span.lo() + BytePos::from_usize(end); + Some(span.with_lo(new_lo).with_hi(new_hi)) + } +} + +impl<'a> server::SourceFile for Rustc<'a> { + fn eq(&mut self, file1: &Self::SourceFile, file2: &Self::SourceFile) -> bool { + Lrc::ptr_eq(file1, file2) + } + fn path(&mut self, file: &Self::SourceFile) -> String { + match file.name { + FileName::Real(ref path) => path + .to_str() + .expect("non-UTF8 file path in `proc_macro::SourceFile::path`") + .to_string(), + _ => file.name.to_string(), + } + } + fn is_real(&mut self, file: &Self::SourceFile) -> bool { + file.is_real_file() + } +} + +impl server::MultiSpan for Rustc<'_> { + fn new(&mut self) -> Self::MultiSpan { + vec![] + } + fn push(&mut self, spans: &mut Self::MultiSpan, span: Self::Span) { + spans.push(span) + } +} + +impl server::Diagnostic for Rustc<'_> { + fn new(&mut self, level: Level, msg: &str, spans: Self::MultiSpan) -> Self::Diagnostic { + let mut diag = Diagnostic::new(level.to_internal(), msg); + diag.set_span(MultiSpan::from_spans(spans)); + diag + } + fn sub( + &mut self, + diag: &mut Self::Diagnostic, + level: Level, + msg: &str, + spans: Self::MultiSpan, + ) { + diag.sub(level.to_internal(), msg, MultiSpan::from_spans(spans), None); + } + fn emit(&mut self, diag: Self::Diagnostic) { + DiagnosticBuilder::new_diagnostic(&self.sess.span_diagnostic, diag).emit() + } +} + +impl server::Span for Rustc<'_> { + fn debug(&mut self, span: Self::Span) -> String { + format!("{:?} bytes({}..{})", span.ctxt(), span.lo().0, span.hi().0) + } + fn def_site(&mut self) -> Self::Span { + self.def_site + } + fn call_site(&mut self) -> Self::Span { + self.call_site + } + fn source_file(&mut self, span: Self::Span) -> Self::SourceFile { + self.sess.source_map().lookup_char_pos(span.lo()).file + } + fn parent(&mut self, span: Self::Span) -> Option<Self::Span> { + span.ctxt().outer().expn_info().map(|i| i.call_site) + } + fn source(&mut self, span: Self::Span) -> Self::Span { + span.source_callsite() + } + fn start(&mut self, span: Self::Span) -> LineColumn { + let loc = self.sess.source_map().lookup_char_pos(span.lo()); + LineColumn { + line: loc.line, + column: loc.col.to_usize(), + } + } + fn end(&mut self, span: Self::Span) -> LineColumn { + let loc = self.sess.source_map().lookup_char_pos(span.hi()); + LineColumn { + line: loc.line, + column: loc.col.to_usize(), + } + } + fn join(&mut self, first: Self::Span, second: Self::Span) -> Option<Self::Span> { + let self_loc = self.sess.source_map().lookup_char_pos(first.lo()); + let other_loc = self.sess.source_map().lookup_char_pos(second.lo()); + + if self_loc.file.name != other_loc.file.name { + return None; + } + + Some(first.to(second)) + } + fn resolved_at(&mut self, span: Self::Span, at: Self::Span) -> Self::Span { + span.with_ctxt(at.ctxt()) + } +} |