Auto merge of #49219 - eddyb:proc-macro-decouple, r=alexcrichton

Decouple proc_macro from the rest of the compiler.

This PR removes all dependencies of `proc_macro` on compiler crates and allows multiple copies of `proc_macro`, built even by different compilers (but from the same source), to interoperate.

Practically, it allows:
* running proc macro tests at stage1 (I moved most from `-fulldeps` to the regular suites)
* using proc macros in the compiler itself (may require some rustbuild trickery)

On the server (i.e. compiler front-end) side:
* `server::*` traits are implemented to provide the concrete types and methods
  * the concrete types are completely separated from the `proc_macro` public API
  * the only use of the type implementing `Server` is to be passed to `Client::run`

On the client (i.e. proc macro) side (potentially using a different `proc_macro` instance!):
* `client::Client` wraps around client-side (expansion) function pointers
  * it encapsulates the `proc_macro` instance used by the client
  * its `run` method can be called by a server, to execute the client-side function
    * the client instance is bridged to the provided server, while it runs
    * ~~currently a thread is spawned, could use process isolation in the future~~
(not the case anymore, see #56058)
* proc macro crates get a generated `static` holding a `&[ProcMacro]`
  * this describes all derives/attr/bang proc macros, replacing the "registrar" function
  * each variant of `ProcMacro` contains an appropriately typed `Client<fn(...) -> ...>`

`proc_macro` public APIs call into the server via an internal "bridge":
* only a currently running proc macro `Client` can interact with those APIs
  * server code might not be able to (if it uses a different `proc_macro` instance)
    * however, it can always create and `run` its own `Client`, but that may be inefficient
* the `bridge` uses serialization, C ABI and integer handles to avoid Rust ABI instability
* each invocation of a proc macro results in disjoint integers in its `proc_macro` handles
  * this prevents using values of those types across invocations (if they even can be kept)

r? @alexcrichton cc @jseyfried @nikomatsakis @Zoxc @thepowersgang

6 files changed, 878 insertions, 139 deletions

diff --git a/src/libsyntax_ext/Cargo.toml b/src/libsyntax_ext/Cargo.toml
index 5a691bde3ec..4979d0b3e92 100644
--- a/src/libsyntax_ext/Cargo.toml
+++ b/src/libsyntax_ext/Cargo.toml
@@ -10,7 +10,6 @@ crate-type = ["dylib"]
 
 [dependencies]
 fmt_macros = { path = "../libfmt_macros" }
-proc_macro = { path = "../libproc_macro" }
 rustc_errors = { path = "../librustc_errors" }
 syntax = { path = "../libsyntax" }
 syntax_pos = { path = "../libsyntax_pos" }
diff --git a/src/libsyntax_ext/deriving/custom.rs b/src/libsyntax_ext/deriving/custom.rs
index 55b3928d68e..5c82d191138 100644
--- a/src/libsyntax_ext/deriving/custom.rs
+++ b/src/libsyntax_ext/deriving/custom.rs
@@ -8,15 +8,18 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use std::panic;
-
 use errors::FatalError;
-use proc_macro::{TokenStream, __internal};
 use syntax::ast::{self, ItemKind, Attribute, Mac};
 use syntax::attr::{mark_used, mark_known};
 use syntax::source_map::Span;
 use syntax::ext::base::*;
+use syntax::parse;
+use syntax::parse::token::{self, Token};
+use syntax::tokenstream;
 use syntax::visit::Visitor;
+use syntax_pos::DUMMY_SP;
+
+use proc_macro_impl::EXEC_STRATEGY;
 
 struct MarkAttrs<'a>(&'a [ast::Name]);
 
@@ -32,14 +35,10 @@ impl<'a> Visitor<'a> for MarkAttrs<'a> {
 }
 
 pub struct ProcMacroDerive {
-    inner: fn(TokenStream) -> TokenStream,
-    attrs: Vec<ast::Name>,
-}
-
-impl ProcMacroDerive {
-    pub fn new(inner: fn(TokenStream) -> TokenStream, attrs: Vec<ast::Name>) -> ProcMacroDerive {
-        ProcMacroDerive { inner: inner, attrs: attrs }
-    }
+    pub client: ::proc_macro::bridge::client::Client<
+        fn(::proc_macro::TokenStream) -> ::proc_macro::TokenStream,
+    >,
+    pub attrs: Vec<ast::Name>,
 }
 
 impl MultiItemModifier for ProcMacroDerive {
@@ -75,21 +74,17 @@ impl MultiItemModifier for ProcMacroDerive {
         // Mark attributes as known, and used.
         MarkAttrs(&self.attrs).visit_item(&item);
 
-        let input = __internal::new_token_stream(ecx.resolver.eliminate_crate_var(item));
-        let res = __internal::set_sess(ecx, || {
-            let inner = self.inner;
-            panic::catch_unwind(panic::AssertUnwindSafe(|| inner(input)))
-        });
+        let item = ecx.resolver.eliminate_crate_var(item);
+        let token = Token::interpolated(token::NtItem(item));
+        let input = tokenstream::TokenTree::Token(DUMMY_SP, token).into();
 
-        let stream = match res {
+        let server = ::proc_macro_server::Rustc::new(ecx);
+        let stream = match self.client.run(&EXEC_STRATEGY, server, input) {
             Ok(stream) => stream,
             Err(e) => {
                 let msg = "proc-macro derive panicked";
                 let mut err = ecx.struct_span_fatal(span, msg);
-                if let Some(s) = e.downcast_ref::<String>() {
-                    err.help(&format!("message: {}", s));
-                }
-                if let Some(s) = e.downcast_ref::<&'static str>() {
+                if let Some(s) = e.as_str() {
                     err.help(&format!("message: {}", s));
                 }
 
@@ -99,21 +94,33 @@ impl MultiItemModifier for ProcMacroDerive {
         };
 
         let error_count_before = ecx.parse_sess.span_diagnostic.err_count();
-        __internal::set_sess(ecx, || {
-            let msg = "proc-macro derive produced unparseable tokens";
-            match __internal::token_stream_parse_items(stream) {
-                // fail if there have been errors emitted
-                Ok(_) if ecx.parse_sess.span_diagnostic.err_count() > error_count_before => {
-                    ecx.struct_span_fatal(span, msg).emit();
-                    FatalError.raise();
+        let msg = "proc-macro derive produced unparseable tokens";
+
+        let mut parser = parse::stream_to_parser(ecx.parse_sess, stream);
+        let mut items = vec![];
+
+        loop {
+            match parser.parse_item() {
+                Ok(None) => break,
+                Ok(Some(item)) => {
+                    items.push(Annotatable::Item(item))
                 }
-                Ok(new_items) => new_items.into_iter().map(Annotatable::Item).collect(),
-                Err(_) => {
+                Err(mut err) => {
                     // FIXME: handle this better
+                    err.cancel();
                     ecx.struct_span_fatal(span, msg).emit();
                     FatalError.raise();
                 }
             }
-        })
+        }
+
+
+        // fail if there have been errors emitted
+        if ecx.parse_sess.span_diagnostic.err_count() > error_count_before {
+            ecx.struct_span_fatal(span, msg).emit();
+            FatalError.raise();
+        }
+
+        items
     }
 }
diff --git a/src/libsyntax_ext/lib.rs b/src/libsyntax_ext/lib.rs
index 7c023fc5c9c..1d814a67876 100644
--- a/src/libsyntax_ext/lib.rs
+++ b/src/libsyntax_ext/lib.rs
@@ -14,7 +14,10 @@
        html_favicon_url = "https://doc.rust-lang.org/favicon.ico",
        html_root_url = "https://doc.rust-lang.org/nightly/")]
 
+#![feature(in_band_lifetimes)]
+#![feature(proc_macro_diagnostic)]
 #![feature(proc_macro_internals)]
+#![feature(proc_macro_span)]
 #![feature(decl_macro)]
 #![feature(nll)]
 #![feature(str_escape)]
@@ -55,10 +58,9 @@ mod trace_macros;
 mod test;
 mod test_case;
 
-pub mod proc_macro_registrar;
-
-
+pub mod proc_macro_decls;
 pub mod proc_macro_impl;
+mod proc_macro_server;
 
 use rustc_data_structures::sync::Lrc;
 use syntax::ast;
diff --git a/src/libsyntax_ext/proc_macro_registrar.rs b/src/libsyntax_ext/proc_macro_decls.rs
index 65e175f95df..c859275ed02 100644
--- a/src/libsyntax_ext/proc_macro_registrar.rs
+++ b/src/libsyntax_ext/proc_macro_decls.rs
@@ -91,7 +91,7 @@ pub fn modify(sess: &ParseSess,
         return krate;
     }
 
-    krate.module.items.push(mk_registrar(&mut cx, &derives, &attr_macros, &bang_macros));
+    krate.module.items.push(mk_decls(&mut cx, &derives, &attr_macros, &bang_macros));
 
     krate
 }
@@ -339,19 +339,21 @@ impl<'a> Visitor<'a> for CollectProcMacros<'a> {
 //      mod $gensym {
 //          extern crate proc_macro;
 //
-//          use proc_macro::__internal::Registry;
+//          use proc_macro::bridge::client::ProcMacro;
 //
-//          #[plugin_registrar]
-//          fn registrar(registrar: &mut Registry) {
-//              registrar.register_custom_derive($name_trait1, ::$name1, &[]);
-//              registrar.register_custom_derive($name_trait2, ::$name2, &["attribute_name"]);
+//          #[rustc_proc_macro_decls]
+//          static DECLS: &[ProcMacro] = &[
+//              ProcMacro::custom_derive($name_trait1, &[], ::$name1);
+//              ProcMacro::custom_derive($name_trait2, &["attribute_name"], ::$name2);
 //              // ...
-//          }
+//          ];
 //      }
-fn mk_registrar(cx: &mut ExtCtxt,
-                custom_derives: &[ProcMacroDerive],
-                custom_attrs: &[ProcMacroDef],
-                custom_macros: &[ProcMacroDef]) -> P<ast::Item> {
+fn mk_decls(
+    cx: &mut ExtCtxt,
+    custom_derives: &[ProcMacroDerive],
+    custom_attrs: &[ProcMacroDef],
+    custom_macros: &[ProcMacroDef],
+) -> P<ast::Item> {
     let mark = Mark::fresh(Mark::root());
     mark.set_expn_info(ExpnInfo {
         call_site: DUMMY_SP,
@@ -370,75 +372,67 @@ fn mk_registrar(cx: &mut ExtCtxt,
                         Vec::new(),
                         ast::ItemKind::ExternCrate(None));
 
-    let __internal = Ident::from_str("__internal");
-    let registry = Ident::from_str("Registry");
-    let registrar = Ident::from_str("_registrar");
-    let register_custom_derive = Ident::from_str("register_custom_derive");
-    let register_attr_proc_macro = Ident::from_str("register_attr_proc_macro");
-    let register_bang_proc_macro = Ident::from_str("register_bang_proc_macro");
+    let bridge = Ident::from_str("bridge");
+    let client = Ident::from_str("client");
+    let proc_macro_ty = Ident::from_str("ProcMacro");
+    let custom_derive = Ident::from_str("custom_derive");
+    let attr = Ident::from_str("attr");
+    let bang = Ident::from_str("bang");
     let crate_kw = Ident::with_empty_ctxt(keywords::Crate.name());
-    let local_path = |cx: &mut ExtCtxt, sp: Span, name: Ident| {
-        cx.path(sp.with_ctxt(span.ctxt()), vec![crate_kw, name])
+
+    let decls = {
+        let local_path = |sp: Span, name| {
+            cx.expr_path(cx.path(sp.with_ctxt(span.ctxt()), vec![crate_kw, name]))
+        };
+        let proc_macro_ty_method_path = |method| cx.expr_path(cx.path(span, vec![
+            proc_macro, bridge, client, proc_macro_ty, method,
+        ]));
+        custom_derives.iter().map(|cd| {
+            cx.expr_call(span, proc_macro_ty_method_path(custom_derive), vec![
+                cx.expr_str(cd.span, cd.trait_name),
+                cx.expr_vec_slice(
+                    span,
+                    cd.attrs.iter().map(|&s| cx.expr_str(cd.span, s)).collect::<Vec<_>>()
+                ),
+                local_path(cd.span, cd.function_name),
+            ])
+        }).chain(custom_attrs.iter().map(|ca| {
+            cx.expr_call(span, proc_macro_ty_method_path(attr), vec![
+                cx.expr_str(ca.span, ca.function_name.name),
+                local_path(ca.span, ca.function_name),
+            ])
+        })).chain(custom_macros.iter().map(|cm| {
+            cx.expr_call(span, proc_macro_ty_method_path(bang), vec![
+                cx.expr_str(cm.span, cm.function_name.name),
+                local_path(cm.span, cm.function_name),
+            ])
+        })).collect()
     };
 
-    let mut stmts = custom_derives.iter().map(|cd| {
-        let path = local_path(cx, cd.span, cd.function_name);
-        let trait_name = cx.expr_str(cd.span, cd.trait_name);
-        let attrs = cx.expr_vec_slice(
-            span,
-            cd.attrs.iter().map(|&s| cx.expr_str(cd.span, s)).collect::<Vec<_>>()
-        );
-        let registrar = cx.expr_ident(span, registrar);
-        let ufcs_path = cx.path(span, vec![proc_macro, __internal, registry,
-                                           register_custom_derive]);
-
-        cx.stmt_expr(cx.expr_call(span, cx.expr_path(ufcs_path),
-                                  vec![registrar, trait_name, cx.expr_path(path), attrs]))
-
-    }).collect::<Vec<_>>();
-
-    stmts.extend(custom_attrs.iter().map(|ca| {
-        let name = cx.expr_str(ca.span, ca.function_name.name);
-        let path = local_path(cx, ca.span, ca.function_name);
-        let registrar = cx.expr_ident(ca.span, registrar);
-
-        let ufcs_path = cx.path(span,
-                                vec![proc_macro, __internal, registry, register_attr_proc_macro]);
-
-        cx.stmt_expr(cx.expr_call(span, cx.expr_path(ufcs_path),
-                                  vec![registrar, name, cx.expr_path(path)]))
-    }));
-
-    stmts.extend(custom_macros.iter().map(|cm| {
-        let name = cx.expr_str(cm.span, cm.function_name.name);
-        let path = local_path(cx, cm.span, cm.function_name);
-        let registrar = cx.expr_ident(cm.span, registrar);
-
-        let ufcs_path = cx.path(span,
-                                vec![proc_macro, __internal, registry, register_bang_proc_macro]);
-
-        cx.stmt_expr(cx.expr_call(span, cx.expr_path(ufcs_path),
-                                  vec![registrar, name, cx.expr_path(path)]))
-    }));
-
-    let path = cx.path(span, vec![proc_macro, __internal, registry]);
-    let registrar_path = cx.ty_path(path);
-    let arg_ty = cx.ty_rptr(span, registrar_path, None, ast::Mutability::Mutable);
-    let func = cx.item_fn(span,
-                          registrar,
-                          vec![cx.arg(span, registrar, arg_ty)],
-                          cx.ty(span, ast::TyKind::Tup(Vec::new())),
-                          cx.block(span, stmts));
-
-    let derive_registrar = cx.meta_word(span, Symbol::intern("rustc_derive_registrar"));
-    let derive_registrar = cx.attribute(span, derive_registrar);
-    let func = func.map(|mut i| {
-        i.attrs.push(derive_registrar);
+    let decls_static = cx.item_static(
+        span,
+        Ident::from_str("_DECLS"),
+        cx.ty_rptr(span,
+            cx.ty(span, ast::TyKind::Slice(
+                cx.ty_path(cx.path(span,
+                    vec![proc_macro, bridge, client, proc_macro_ty])))),
+            None, ast::Mutability::Immutable),
+        ast::Mutability::Immutable,
+        cx.expr_vec_slice(span, decls),
+    ).map(|mut i| {
+        let attr = cx.meta_word(span, Symbol::intern("rustc_proc_macro_decls"));
+        i.attrs.push(cx.attribute(span, attr));
         i.vis = respan(span, ast::VisibilityKind::Public);
         i
     });
-    let ident = ast::Ident::with_empty_ctxt(Symbol::gensym("registrar"));
-    let module = cx.item_mod(span, span, ident, Vec::new(), vec![krate, func]).map(|mut i| {
+
+    let module = cx.item_mod(
+        span,
+        span,
+        ast::Ident::with_empty_ctxt(Symbol::gensym("decls")),
+        vec![],
+        vec![krate, decls_static],
+    ).map(|mut i| {
         i.vis = respan(span, ast::VisibilityKind::Public);
         i
     });
diff --git a/src/libsyntax_ext/proc_macro_impl.rs b/src/libsyntax_ext/proc_macro_impl.rs
index ff60262055b..43ef31a00ba 100644
--- a/src/libsyntax_ext/proc_macro_impl.rs
+++ b/src/libsyntax_ext/proc_macro_impl.rs
@@ -8,7 +8,6 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use std::panic;
 
 use errors::FatalError;
 
@@ -17,11 +16,13 @@ use syntax::ext::base::*;
 use syntax::tokenstream::TokenStream;
 use syntax::ext::base;
 
-use proc_macro::TokenStream as TsShim;
-use proc_macro::__internal;
+pub const EXEC_STRATEGY: ::proc_macro::bridge::server::SameThread =
+    ::proc_macro::bridge::server::SameThread;
 
 pub struct AttrProcMacro {
-    pub inner: fn(TsShim, TsShim) -> TsShim,
+    pub client: ::proc_macro::bridge::client::Client<
+        fn(::proc_macro::TokenStream, ::proc_macro::TokenStream) -> ::proc_macro::TokenStream,
+    >,
 }
 
 impl base::AttrProcMacro for AttrProcMacro {
@@ -31,22 +32,13 @@ impl base::AttrProcMacro for AttrProcMacro {
                    annotation: TokenStream,
                    annotated: TokenStream)
                    -> TokenStream {
-        let annotation = __internal::token_stream_wrap(annotation);
-        let annotated = __internal::token_stream_wrap(annotated);
-
-        let res = __internal::set_sess(ecx, || {
-            panic::catch_unwind(panic::AssertUnwindSafe(|| (self.inner)(annotation, annotated)))
-        });
-
-        match res {
-            Ok(stream) => __internal::token_stream_inner(stream),
+        let server = ::proc_macro_server::Rustc::new(ecx);
+        match self.client.run(&EXEC_STRATEGY, server, annotation, annotated) {
+            Ok(stream) => stream,
             Err(e) => {
                 let msg = "custom attribute panicked";
                 let mut err = ecx.struct_span_fatal(span, msg);
-                if let Some(s) = e.downcast_ref::<String>() {
-                    err.help(&format!("message: {}", s));
-                }
-                if let Some(s) = e.downcast_ref::<&'static str>() {
+                if let Some(s) = e.as_str() {
                     err.help(&format!("message: {}", s));
                 }
 
@@ -58,7 +50,9 @@ impl base::AttrProcMacro for AttrProcMacro {
 }
 
 pub struct BangProcMacro {
-    pub inner: fn(TsShim) -> TsShim,
+    pub client: ::proc_macro::bridge::client::Client<
+        fn(::proc_macro::TokenStream) -> ::proc_macro::TokenStream,
+    >,
 }
 
 impl base::ProcMacro for BangProcMacro {
@@ -67,21 +61,13 @@ impl base::ProcMacro for BangProcMacro {
                    span: Span,
                    input: TokenStream)
                    -> TokenStream {
-        let input = __internal::token_stream_wrap(input);
-
-        let res = __internal::set_sess(ecx, || {
-            panic::catch_unwind(panic::AssertUnwindSafe(|| (self.inner)(input)))
-        });
-
-        match res {
-            Ok(stream) => __internal::token_stream_inner(stream),
+        let server = ::proc_macro_server::Rustc::new(ecx);
+        match self.client.run(&EXEC_STRATEGY, server, input) {
+            Ok(stream) => stream,
             Err(e) => {
                 let msg = "proc macro panicked";
                 let mut err = ecx.struct_span_fatal(span, msg);
-                if let Some(s) = e.downcast_ref::<String>() {
-                    err.help(&format!("message: {}", s));
-                }
-                if let Some(s) = e.downcast_ref::<&'static str>() {
+                if let Some(s) = e.as_str() {
                     err.help(&format!("message: {}", s));
                 }
 
diff --git a/src/libsyntax_ext/proc_macro_server.rs b/src/libsyntax_ext/proc_macro_server.rs
new file mode 100644
index 00000000000..56bd58b28a6
--- /dev/null
+++ b/src/libsyntax_ext/proc_macro_server.rs
@@ -0,0 +1,751 @@
+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+use errors::{self, Diagnostic, DiagnosticBuilder};
+use std::panic;
+
+use proc_macro::bridge::{server, TokenTree};
+use proc_macro::{Delimiter, Level, LineColumn, Spacing};
+
+use rustc_data_structures::sync::Lrc;
+use std::ascii;
+use std::ops::Bound;
+use syntax::ast;
+use syntax::ext::base::ExtCtxt;
+use syntax::parse::lexer::comments;
+use syntax::parse::{self, token, ParseSess};
+use syntax::tokenstream::{self, DelimSpan, TokenStream};
+use syntax_pos::hygiene::{SyntaxContext, Transparency};
+use syntax_pos::symbol::{keywords, Symbol};
+use syntax_pos::{BytePos, FileName, MultiSpan, Pos, SourceFile, Span};
+
+trait FromInternal<T> {
+    fn from_internal(x: T) -> Self;
+}
+
+trait ToInternal<T> {
+    fn to_internal(self) -> T;
+}
+
+impl FromInternal<token::DelimToken> for Delimiter {
+    fn from_internal(delim: token::DelimToken) -> Delimiter {
+        match delim {
+            token::Paren => Delimiter::Parenthesis,
+            token::Brace => Delimiter::Brace,
+            token::Bracket => Delimiter::Bracket,
+            token::NoDelim => Delimiter::None,
+        }
+    }
+}
+
+impl ToInternal<token::DelimToken> for Delimiter {
+    fn to_internal(self) -> token::DelimToken {
+        match self {
+            Delimiter::Parenthesis => token::Paren,
+            Delimiter::Brace => token::Brace,
+            Delimiter::Bracket => token::Bracket,
+            Delimiter::None => token::NoDelim,
+        }
+    }
+}
+
+impl FromInternal<(TokenStream, &'_ ParseSess, &'_ mut Vec<Self>)>
+    for TokenTree<Group, Punct, Ident, Literal>
+{
+    fn from_internal((stream, sess, stack): (TokenStream, &ParseSess, &mut Vec<Self>)) -> Self {
+        use syntax::parse::token::*;
+
+        let (tree, joint) = stream.as_tree();
+        let (span, token) = match tree {
+            tokenstream::TokenTree::Delimited(span, delimed) => {
+                let delimiter = Delimiter::from_internal(delimed.delim);
+                return TokenTree::Group(Group {
+                    delimiter,
+                    stream: delimed.tts.into(),
+                    span,
+                });
+            }
+            tokenstream::TokenTree::Token(span, token) => (span, token),
+        };
+
+        macro_rules! tt {
+            ($ty:ident { $($field:ident $(: $value:expr)*),+ $(,)* }) => (
+                TokenTree::$ty(self::$ty {
+                    $($field $(: $value)*,)*
+                    span,
+                })
+            )
+        }
+        macro_rules! op {
+            ($a:expr) => {
+                tt!(Punct { ch: $a, joint })
+            };
+            ($a:expr, $b:expr) => {{
+                stack.push(tt!(Punct { ch: $b, joint }));
+                tt!(Punct {
+                    ch: $a,
+                    joint: true
+                })
+            }};
+            ($a:expr, $b:expr, $c:expr) => {{
+                stack.push(tt!(Punct { ch: $c, joint }));
+                stack.push(tt!(Punct {
+                    ch: $b,
+                    joint: true
+                }));
+                tt!(Punct {
+                    ch: $a,
+                    joint: true
+                })
+            }};
+        }
+
+        match token {
+            Eq => op!('='),
+            Lt => op!('<'),
+            Le => op!('<', '='),
+            EqEq => op!('=', '='),
+            Ne => op!('!', '='),
+            Ge => op!('>', '='),
+            Gt => op!('>'),
+            AndAnd => op!('&', '&'),
+            OrOr => op!('|', '|'),
+            Not => op!('!'),
+            Tilde => op!('~'),
+            BinOp(Plus) => op!('+'),
+            BinOp(Minus) => op!('-'),
+            BinOp(Star) => op!('*'),
+            BinOp(Slash) => op!('/'),
+            BinOp(Percent) => op!('%'),
+            BinOp(Caret) => op!('^'),
+            BinOp(And) => op!('&'),
+            BinOp(Or) => op!('|'),
+            BinOp(Shl) => op!('<', '<'),
+            BinOp(Shr) => op!('>', '>'),
+            BinOpEq(Plus) => op!('+', '='),
+            BinOpEq(Minus) => op!('-', '='),
+            BinOpEq(Star) => op!('*', '='),
+            BinOpEq(Slash) => op!('/', '='),
+            BinOpEq(Percent) => op!('%', '='),
+            BinOpEq(Caret) => op!('^', '='),
+            BinOpEq(And) => op!('&', '='),
+            BinOpEq(Or) => op!('|', '='),
+            BinOpEq(Shl) => op!('<', '<', '='),
+            BinOpEq(Shr) => op!('>', '>', '='),
+            At => op!('@'),
+            Dot => op!('.'),
+            DotDot => op!('.', '.'),
+            DotDotDot => op!('.', '.', '.'),
+            DotDotEq => op!('.', '.', '='),
+            Comma => op!(','),
+            Semi => op!(';'),
+            Colon => op!(':'),
+            ModSep => op!(':', ':'),
+            RArrow => op!('-', '>'),
+            LArrow => op!('<', '-'),
+            FatArrow => op!('=', '>'),
+            Pound => op!('#'),
+            Dollar => op!('$'),
+            Question => op!('?'),
+            SingleQuote => op!('\''),
+
+            Ident(ident, is_raw) => tt!(Ident {
+                sym: ident.name,
+                is_raw
+            }),
+            Lifetime(ident) => {
+                let ident = ident.without_first_quote();
+                stack.push(tt!(Ident {
+                    sym: ident.name,
+                    is_raw: false
+                }));
+                tt!(Punct {
+                    ch: '\'',
+                    joint: true
+                })
+            }
+            Literal(lit, suffix) => tt!(Literal { lit, suffix }),
+            DocComment(c) => {
+                let style = comments::doc_comment_style(&c.as_str());
+                let stripped = comments::strip_doc_comment_decoration(&c.as_str());
+                let mut escaped = String::new();
+                for ch in stripped.chars() {
+                    escaped.extend(ch.escape_debug());
+                }
+                let stream = vec![
+                    Ident(ast::Ident::new(Symbol::intern("doc"), span), false),
+                    Eq,
+                    Literal(Lit::Str_(Symbol::intern(&escaped)), None),
+                ]
+                .into_iter()
+                .map(|token| tokenstream::TokenTree::Token(span, token))
+                .collect();
+                stack.push(TokenTree::Group(Group {
+                    delimiter: Delimiter::Bracket,
+                    stream,
+                    span: DelimSpan::from_single(span),
+                }));
+                if style == ast::AttrStyle::Inner {
+                    stack.push(tt!(Punct {
+                        ch: '!',
+                        joint: false
+                    }));
+                }
+                tt!(Punct {
+                    ch: '#',
+                    joint: false
+                })
+            }
+
+            Interpolated(_) => {
+                let stream = token.interpolated_to_tokenstream(sess, span);
+                TokenTree::Group(Group {
+                    delimiter: Delimiter::None,
+                    stream,
+                    span: DelimSpan::from_single(span),
+                })
+            }
+
+            DotEq => op!('.', '='),
+            OpenDelim(..) | CloseDelim(..) => unreachable!(),
+            Whitespace | Comment | Shebang(..) | Eof => unreachable!(),
+        }
+    }
+}
+
+impl ToInternal<TokenStream> for TokenTree<Group, Punct, Ident, Literal> {
+    fn to_internal(self) -> TokenStream {
+        use syntax::parse::token::*;
+
+        let (ch, joint, span) = match self {
+            TokenTree::Punct(Punct { ch, joint, span }) => (ch, joint, span),
+            TokenTree::Group(Group {
+                delimiter,
+                stream,
+                span,
+            }) => {
+                return tokenstream::TokenTree::Delimited(
+                    span,
+                    tokenstream::Delimited {
+                        delim: delimiter.to_internal(),
+                        tts: stream.into(),
+                    },
+                )
+                .into();
+            }
+            TokenTree::Ident(self::Ident { sym, span, is_raw }) => {
+                let token = Ident(ast::Ident::new(sym, span), is_raw);
+                return tokenstream::TokenTree::Token(span, token).into();
+            }
+            TokenTree::Literal(self::Literal {
+                lit: Lit::Integer(ref a),
+                suffix,
+                span,
+            }) if a.as_str().starts_with("-") => {
+                let minus = BinOp(BinOpToken::Minus);
+                let integer = Symbol::intern(&a.as_str()[1..]);
+                let integer = Literal(Lit::Integer(integer), suffix);
+                let a = tokenstream::TokenTree::Token(span, minus);
+                let b = tokenstream::TokenTree::Token(span, integer);
+                return vec![a, b].into_iter().collect();
+            }
+            TokenTree::Literal(self::Literal {
+                lit: Lit::Float(ref a),
+                suffix,
+                span,
+            }) if a.as_str().starts_with("-") => {
+                let minus = BinOp(BinOpToken::Minus);
+                let float = Symbol::intern(&a.as_str()[1..]);
+                let float = Literal(Lit::Float(float), suffix);
+                let a = tokenstream::TokenTree::Token(span, minus);
+                let b = tokenstream::TokenTree::Token(span, float);
+                return vec![a, b].into_iter().collect();
+            }
+            TokenTree::Literal(self::Literal { lit, suffix, span }) => {
+                return tokenstream::TokenTree::Token(span, Literal(lit, suffix)).into()
+            }
+        };
+
+        let token = match ch {
+            '=' => Eq,
+            '<' => Lt,
+            '>' => Gt,
+            '!' => Not,
+            '~' => Tilde,
+            '+' => BinOp(Plus),
+            '-' => BinOp(Minus),
+            '*' => BinOp(Star),
+            '/' => BinOp(Slash),
+            '%' => BinOp(Percent),
+            '^' => BinOp(Caret),
+            '&' => BinOp(And),
+            '|' => BinOp(Or),
+            '@' => At,
+            '.' => Dot,
+            ',' => Comma,
+            ';' => Semi,
+            ':' => Colon,
+            '#' => Pound,
+            '$' => Dollar,
+            '?' => Question,
+            '\'' => SingleQuote,
+            _ => unreachable!(),
+        };
+
+        let tree = tokenstream::TokenTree::Token(span, token);
+        if joint {
+            tree.joint()
+        } else {
+            tree.into()
+        }
+    }
+}
+
+impl ToInternal<errors::Level> for Level {
+    fn to_internal(self) -> errors::Level {
+        match self {
+            Level::Error => errors::Level::Error,
+            Level::Warning => errors::Level::Warning,
+            Level::Note => errors::Level::Note,
+            Level::Help => errors::Level::Help,
+            _ => unreachable!("unknown proc_macro::Level variant: {:?}", self),
+        }
+    }
+}
+
+#[derive(Clone)]
+pub struct TokenStreamIter {
+    cursor: tokenstream::Cursor,
+    stack: Vec<TokenTree<Group, Punct, Ident, Literal>>,
+}
+
+#[derive(Clone)]
+pub struct Group {
+    delimiter: Delimiter,
+    stream: TokenStream,
+    span: DelimSpan,
+}
+
+#[derive(Copy, Clone, PartialEq, Eq, Hash)]
+pub struct Punct {
+    ch: char,
+    // NB. not using `Spacing` here because it doesn't implement `Hash`.
+    joint: bool,
+    span: Span,
+}
+
+#[derive(Copy, Clone, PartialEq, Eq, Hash)]
+pub struct Ident {
+    sym: Symbol,
+    span: Span,
+    is_raw: bool,
+}
+
+// FIXME(eddyb) `Literal` should not expose internal `Debug` impls.
+#[derive(Clone, Debug)]
+pub struct Literal {
+    lit: token::Lit,
+    suffix: Option<Symbol>,
+    span: Span,
+}
+
+pub(crate) struct Rustc<'a> {
+    sess: &'a ParseSess,
+    def_site: Span,
+    call_site: Span,
+}
+
+impl<'a> Rustc<'a> {
+    pub fn new(cx: &'a ExtCtxt) -> Self {
+        // No way to determine def location for a proc macro right now, so use call location.
+        let location = cx.current_expansion.mark.expn_info().unwrap().call_site;
+        let to_span = |transparency| {
+            location.with_ctxt(
+                SyntaxContext::empty()
+                    .apply_mark_with_transparency(cx.current_expansion.mark, transparency),
+            )
+        };
+        Rustc {
+            sess: cx.parse_sess,
+            def_site: to_span(Transparency::Opaque),
+            call_site: to_span(Transparency::Transparent),
+        }
+    }
+}
+
+impl server::Types for Rustc<'_> {
+    type TokenStream = TokenStream;
+    type TokenStreamBuilder = tokenstream::TokenStreamBuilder;
+    type TokenStreamIter = TokenStreamIter;
+    type Group = Group;
+    type Punct = Punct;
+    type Ident = Ident;
+    type Literal = Literal;
+    type SourceFile = Lrc<SourceFile>;
+    type MultiSpan = Vec<Span>;
+    type Diagnostic = Diagnostic;
+    type Span = Span;
+}
+
+impl server::TokenStream for Rustc<'_> {
+    fn new(&mut self) -> Self::TokenStream {
+        TokenStream::empty()
+    }
+    fn is_empty(&mut self, stream: &Self::TokenStream) -> bool {
+        stream.is_empty()
+    }
+    fn from_str(&mut self, src: &str) -> Self::TokenStream {
+        parse::parse_stream_from_source_str(
+            FileName::ProcMacroSourceCode,
+            src.to_string(),
+            self.sess,
+            Some(self.call_site),
+        )
+    }
+    fn to_string(&mut self, stream: &Self::TokenStream) -> String {
+        stream.to_string()
+    }
+    fn from_token_tree(
+        &mut self,
+        tree: TokenTree<Self::Group, Self::Punct, Self::Ident, Self::Literal>,
+    ) -> Self::TokenStream {
+        tree.to_internal()
+    }
+    fn into_iter(&mut self, stream: Self::TokenStream) -> Self::TokenStreamIter {
+        TokenStreamIter {
+            cursor: stream.trees(),
+            stack: vec![],
+        }
+    }
+}
+
+impl server::TokenStreamBuilder for Rustc<'_> {
+    fn new(&mut self) -> Self::TokenStreamBuilder {
+        tokenstream::TokenStreamBuilder::new()
+    }
+    fn push(&mut self, builder: &mut Self::TokenStreamBuilder, stream: Self::TokenStream) {
+        builder.push(stream);
+    }
+    fn build(&mut self, builder: Self::TokenStreamBuilder) -> Self::TokenStream {
+        builder.build()
+    }
+}
+
+impl server::TokenStreamIter for Rustc<'_> {
+    fn next(
+        &mut self,
+        iter: &mut Self::TokenStreamIter,
+    ) -> Option<TokenTree<Self::Group, Self::Punct, Self::Ident, Self::Literal>> {
+        loop {
+            let tree = iter.stack.pop().or_else(|| {
+                let next = iter.cursor.next_as_stream()?;
+                Some(TokenTree::from_internal((next, self.sess, &mut iter.stack)))
+            })?;
+            // HACK: The condition "dummy span + group with empty delimiter" represents an AST
+            // fragment approximately converted into a token stream. This may happen, for
+            // example, with inputs to proc macro attributes, including derives. Such "groups"
+            // need to flattened during iteration over stream's token trees.
+            // Eventually this needs to be removed in favor of keeping original token trees
+            // and not doing the roundtrip through AST.
+            if let TokenTree::Group(ref group) = tree {
+                if group.delimiter == Delimiter::None && group.span.entire().is_dummy() {
+                    iter.cursor.insert(group.stream.clone());
+                    continue;
+                }
+            }
+            return Some(tree);
+        }
+    }
+}
+
+impl server::Group for Rustc<'_> {
+    fn new(&mut self, delimiter: Delimiter, stream: Self::TokenStream) -> Self::Group {
+        Group {
+            delimiter,
+            stream,
+            span: DelimSpan::from_single(server::Span::call_site(self)),
+        }
+    }
+    fn delimiter(&mut self, group: &Self::Group) -> Delimiter {
+        group.delimiter
+    }
+    fn stream(&mut self, group: &Self::Group) -> Self::TokenStream {
+        group.stream.clone()
+    }
+    fn span(&mut self, group: &Self::Group) -> Self::Span {
+        group.span.entire()
+    }
+    fn span_open(&mut self, group: &Self::Group) -> Self::Span {
+        group.span.open
+    }
+    fn span_close(&mut self, group: &Self::Group) -> Self::Span {
+        group.span.close
+    }
+    fn set_span(&mut self, group: &mut Self::Group, span: Self::Span) {
+        group.span = DelimSpan::from_single(span);
+    }
+}
+
+impl server::Punct for Rustc<'_> {
+    fn new(&mut self, ch: char, spacing: Spacing) -> Self::Punct {
+        Punct {
+            ch,
+            joint: spacing == Spacing::Joint,
+            span: server::Span::call_site(self),
+        }
+    }
+    fn as_char(&mut self, punct: Self::Punct) -> char {
+        punct.ch
+    }
+    fn spacing(&mut self, punct: Self::Punct) -> Spacing {
+        if punct.joint {
+            Spacing::Joint
+        } else {
+            Spacing::Alone
+        }
+    }
+    fn span(&mut self, punct: Self::Punct) -> Self::Span {
+        punct.span
+    }
+    fn with_span(&mut self, punct: Self::Punct, span: Self::Span) -> Self::Punct {
+        Punct { span, ..punct }
+    }
+}
+
+impl server::Ident for Rustc<'_> {
+    fn new(&mut self, string: &str, span: Self::Span, is_raw: bool) -> Self::Ident {
+        let sym = Symbol::intern(string);
+        if is_raw
+            && (sym == keywords::Underscore.name()
+                || ast::Ident::with_empty_ctxt(sym).is_path_segment_keyword())
+        {
+            panic!("`{:?}` is not a valid raw identifier", string)
+        }
+        Ident { sym, span, is_raw }
+    }
+    fn span(&mut self, ident: Self::Ident) -> Self::Span {
+        ident.span
+    }
+    fn with_span(&mut self, ident: Self::Ident, span: Self::Span) -> Self::Ident {
+        Ident { span, ..ident }
+    }
+}
+
+impl server::Literal for Rustc<'_> {
+    // FIXME(eddyb) `Literal` should not expose internal `Debug` impls.
+    fn debug(&mut self, literal: &Self::Literal) -> String {
+        format!("{:?}", literal)
+    }
+    fn integer(&mut self, n: &str) -> Self::Literal {
+        Literal {
+            lit: token::Lit::Integer(Symbol::intern(n)),
+            suffix: None,
+            span: server::Span::call_site(self),
+        }
+    }
+    fn typed_integer(&mut self, n: &str, kind: &str) -> Self::Literal {
+        Literal {
+            lit: token::Lit::Integer(Symbol::intern(n)),
+            suffix: Some(Symbol::intern(kind)),
+            span: server::Span::call_site(self),
+        }
+    }
+    fn float(&mut self, n: &str) -> Self::Literal {
+        Literal {
+            lit: token::Lit::Float(Symbol::intern(n)),
+            suffix: None,
+            span: server::Span::call_site(self),
+        }
+    }
+    fn f32(&mut self, n: &str) -> Self::Literal {
+        Literal {
+            lit: token::Lit::Float(Symbol::intern(n)),
+            suffix: Some(Symbol::intern("f32")),
+            span: server::Span::call_site(self),
+        }
+    }
+    fn f64(&mut self, n: &str) -> Self::Literal {
+        Literal {
+            lit: token::Lit::Float(Symbol::intern(n)),
+            suffix: Some(Symbol::intern("f64")),
+            span: server::Span::call_site(self),
+        }
+    }
+    fn string(&mut self, string: &str) -> Self::Literal {
+        let mut escaped = String::new();
+        for ch in string.chars() {
+            escaped.extend(ch.escape_debug());
+        }
+        Literal {
+            lit: token::Lit::Str_(Symbol::intern(&escaped)),
+            suffix: None,
+            span: server::Span::call_site(self),
+        }
+    }
+    fn character(&mut self, ch: char) -> Self::Literal {
+        let mut escaped = String::new();
+        escaped.extend(ch.escape_unicode());
+        Literal {
+            lit: token::Lit::Char(Symbol::intern(&escaped)),
+            suffix: None,
+            span: server::Span::call_site(self),
+        }
+    }
+    fn byte_string(&mut self, bytes: &[u8]) -> Self::Literal {
+        let string = bytes
+            .iter()
+            .cloned()
+            .flat_map(ascii::escape_default)
+            .map(Into::<char>::into)
+            .collect::<String>();
+        Literal {
+            lit: token::Lit::ByteStr(Symbol::intern(&string)),
+            suffix: None,
+            span: server::Span::call_site(self),
+        }
+    }
+    fn span(&mut self, literal: &Self::Literal) -> Self::Span {
+        literal.span
+    }
+    fn set_span(&mut self, literal: &mut Self::Literal, span: Self::Span) {
+        literal.span = span;
+    }
+    fn subspan(
+        &mut self,
+        literal: &Self::Literal,
+        start: Bound<usize>,
+        end: Bound<usize>,
+    ) -> Option<Self::Span> {
+        let span = literal.span;
+        let length = span.hi().to_usize() - span.lo().to_usize();
+
+        let start = match start {
+            Bound::Included(lo) => lo,
+            Bound::Excluded(lo) => lo + 1,
+            Bound::Unbounded => 0,
+        };
+
+        let end = match end {
+            Bound::Included(hi) => hi + 1,
+            Bound::Excluded(hi) => hi,
+            Bound::Unbounded => length,
+        };
+
+        // Bounds check the values, preventing addition overflow and OOB spans.
+        if start > u32::max_value() as usize
+            || end > u32::max_value() as usize
+            || (u32::max_value() - start as u32) < span.lo().to_u32()
+            || (u32::max_value() - end as u32) < span.lo().to_u32()
+            || start >= end
+            || end > length
+        {
+            return None;
+        }
+
+        let new_lo = span.lo() + BytePos::from_usize(start);
+        let new_hi = span.lo() + BytePos::from_usize(end);
+        Some(span.with_lo(new_lo).with_hi(new_hi))
+    }
+}
+
+impl<'a> server::SourceFile for Rustc<'a> {
+    fn eq(&mut self, file1: &Self::SourceFile, file2: &Self::SourceFile) -> bool {
+        Lrc::ptr_eq(file1, file2)
+    }
+    fn path(&mut self, file: &Self::SourceFile) -> String {
+        match file.name {
+            FileName::Real(ref path) => path
+                .to_str()
+                .expect("non-UTF8 file path in `proc_macro::SourceFile::path`")
+                .to_string(),
+            _ => file.name.to_string(),
+        }
+    }
+    fn is_real(&mut self, file: &Self::SourceFile) -> bool {
+        file.is_real_file()
+    }
+}
+
+impl server::MultiSpan for Rustc<'_> {
+    fn new(&mut self) -> Self::MultiSpan {
+        vec![]
+    }
+    fn push(&mut self, spans: &mut Self::MultiSpan, span: Self::Span) {
+        spans.push(span)
+    }
+}
+
+impl server::Diagnostic for Rustc<'_> {
+    fn new(&mut self, level: Level, msg: &str, spans: Self::MultiSpan) -> Self::Diagnostic {
+        let mut diag = Diagnostic::new(level.to_internal(), msg);
+        diag.set_span(MultiSpan::from_spans(spans));
+        diag
+    }
+    fn sub(
+        &mut self,
+        diag: &mut Self::Diagnostic,
+        level: Level,
+        msg: &str,
+        spans: Self::MultiSpan,
+    ) {
+        diag.sub(level.to_internal(), msg, MultiSpan::from_spans(spans), None);
+    }
+    fn emit(&mut self, diag: Self::Diagnostic) {
+        DiagnosticBuilder::new_diagnostic(&self.sess.span_diagnostic, diag).emit()
+    }
+}
+
+impl server::Span for Rustc<'_> {
+    fn debug(&mut self, span: Self::Span) -> String {
+        format!("{:?} bytes({}..{})", span.ctxt(), span.lo().0, span.hi().0)
+    }
+    fn def_site(&mut self) -> Self::Span {
+        self.def_site
+    }
+    fn call_site(&mut self) -> Self::Span {
+        self.call_site
+    }
+    fn source_file(&mut self, span: Self::Span) -> Self::SourceFile {
+        self.sess.source_map().lookup_char_pos(span.lo()).file
+    }
+    fn parent(&mut self, span: Self::Span) -> Option<Self::Span> {
+        span.ctxt().outer().expn_info().map(|i| i.call_site)
+    }
+    fn source(&mut self, span: Self::Span) -> Self::Span {
+        span.source_callsite()
+    }
+    fn start(&mut self, span: Self::Span) -> LineColumn {
+        let loc = self.sess.source_map().lookup_char_pos(span.lo());
+        LineColumn {
+            line: loc.line,
+            column: loc.col.to_usize(),
+        }
+    }
+    fn end(&mut self, span: Self::Span) -> LineColumn {
+        let loc = self.sess.source_map().lookup_char_pos(span.hi());
+        LineColumn {
+            line: loc.line,
+            column: loc.col.to_usize(),
+        }
+    }
+    fn join(&mut self, first: Self::Span, second: Self::Span) -> Option<Self::Span> {
+        let self_loc = self.sess.source_map().lookup_char_pos(first.lo());
+        let other_loc = self.sess.source_map().lookup_char_pos(second.lo());
+
+        if self_loc.file.name != other_loc.file.name {
+            return None;
+        }
+
+        Some(first.to(second))
+    }
+    fn resolved_at(&mut self, span: Self::Span, at: Self::Span) -> Self::Span {
+        span.with_ctxt(at.ctxt())
+    }
+}