Auto merge of #37321 - nrc:lib-proc-macro, r=@alexcrichton

Split up libproc_macro_plugin

Separate the plugin code from non-plugin code to break a potential cycle in crates.

This will allow us to merge the new libproc_macro_tokens into libproc_macro.

r? @alexcrichton

15 files changed, 142 insertions, 87 deletions

diff --git a/mk/crates.mk b/mk/crates.mk
index efe7a579801..c7abf271e51 100644
--- a/mk/crates.mk
+++ b/mk/crates.mk
@@ -60,8 +60,8 @@ RUSTC_CRATES := rustc rustc_typeck rustc_mir rustc_borrowck rustc_resolve rustc_
                 rustc_data_structures rustc_platform_intrinsics rustc_errors \
                 rustc_plugin rustc_metadata rustc_passes rustc_save_analysis \
                 rustc_const_eval rustc_const_math rustc_incremental proc_macro
-HOST_CRATES := syntax syntax_ext proc_macro_plugin syntax_pos $(RUSTC_CRATES) rustdoc fmt_macros \
-		flate arena graphviz log serialize
+HOST_CRATES := syntax syntax_ext proc_macro_tokens proc_macro_plugin syntax_pos $(RUSTC_CRATES) \
+		rustdoc fmt_macros flate arena graphviz log serialize
 TOOLS := compiletest rustdoc rustc rustbook error_index_generator
 
 DEPS_core :=
@@ -102,8 +102,9 @@ DEPS_test := std getopts term native:rust_test_helpers
 
 DEPS_syntax := std term serialize log arena libc rustc_bitflags rustc_unicode rustc_errors syntax_pos
 DEPS_syntax_ext := syntax syntax_pos rustc_errors fmt_macros proc_macro
-DEPS_proc_macro_plugin := syntax syntax_pos rustc_plugin log
 DEPS_syntax_pos := serialize
+DEPS_proc_macro_tokens := syntax syntax_pos log
+DEPS_proc_macro_plugin := syntax syntax_pos rustc_plugin log proc_macro_tokens
 
 DEPS_rustc_const_math := std syntax log serialize
 DEPS_rustc_const_eval := rustc_const_math rustc syntax log serialize \
diff --git a/src/Cargo.lock b/src/Cargo.lock
index 683ba90adf9..64c8ec10191 100644
--- a/src/Cargo.lock
+++ b/src/Cargo.lock
@@ -19,7 +19,7 @@ version = "0.0.0"
 dependencies = [
  "build_helper 0.1.0",
  "core 0.0.0",
- "gcc 0.3.35 (registry+https://github.com/rust-lang/crates.io-index)",
+ "gcc 0.3.38 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.0.0",
 ]
 
@@ -42,10 +42,10 @@ dependencies = [
  "build_helper 0.1.0",
  "cmake 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)",
  "filetime 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
- "gcc 0.3.35 (git+https://github.com/alexcrichton/gcc-rs)",
+ "gcc 0.3.38 (git+https://github.com/alexcrichton/gcc-rs)",
  "getopts 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)",
  "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "libc 0.2.15 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)",
  "md5 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "num_cpus 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -66,7 +66,7 @@ name = "cmake"
 version = "0.1.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "gcc 0.3.35 (registry+https://github.com/rust-lang/crates.io-index)",
+ "gcc 0.3.38 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -83,7 +83,7 @@ name = "compiler_builtins"
 version = "0.0.0"
 dependencies = [
  "core 0.0.0",
- "gcc 0.3.35 (registry+https://github.com/rust-lang/crates.io-index)",
+ "gcc 0.3.38 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -116,7 +116,7 @@ name = "filetime"
 version = "0.1.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "libc 0.2.15 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -124,7 +124,7 @@ name = "flate"
 version = "0.0.0"
 dependencies = [
  "build_helper 0.1.0",
- "gcc 0.3.35 (registry+https://github.com/rust-lang/crates.io-index)",
+ "gcc 0.3.38 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -133,12 +133,12 @@ version = "0.0.0"
 
 [[package]]
 name = "gcc"
-version = "0.3.35"
-source = "git+https://github.com/alexcrichton/gcc-rs#03e22a4425c011fa8c96681591432456fa70d60c"
+version = "0.3.38"
+source = "git+https://github.com/alexcrichton/gcc-rs#be620ac6d3ddb498cd0c700d5312c6a4c3c19597"
 
 [[package]]
 name = "gcc"
-version = "0.3.35"
+version = "0.3.38"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
@@ -159,7 +159,7 @@ name = "idna"
 version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "matches 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "matches 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "unicode-bidi 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "unicode-normalization 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
@@ -182,14 +182,14 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.15"
+version = "0.2.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
 name = "linkchecker"
 version = "0.1.0"
 dependencies = [
- "url 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "url 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -203,7 +203,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
 name = "matches"
-version = "0.1.2"
+version = "0.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
@@ -216,7 +216,7 @@ name = "num_cpus"
 version = "0.2.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "libc 0.2.15 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -249,12 +249,22 @@ name = "proc_macro_plugin"
 version = "0.0.0"
 dependencies = [
  "log 0.0.0",
+ "proc_macro_tokens 0.0.0",
  "rustc_plugin 0.0.0",
  "syntax 0.0.0",
  "syntax_pos 0.0.0",
 ]
 
 [[package]]
+name = "proc_macro_tokens"
+version = "0.0.0"
+dependencies = [
+ "log 0.0.0",
+ "syntax 0.0.0",
+ "syntax_pos 0.0.0",
+]
+
+[[package]]
 name = "rand"
 version = "0.0.0"
 dependencies = [
@@ -431,7 +441,7 @@ name = "rustc_llvm"
 version = "0.0.0"
 dependencies = [
  "build_helper 0.1.0",
- "gcc 0.3.35 (registry+https://github.com/rust-lang/crates.io-index)",
+ "gcc 0.3.38 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustc_bitflags 0.0.0",
 ]
 
@@ -585,7 +595,7 @@ version = "0.0.0"
 dependencies = [
  "arena 0.0.0",
  "build_helper 0.1.0",
- "gcc 0.3.35 (registry+https://github.com/rust-lang/crates.io-index)",
+ "gcc 0.3.38 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.0.0",
  "rustc 0.0.0",
  "rustc_back 0.0.0",
@@ -621,7 +631,7 @@ dependencies = [
  "collections 0.0.0",
  "compiler_builtins 0.0.0",
  "core 0.0.0",
- "gcc 0.3.35 (registry+https://github.com/rust-lang/crates.io-index)",
+ "gcc 0.3.38 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.0.0",
  "panic_abort 0.0.0",
  "panic_unwind 0.0.0",
@@ -704,7 +714,7 @@ name = "unicode-bidi"
 version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "matches 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "matches 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -714,11 +724,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
 name = "url"
-version = "1.2.0"
+version = "1.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "idna 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "matches 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "matches 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -735,20 +745,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum cmake 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)" = "dfcf5bcece56ef953b8ea042509e9dcbdfe97820b7e20d86beb53df30ed94978"
 "checksum env_logger 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "15abd780e45b3ea4f76b4e9a26ff4843258dd8a3eed2775a0e7368c2e7936c2f"
 "checksum filetime 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "5363ab8e4139b8568a6237db5248646e5a8a2f89bd5ccb02092182b11fd3e922"
-"checksum gcc 0.3.35 (git+https://github.com/alexcrichton/gcc-rs)" = "<none>"
-"checksum gcc 0.3.35 (registry+https://github.com/rust-lang/crates.io-index)" = "91ecd03771effb0c968fd6950b37e89476a578aaf1c70297d8e92b6516ec3312"
+"checksum gcc 0.3.38 (git+https://github.com/alexcrichton/gcc-rs)" = "<none>"
+"checksum gcc 0.3.38 (registry+https://github.com/rust-lang/crates.io-index)" = "553f11439bdefe755bf366b264820f1da70f3aaf3924e594b886beb9c831bcf5"
 "checksum getopts 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)" = "d9047cfbd08a437050b363d35ef160452c5fe8ea5187ae0a624708c91581d685"
 "checksum idna 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1053236e00ce4f668aeca4a769a09b3bf5a682d802abd6f3cb39374f6b162c11"
 "checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d"
-"checksum libc 0.2.15 (registry+https://github.com/rust-lang/crates.io-index)" = "23e3757828fa702a20072c37ff47938e9dd331b92fac6e223d26d4b7a55f7ee2"
+"checksum libc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)" = "044d1360593a78f5c8e5e710beccdc24ab71d1f01bc19a29bcacdba22e8475d8"
 "checksum log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "ab83497bf8bf4ed2a74259c1c802351fcd67a65baa86394b6ba73c36f4838054"
-"checksum matches 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "15305656809ce5a4805b1ff2946892810992197ce1270ff79baded852187942e"
+"checksum matches 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "bcc3ad8109fa4b522f9b0cd81440422781f564aaf8c195de6b9d6642177ad0dd"
 "checksum md5 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a5539a8dee9b4ae308c9c406a379838b435a8f2c84cf9fedc6d5a576be9888db"
 "checksum num_cpus 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)" = "cee7e88156f3f9e19bdd598f8d6c9db7bf4078f99f8381f43a55b09648d1a6e3"
 "checksum rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)" = "6159e4e6e559c81bd706afe9c8fd68f547d3e851ce12e76b1de7914bab61691b"
 "checksum toml 0.1.30 (registry+https://github.com/rust-lang/crates.io-index)" = "0590d72182e50e879c4da3b11c6488dae18fccb1ae0c7a3eda18e16795844796"
 "checksum unicode-bidi 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c1f7ceb96afdfeedee42bade65a0d585a6a0106f681b6749c8ff4daa8df30b3f"
 "checksum unicode-normalization 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "26643a2f83bac55f1976fb716c10234485f9202dcd65cfbdf9da49867b271172"
-"checksum url 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "afe9ec54bc4db14bc8744b7fed060d785ac756791450959b2248443319d5b119"
+"checksum url 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9ba5a45db1d2e0effb7a1c00cc73ffc63a973da8c7d1fcd5b46f24285ade6c54"
 "checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a"
 "checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc"
diff --git a/src/libproc_macro_plugin/Cargo.toml b/src/libproc_macro_plugin/Cargo.toml
index 70bb86d0f58..4bc3f488d32 100644
--- a/src/libproc_macro_plugin/Cargo.toml
+++ b/src/libproc_macro_plugin/Cargo.toml
@@ -12,3 +12,4 @@ log = { path = "../liblog" }
 rustc_plugin = { path = "../librustc_plugin" }
 syntax = { path = "../libsyntax" }
 syntax_pos = { path = "../libsyntax_pos" }
+proc_macro_tokens = { path = "../libproc_macro_tokens" }
diff --git a/src/libproc_macro_plugin/lib.rs b/src/libproc_macro_plugin/lib.rs
index e82e97b5134..c45762bfb6e 100644
--- a/src/libproc_macro_plugin/lib.rs
+++ b/src/libproc_macro_plugin/lib.rs
@@ -13,43 +13,14 @@
 //! A library for procedural macro writers.
 //!
 //! ## Usage
-//! This package provides the `qquote!` macro for syntax creation, and the prelude
-//! (at libproc_macro::prelude) provides a number of operations:
-//! - `concat`, for concatenating two TokenStreams.
-//! - `ident_eq`, for checking if two identifiers are equal regardless of syntax context.
-//! - `str_to_token_ident`, for converting an `&str` into a Token.
-//! - `keyword_to_token_delim`, for converting a `parse::token::keywords::Keyword` into a
-//!    Token.
-//! - `build_delimited`, for creating a new TokenStream from an existing one and a delimiter
-//!    by wrapping the TokenStream in the delimiter.
-//! - `build_bracket_delimited`, `build_brace_delimited`, and `build_paren_delimited`, for
-//!    easing the above.
-//! - `build_empty_args`, which returns a TokenStream containing `()`.
-//! - `lex`, which takes an `&str` and returns the TokenStream it represents.
-//!
-//! The `qquote!` macro also imports `syntax::ext::proc_macro_shim::prelude::*`, so you
+//! This crate provides the `qquote!` macro for syntax creation.
+//!
+//! The `qquote!` macro imports `syntax::ext::proc_macro_shim::prelude::*`, so you
 //! will need to `extern crate syntax` for usage. (This is a temporary solution until more
-//! of the external API in libproc_macro is stabilized to support the token construction
+//! of the external API in libproc_macro_tokens is stabilized to support the token construction
 //! operations that the qausiquoter relies on.) The shim file also provides additional
 //! operations, such as `build_block_emitter` (as used in the `cond` example below).
 //!
-//! ## TokenStreams
-//!
-//! TokenStreams serve as the basis of the macro system. They are, in essence, vectors of
-//! TokenTrees, where indexing treats delimited values as a single term. That is, the term
-//! `even(a+c) && even(b)` will be indexibly encoded as `even | (a+c) | even | (b)` where,
-//! in reality, `(a+c)` is actually a decorated pointer to `a | + | c`.
-//!
-//! If a user has a TokenStream that is a single, delimited value, they can use
-//! `maybe_delimited` to destruct it and receive the internal vector as a new TokenStream
-//! as:
-//! ```
-//! `(a+c)`.maybe_delimited() ~> Some(a | + | c)`
-//! ```
-//!
-//! Check the TokenStream documentation for more information; the structure also provides
-//! cheap concatenation and slicing.
-//!
 //! ## Quasiquotation
 //!
 //! The quasiquoter creates output that, when run, constructs the tokenstream specified as
@@ -118,12 +89,11 @@
 extern crate rustc_plugin;
 extern crate syntax;
 extern crate syntax_pos;
+extern crate proc_macro_tokens;
 #[macro_use] extern crate log;
 
 mod qquote;
-pub mod build;
-pub mod parse;
-pub mod prelude;
+
 use qquote::qquote;
 
 use rustc_plugin::Registry;
diff --git a/src/libproc_macro_plugin/qquote.rs b/src/libproc_macro_plugin/qquote.rs
index b73d085656e..e5a3abc2ea9 100644
--- a/src/libproc_macro_plugin/qquote.rs
+++ b/src/libproc_macro_plugin/qquote.rs
@@ -24,12 +24,9 @@
 //! TokenStream that resembles the output syntax.
 //!
 
-extern crate rustc_plugin;
-extern crate syntax;
-extern crate syntax_pos;
+use proc_macro_tokens::build::*;
+use proc_macro_tokens::parse::lex;
 
-use build::*;
-use parse::lex;
 use qquote::int_build::*;
 
 use syntax::ast::Ident;
@@ -51,7 +48,7 @@ pub fn qquote<'cx>(cx: &'cx mut ExtCtxt, sp: Span, tts: &[TokenTree])
     let output = qquoter(cx, TokenStream::from_tts(tts.clone().to_owned()));
     debug!("\nQQ out: {}\n", pprust::tts_to_string(&output.to_tts()[..]));
     let imports = concat(lex("use syntax::ext::proc_macro_shim::prelude::*;"),
-                         lex("use proc_macro_plugin::prelude::*;"));
+                         lex("use proc_macro_tokens::prelude::*;"));
     build_block_emitter(cx, sp, build_brace_delimited(concat(imports, output)))
 }
 
@@ -219,7 +216,7 @@ fn convert_complex_tts<'cx>(cx: &'cx mut ExtCtxt, tts: Vec<QTT>) -> (Bindings, T
 
                 let sep = build_delim_tok(qdl.delim);
 
-                pushes.push(build_mod_call(vec![str_to_ident("proc_macro_plugin"),
+                pushes.push(build_mod_call(vec![str_to_ident("proc_macro_tokens"),
                                                str_to_ident("build"),
                                                str_to_ident("build_delimited")],
                                           concat(from_tokens(vec![Token::Ident(new_id)]),
@@ -264,11 +261,8 @@ fn is_qquote(id: Ident) -> bool {
 }
 
 mod int_build {
-    extern crate syntax;
-    extern crate syntax_pos;
-
-    use parse::*;
-    use build::*;
+    use proc_macro_tokens::build::*;
+    use proc_macro_tokens::parse::*;
 
     use syntax::ast::{self, Ident};
     use syntax::codemap::{DUMMY_SP};
diff --git a/src/libproc_macro_tokens/Cargo.toml b/src/libproc_macro_tokens/Cargo.toml
new file mode 100644
index 00000000000..2b66d56759f
--- /dev/null
+++ b/src/libproc_macro_tokens/Cargo.toml
@@ -0,0 +1,13 @@
+[package]
+authors = ["The Rust Project Developers"]
+name = "proc_macro_tokens"
+version = "0.0.0"
+
+[lib]
+path = "lib.rs"
+crate-type = ["dylib"]
+
+[dependencies]
+syntax = { path = "../libsyntax" }
+syntax_pos = { path = "../libsyntax_pos" }
+log = { path = "../liblog" }
diff --git a/src/libproc_macro_plugin/build.rs b/src/libproc_macro_tokens/build.rs
index 7b7590b863b..7b7590b863b 100644
--- a/src/libproc_macro_plugin/build.rs
+++ b/src/libproc_macro_tokens/build.rs
diff --git a/src/libproc_macro_tokens/lib.rs b/src/libproc_macro_tokens/lib.rs
new file mode 100644
index 00000000000..3bfa2fbb29f
--- /dev/null
+++ b/src/libproc_macro_tokens/lib.rs
@@ -0,0 +1,66 @@
+// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+//! # Proc_Macro
+//!
+//! A library for procedural macro writers.
+//!
+//! ## Usage
+//! This crate provides the prelude (at libproc_macro_tokens::prelude), which
+//! provides a number of operations:
+//! - `concat`, for concatenating two TokenStreams.
+//! - `ident_eq`, for checking if two identifiers are equal regardless of syntax context.
+//! - `str_to_token_ident`, for converting an `&str` into a Token.
+//! - `keyword_to_token_delim`, for converting a `parse::token::keywords::Keyword` into a
+//!    Token.
+//! - `build_delimited`, for creating a new TokenStream from an existing one and a delimiter
+//!    by wrapping the TokenStream in the delimiter.
+//! - `build_bracket_delimited`, `build_brace_delimited`, and `build_paren_delimited`, for
+//!    easing the above.
+//! - `build_empty_args`, which returns a TokenStream containing `()`.
+//! - `lex`, which takes an `&str` and returns the TokenStream it represents.
+//!
+//! ## TokenStreams
+//!
+//! TokenStreams serve as the basis of the macro system. They are, in essence, vectors of
+//! TokenTrees, where indexing treats delimited values as a single term. That is, the term
+//! `even(a+c) && even(b)` will be indexibly encoded as `even | (a+c) | even | (b)` where,
+//! in reality, `(a+c)` is actually a decorated pointer to `a | + | c`.
+//!
+//! If a user has a TokenStream that is a single, delimited value, they can use
+//! `maybe_delimited` to destruct it and receive the internal vector as a new TokenStream
+//! as:
+//! ```
+//! `(a+c)`.maybe_delimited() ~> Some(a | + | c)`
+//! ```
+//!
+//! Check the TokenStream documentation for more information; the structure also provides
+//! cheap concatenation and slicing.
+//!
+
+#![crate_name = "proc_macro_tokens"]
+#![unstable(feature = "rustc_private", issue = "27812")]
+#![crate_type = "dylib"]
+#![crate_type = "rlib"]
+#![doc(html_logo_url = "https://www.rust-lang.org/logos/rust-logo-128x128-blk-v2.png",
+       html_favicon_url = "https://doc.rust-lang.org/favicon.ico",
+       html_root_url = "https://doc.rust-lang.org/nightly/")]
+#![cfg_attr(not(stage0), deny(warnings))]
+
+#![feature(staged_api)]
+#![feature(rustc_private)]
+
+extern crate syntax;
+extern crate syntax_pos;
+#[macro_use] extern crate log;
+
+pub mod build;
+pub mod parse;
+pub mod prelude;
diff --git a/src/libproc_macro_plugin/parse.rs b/src/libproc_macro_tokens/parse.rs
index 9af8a68cdcf..9af8a68cdcf 100644
--- a/src/libproc_macro_plugin/parse.rs
+++ b/src/libproc_macro_tokens/parse.rs
diff --git a/src/libproc_macro_plugin/prelude.rs b/src/libproc_macro_tokens/prelude.rs
index 4c0c8ba6c66..4c0c8ba6c66 100644
--- a/src/libproc_macro_plugin/prelude.rs
+++ b/src/libproc_macro_tokens/prelude.rs
diff --git a/src/test/run-pass-fulldeps/auxiliary/cond_noprelude_plugin.rs b/src/test/run-pass-fulldeps/auxiliary/cond_noprelude_plugin.rs
index 78589546084..48919fe876a 100644
--- a/src/test/run-pass-fulldeps/auxiliary/cond_noprelude_plugin.rs
+++ b/src/test/run-pass-fulldeps/auxiliary/cond_noprelude_plugin.rs
@@ -15,10 +15,10 @@
 #![plugin(proc_macro_plugin)]
 
 extern crate rustc_plugin;
-extern crate proc_macro_plugin;
+extern crate proc_macro_tokens;
 extern crate syntax;
 
-use proc_macro_plugin::build::ident_eq;
+use proc_macro_tokens::build::ident_eq;
 
 use syntax::ext::base::{ExtCtxt, MacResult};
 use syntax::ext::proc_macro_shim::build_block_emitter;
diff --git a/src/test/run-pass-fulldeps/auxiliary/cond_plugin.rs b/src/test/run-pass-fulldeps/auxiliary/cond_plugin.rs
index 11322bf76ff..0ea4cec75cd 100644
--- a/src/test/run-pass-fulldeps/auxiliary/cond_plugin.rs
+++ b/src/test/run-pass-fulldeps/auxiliary/cond_plugin.rs
@@ -15,10 +15,10 @@
 #![plugin(proc_macro_plugin)]
 
 extern crate rustc_plugin;
-extern crate proc_macro_plugin;
+extern crate proc_macro_tokens;
 extern crate syntax;
 
-use proc_macro_plugin::prelude::*;
+use proc_macro_tokens::prelude::*;
 
 use rustc_plugin::Registry;
 
diff --git a/src/test/run-pass-fulldeps/auxiliary/cond_prelude_plugin.rs b/src/test/run-pass-fulldeps/auxiliary/cond_prelude_plugin.rs
index 232a7166e3b..169c96b4385 100644
--- a/src/test/run-pass-fulldeps/auxiliary/cond_prelude_plugin.rs
+++ b/src/test/run-pass-fulldeps/auxiliary/cond_prelude_plugin.rs
@@ -15,11 +15,11 @@
 #![plugin(proc_macro_plugin)]
 
 extern crate rustc_plugin;
-extern crate proc_macro_plugin;
+extern crate proc_macro_tokens;
 extern crate syntax;
 
 use syntax::ext::proc_macro_shim::prelude::*;
-use proc_macro_plugin::prelude::*;
+use proc_macro_tokens::prelude::*;
 
 use rustc_plugin::Registry;
 
diff --git a/src/test/run-pass-fulldeps/auxiliary/proc_macro_def.rs b/src/test/run-pass-fulldeps/auxiliary/proc_macro_def.rs
index 961df6d5c0c..9fce19f46f6 100644
--- a/src/test/run-pass-fulldeps/auxiliary/proc_macro_def.rs
+++ b/src/test/run-pass-fulldeps/auxiliary/proc_macro_def.rs
@@ -10,11 +10,11 @@
 
 #![feature(plugin, plugin_registrar, rustc_private)]
 
-extern crate proc_macro_plugin;
+extern crate proc_macro_tokens;
 extern crate rustc_plugin;
 extern crate syntax;
 
-use proc_macro_plugin::prelude::*;
+use proc_macro_tokens::prelude::*;
 use rustc_plugin::Registry;
 use syntax::ext::base::SyntaxExtension;
 use syntax::ext::proc_macro_shim::prelude::*;
diff --git a/src/test/run-pass-fulldeps/macro-quote-1.rs b/src/test/run-pass-fulldeps/macro-quote-1.rs
index a5ac546cba4..914da3f7467 100644
--- a/src/test/run-pass-fulldeps/macro-quote-1.rs
+++ b/src/test/run-pass-fulldeps/macro-quote-1.rs
@@ -14,8 +14,8 @@
 #![feature(rustc_private)]
 #![plugin(proc_macro_plugin)]
 
-extern crate proc_macro_plugin;
-use proc_macro_plugin::prelude::*;
+extern crate proc_macro_tokens;
+use proc_macro_tokens::prelude::*;
 
 extern crate syntax;
 use syntax::ast::Ident;