Merge remote-tracking branch 'rust-lang/master' into hermit

18 files changed, 2536 insertions, 98 deletions

diff --git a/src/libsyntax_expand/Cargo.toml b/src/libsyntax_expand/Cargo.toml
index f063753f599..653b87f0d82 100644
--- a/src/libsyntax_expand/Cargo.toml
+++ b/src/libsyntax_expand/Cargo.toml
@@ -11,16 +11,12 @@ path = "lib.rs"
 doctest = false
 
 [dependencies]
-bitflags = "1.0"
 rustc_serialize = { path = "../libserialize", package = "serialize" }
 log = "0.4"
-scoped-tls = "1.0"
-lazy_static = "1.0.0"
 syntax_pos = { path = "../libsyntax_pos" }
 errors = { path = "../librustc_errors", package = "rustc_errors" }
 rustc_data_structures = { path = "../librustc_data_structures" }
-rustc_index = { path = "../librustc_index" }
 rustc_lexer = { path = "../librustc_lexer" }
-rustc_target = { path = "../librustc_target" }
-smallvec = { version = "0.6.7", features = ["union", "may_dangle"] }
+rustc_parse = { path = "../librustc_parse" }
+smallvec = { version = "1.0", features = ["union", "may_dangle"] }
 syntax = { path = "../libsyntax" }
diff --git a/src/libsyntax_expand/base.rs b/src/libsyntax_expand/base.rs
index d79b6910587..97ee241d838 100644
--- a/src/libsyntax_expand/base.rs
+++ b/src/libsyntax_expand/base.rs
@@ -1,27 +1,26 @@
 use crate::expand::{self, AstFragment, Invocation};
 
+use rustc_parse::{self, parser, DirectoryOwnership, MACRO_ARGUMENTS};
 use syntax::ast::{self, NodeId, Attribute, Name, PatKind};
 use syntax::attr::{self, HasAttrs, Stability, Deprecation};
 use syntax::source_map::SourceMap;
 use syntax::edition::Edition;
 use syntax::mut_visit::{self, MutVisitor};
-use syntax::parse::{self, parser, DirectoryOwnership};
-use syntax::parse::token;
 use syntax::ptr::P;
 use syntax::sess::ParseSess;
 use syntax::symbol::{kw, sym, Ident, Symbol};
-use syntax::{ThinVec, MACRO_ARGUMENTS};
+use syntax::ThinVec;
+use syntax::token;
 use syntax::tokenstream::{self, TokenStream};
 use syntax::visit::Visitor;
-crate use syntax::expand::SpecialDerives;
 
 use errors::{DiagnosticBuilder, DiagnosticId};
 use smallvec::{smallvec, SmallVec};
 use syntax_pos::{FileName, Span, MultiSpan, DUMMY_SP};
 use syntax_pos::hygiene::{AstPass, ExpnId, ExpnData, ExpnKind};
-
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::sync::{self, Lrc};
+
 use std::iter;
 use std::path::PathBuf;
 use std::rc::Rc;
@@ -860,8 +859,8 @@ pub trait Resolver {
 
     fn check_unused_macros(&mut self);
 
-    fn has_derives(&self, expn_id: ExpnId, derives: SpecialDerives) -> bool;
-    fn add_derives(&mut self, expn_id: ExpnId, derives: SpecialDerives);
+    fn has_derive_copy(&self, expn_id: ExpnId) -> bool;
+    fn add_derive_copy(&mut self, expn_id: ExpnId);
 }
 
 #[derive(Clone)]
@@ -923,7 +922,7 @@ impl<'a> ExtCtxt<'a> {
         expand::MacroExpander::new(self, true)
     }
     pub fn new_parser_from_tts(&self, stream: TokenStream) -> parser::Parser<'a> {
-        parse::stream_to_parser(self.parse_sess, stream, MACRO_ARGUMENTS)
+        rustc_parse::stream_to_parser(self.parse_sess, stream, MACRO_ARGUMENTS)
     }
     pub fn source_map(&self) -> &'a SourceMap { self.parse_sess.source_map() }
     pub fn parse_sess(&self) -> &'a ParseSess { self.parse_sess }
@@ -954,18 +953,7 @@ impl<'a> ExtCtxt<'a> {
     ///
     /// Stops backtracing at include! boundary.
     pub fn expansion_cause(&self) -> Option<Span> {
-        let mut expn_id = self.current_expansion.id;
-        let mut last_macro = None;
-        loop {
-            let expn_data = expn_id.expn_data();
-            // Stop going up the backtrace once include! is encountered
-            if expn_data.is_root() || expn_data.kind.descr() == sym::include {
-                break;
-            }
-            expn_id = expn_data.call_site.ctxt().outer_expn();
-            last_macro = Some(expn_data.call_site);
-        }
-        last_macro
+        self.current_expansion.id.expansion_cause()
     }
 
     pub fn struct_span_warn<S: Into<MultiSpan>>(&self,
diff --git a/src/libsyntax_expand/config.rs b/src/libsyntax_expand/config.rs
new file mode 100644
index 00000000000..7b927fb55f9
--- /dev/null
+++ b/src/libsyntax_expand/config.rs
@@ -0,0 +1,365 @@
+use rustc_parse::validate_attr;
+use syntax::attr::HasAttrs;
+use syntax::feature_gate::{
+    feature_err,
+    EXPLAIN_STMT_ATTR_SYNTAX,
+    Features,
+    get_features,
+    GateIssue,
+};
+use syntax::attr;
+use syntax::ast;
+use syntax::edition::Edition;
+use syntax::mut_visit::*;
+use syntax::ptr::P;
+use syntax::sess::ParseSess;
+use syntax::util::map_in_place::MapInPlace;
+use syntax_pos::symbol::sym;
+
+use errors::Applicability;
+use smallvec::SmallVec;
+
+/// A folder that strips out items that do not belong in the current configuration.
+pub struct StripUnconfigured<'a> {
+    pub sess: &'a ParseSess,
+    pub features: Option<&'a Features>,
+}
+
+// `cfg_attr`-process the crate's attributes and compute the crate's features.
+pub fn features(mut krate: ast::Crate, sess: &ParseSess, edition: Edition,
+                allow_features: &Option<Vec<String>>) -> (ast::Crate, Features) {
+    let features;
+    {
+        let mut strip_unconfigured = StripUnconfigured {
+            sess,
+            features: None,
+        };
+
+        let unconfigured_attrs = krate.attrs.clone();
+        let err_count = sess.span_diagnostic.err_count();
+        if let Some(attrs) = strip_unconfigured.configure(krate.attrs) {
+            krate.attrs = attrs;
+        } else { // the entire crate is unconfigured
+            krate.attrs = Vec::new();
+            krate.module.items = Vec::new();
+            return (krate, Features::new());
+        }
+
+        features = get_features(&sess.span_diagnostic, &krate.attrs, edition, allow_features);
+
+        // Avoid reconfiguring malformed `cfg_attr`s
+        if err_count == sess.span_diagnostic.err_count() {
+            strip_unconfigured.features = Some(&features);
+            strip_unconfigured.configure(unconfigured_attrs);
+        }
+    }
+
+    (krate, features)
+}
+
+#[macro_export]
+macro_rules! configure {
+    ($this:ident, $node:ident) => {
+        match $this.configure($node) {
+            Some(node) => node,
+            None => return Default::default(),
+        }
+    }
+}
+
+impl<'a> StripUnconfigured<'a> {
+    pub fn configure<T: HasAttrs>(&mut self, mut node: T) -> Option<T> {
+        self.process_cfg_attrs(&mut node);
+        if self.in_cfg(node.attrs()) { Some(node) } else { None }
+    }
+
+    /// Parse and expand all `cfg_attr` attributes into a list of attributes
+    /// that are within each `cfg_attr` that has a true configuration predicate.
+    ///
+    /// Gives compiler warnigns if any `cfg_attr` does not contain any
+    /// attributes and is in the original source code. Gives compiler errors if
+    /// the syntax of any `cfg_attr` is incorrect.
+    pub fn process_cfg_attrs<T: HasAttrs>(&mut self, node: &mut T) {
+        node.visit_attrs(|attrs| {
+            attrs.flat_map_in_place(|attr| self.process_cfg_attr(attr));
+        });
+    }
+
+    /// Parse and expand a single `cfg_attr` attribute into a list of attributes
+    /// when the configuration predicate is true, or otherwise expand into an
+    /// empty list of attributes.
+    ///
+    /// Gives a compiler warning when the `cfg_attr` contains no attributes and
+    /// is in the original source file. Gives a compiler error if the syntax of
+    /// the attribute is incorrect.
+    fn process_cfg_attr(&mut self, attr: ast::Attribute) -> Vec<ast::Attribute> {
+        if !attr.has_name(sym::cfg_attr) {
+            return vec![attr];
+        }
+        if attr.get_normal_item().tokens.is_empty() {
+            self.sess.span_diagnostic
+                .struct_span_err(
+                    attr.span,
+                    "malformed `cfg_attr` attribute input",
+                ).span_suggestion(
+                    attr.span,
+                    "missing condition and attribute",
+                    "#[cfg_attr(condition, attribute, other_attribute, ...)]".to_owned(),
+                    Applicability::HasPlaceholders,
+                ).note("for more information, visit \
+                       <https://doc.rust-lang.org/reference/conditional-compilation.html\
+                       #the-cfg_attr-attribute>")
+                .emit();
+            return vec![];
+        }
+
+        let res = rustc_parse::parse_in_attr(self.sess, &attr, |p| p.parse_cfg_attr());
+        let (cfg_predicate, expanded_attrs) = match res {
+            Ok(result) => result,
+            Err(mut e) => {
+                e.emit();
+                return vec![];
+            }
+        };
+
+        // Lint on zero attributes in source.
+        if expanded_attrs.is_empty() {
+            return vec![attr];
+        }
+
+        // At this point we know the attribute is considered used.
+        attr::mark_used(&attr);
+
+        if attr::cfg_matches(&cfg_predicate, self.sess, self.features) {
+            // We call `process_cfg_attr` recursively in case there's a
+            // `cfg_attr` inside of another `cfg_attr`. E.g.
+            //  `#[cfg_attr(false, cfg_attr(true, some_attr))]`.
+            expanded_attrs.into_iter()
+            .flat_map(|(item, span)| self.process_cfg_attr(attr::mk_attr_from_item(
+                attr.style,
+                item,
+                span,
+            )))
+            .collect()
+        } else {
+            vec![]
+        }
+    }
+
+    /// Determines if a node with the given attributes should be included in this configuration.
+    pub fn in_cfg(&self, attrs: &[ast::Attribute]) -> bool {
+        attrs.iter().all(|attr| {
+            if !is_cfg(attr) {
+                return true;
+            }
+
+            let error = |span, msg, suggestion: &str| {
+                let mut err = self.sess.span_diagnostic.struct_span_err(span, msg);
+                if !suggestion.is_empty() {
+                    err.span_suggestion(
+                        span,
+                        "expected syntax is",
+                        suggestion.into(),
+                        Applicability::MaybeIncorrect,
+                    );
+                }
+                err.emit();
+                true
+            };
+
+            let meta_item = match validate_attr::parse_meta(self.sess, attr) {
+                Ok(meta_item) => meta_item,
+                Err(mut err) => { err.emit(); return true; }
+            };
+            let nested_meta_items = if let Some(nested_meta_items) = meta_item.meta_item_list() {
+                nested_meta_items
+            } else {
+                return error(meta_item.span, "`cfg` is not followed by parentheses",
+                                             "cfg(/* predicate */)");
+            };
+
+            if nested_meta_items.is_empty() {
+                return error(meta_item.span, "`cfg` predicate is not specified", "");
+            } else if nested_meta_items.len() > 1 {
+                return error(nested_meta_items.last().unwrap().span(),
+                             "multiple `cfg` predicates are specified", "");
+            }
+
+            match nested_meta_items[0].meta_item() {
+                Some(meta_item) => attr::cfg_matches(meta_item, self.sess, self.features),
+                None => error(nested_meta_items[0].span(),
+                              "`cfg` predicate key cannot be a literal", ""),
+            }
+        })
+    }
+
+    /// Visit attributes on expression and statements (but not attributes on items in blocks).
+    fn visit_expr_attrs(&mut self, attrs: &[ast::Attribute]) {
+        // flag the offending attributes
+        for attr in attrs.iter() {
+            self.maybe_emit_expr_attr_err(attr);
+        }
+    }
+
+    /// If attributes are not allowed on expressions, emit an error for `attr`
+    pub fn maybe_emit_expr_attr_err(&self, attr: &ast::Attribute) {
+        if !self.features.map(|features| features.stmt_expr_attributes).unwrap_or(true) {
+            let mut err = feature_err(self.sess,
+                                      sym::stmt_expr_attributes,
+                                      attr.span,
+                                      GateIssue::Language,
+                                      EXPLAIN_STMT_ATTR_SYNTAX);
+
+            if attr.is_doc_comment() {
+                err.help("`///` is for documentation comments. For a plain comment, use `//`.");
+            }
+
+            err.emit();
+        }
+    }
+
+    pub fn configure_foreign_mod(&mut self, foreign_mod: &mut ast::ForeignMod) {
+        let ast::ForeignMod { abi: _, items } = foreign_mod;
+        items.flat_map_in_place(|item| self.configure(item));
+    }
+
+    pub fn configure_generic_params(&mut self, params: &mut Vec<ast::GenericParam>) {
+        params.flat_map_in_place(|param| self.configure(param));
+    }
+
+    fn configure_variant_data(&mut self, vdata: &mut ast::VariantData) {
+        match vdata {
+            ast::VariantData::Struct(fields, ..) | ast::VariantData::Tuple(fields, _) =>
+                fields.flat_map_in_place(|field| self.configure(field)),
+            ast::VariantData::Unit(_) => {}
+        }
+    }
+
+    pub fn configure_item_kind(&mut self, item: &mut ast::ItemKind) {
+        match item {
+            ast::ItemKind::Struct(def, _generics) |
+            ast::ItemKind::Union(def, _generics) => self.configure_variant_data(def),
+            ast::ItemKind::Enum(ast::EnumDef { variants }, _generics) => {
+                variants.flat_map_in_place(|variant| self.configure(variant));
+                for variant in variants {
+                    self.configure_variant_data(&mut variant.data);
+                }
+            }
+            _ => {}
+        }
+    }
+
+    pub fn configure_expr_kind(&mut self, expr_kind: &mut ast::ExprKind) {
+        match expr_kind {
+            ast::ExprKind::Match(_m, arms) => {
+                arms.flat_map_in_place(|arm| self.configure(arm));
+            }
+            ast::ExprKind::Struct(_path, fields, _base) => {
+                fields.flat_map_in_place(|field| self.configure(field));
+            }
+            _ => {}
+        }
+    }
+
+    pub fn configure_expr(&mut self, expr: &mut P<ast::Expr>) {
+        self.visit_expr_attrs(expr.attrs());
+
+        // If an expr is valid to cfg away it will have been removed by the
+        // outer stmt or expression folder before descending in here.
+        // Anything else is always required, and thus has to error out
+        // in case of a cfg attr.
+        //
+        // N.B., this is intentionally not part of the visit_expr() function
+        //     in order for filter_map_expr() to be able to avoid this check
+        if let Some(attr) = expr.attrs().iter().find(|a| is_cfg(a)) {
+            let msg = "removing an expression is not supported in this position";
+            self.sess.span_diagnostic.span_err(attr.span, msg);
+        }
+
+        self.process_cfg_attrs(expr)
+    }
+
+    pub fn configure_pat(&mut self, pat: &mut P<ast::Pat>) {
+        if let ast::PatKind::Struct(_path, fields, _etc) = &mut pat.kind {
+            fields.flat_map_in_place(|field| self.configure(field));
+        }
+    }
+
+    pub fn configure_fn_decl(&mut self, fn_decl: &mut ast::FnDecl) {
+        fn_decl.inputs.flat_map_in_place(|arg| self.configure(arg));
+    }
+}
+
+impl<'a> MutVisitor for StripUnconfigured<'a> {
+    fn visit_foreign_mod(&mut self, foreign_mod: &mut ast::ForeignMod) {
+        self.configure_foreign_mod(foreign_mod);
+        noop_visit_foreign_mod(foreign_mod, self);
+    }
+
+    fn visit_item_kind(&mut self, item: &mut ast::ItemKind) {
+        self.configure_item_kind(item);
+        noop_visit_item_kind(item, self);
+    }
+
+    fn visit_expr(&mut self, expr: &mut P<ast::Expr>) {
+        self.configure_expr(expr);
+        self.configure_expr_kind(&mut expr.kind);
+        noop_visit_expr(expr, self);
+    }
+
+    fn filter_map_expr(&mut self, expr: P<ast::Expr>) -> Option<P<ast::Expr>> {
+        let mut expr = configure!(self, expr);
+        self.configure_expr_kind(&mut expr.kind);
+        noop_visit_expr(&mut expr, self);
+        Some(expr)
+    }
+
+    fn flat_map_stmt(&mut self, stmt: ast::Stmt) -> SmallVec<[ast::Stmt; 1]> {
+        noop_flat_map_stmt(configure!(self, stmt), self)
+    }
+
+    fn flat_map_item(&mut self, item: P<ast::Item>) -> SmallVec<[P<ast::Item>; 1]> {
+        noop_flat_map_item(configure!(self, item), self)
+    }
+
+    fn flat_map_impl_item(&mut self, item: ast::ImplItem) -> SmallVec<[ast::ImplItem; 1]> {
+        noop_flat_map_impl_item(configure!(self, item), self)
+    }
+
+    fn flat_map_trait_item(&mut self, item: ast::TraitItem) -> SmallVec<[ast::TraitItem; 1]> {
+        noop_flat_map_trait_item(configure!(self, item), self)
+    }
+
+    fn visit_mac(&mut self, _mac: &mut ast::Mac) {
+        // Don't configure interpolated AST (cf. issue #34171).
+        // Interpolated AST will get configured once the surrounding tokens are parsed.
+    }
+
+    fn visit_pat(&mut self, pat: &mut P<ast::Pat>) {
+        self.configure_pat(pat);
+        noop_visit_pat(pat, self)
+    }
+
+    fn visit_fn_decl(&mut self, mut fn_decl: &mut P<ast::FnDecl>) {
+        self.configure_fn_decl(&mut fn_decl);
+        noop_visit_fn_decl(fn_decl, self);
+    }
+}
+
+fn is_cfg(attr: &ast::Attribute) -> bool {
+    attr.check_name(sym::cfg)
+}
+
+/// Process the potential `cfg` attributes on a module.
+/// Also determine if the module should be included in this configuration.
+pub fn process_configure_mod(
+    sess: &ParseSess,
+    cfg_mods: bool,
+    attrs: &[ast::Attribute],
+) -> (bool, Vec<ast::Attribute>) {
+    // Don't perform gated feature checking.
+    let mut strip_unconfigured = StripUnconfigured { sess, features: None };
+    let mut attrs = attrs.to_owned();
+    strip_unconfigured.process_cfg_attrs(&mut attrs);
+    (!cfg_mods || strip_unconfigured.in_cfg(&attrs), attrs)
+}
diff --git a/src/libsyntax_expand/expand.rs b/src/libsyntax_expand/expand.rs
index bdb50dbfb4f..b7f6364c2b5 100644
--- a/src/libsyntax_expand/expand.rs
+++ b/src/libsyntax_expand/expand.rs
@@ -3,27 +3,28 @@ use crate::proc_macro::{collect_derives, MarkAttrs};
 use crate::hygiene::{ExpnId, SyntaxContext, ExpnData, ExpnKind};
 use crate::mbe::macro_rules::annotate_err_with_kind;
 use crate::placeholders::{placeholder, PlaceholderExpander};
+use crate::config::StripUnconfigured;
+use crate::configure;
 
+use rustc_parse::DirectoryOwnership;
+use rustc_parse::parser::Parser;
+use rustc_parse::validate_attr;
 use syntax::ast::{self, AttrItem, Block, Ident, LitKind, NodeId, PatKind, Path};
 use syntax::ast::{MacStmtStyle, StmtKind, ItemKind};
 use syntax::attr::{self, HasAttrs};
 use syntax::source_map::respan;
-use syntax::configure;
-use syntax::config::StripUnconfigured;
 use syntax::feature_gate::{self, Features, GateIssue, is_builtin_attr, emit_feature_err};
 use syntax::mut_visit::*;
-use syntax::parse::{DirectoryOwnership, PResult};
-use syntax::parse::token;
-use syntax::parse::parser::Parser;
 use syntax::print::pprust;
 use syntax::ptr::P;
 use syntax::sess::ParseSess;
 use syntax::symbol::{sym, Symbol};
+use syntax::token;
 use syntax::tokenstream::{TokenStream, TokenTree};
 use syntax::visit::{self, Visitor};
 use syntax::util::map_in_place::MapInPlace;
 
-use errors::{Applicability, FatalError};
+use errors::{PResult, Applicability, FatalError};
 use smallvec::{smallvec, SmallVec};
 use syntax_pos::{Span, DUMMY_SP, FileName};
 
@@ -419,7 +420,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                     }
 
                     let mut item = self.fully_configure(item);
-                    item.visit_attrs(|attrs| attrs.retain(|a| a.path != sym::derive));
+                    item.visit_attrs(|attrs| attrs.retain(|a| !a.has_name(sym::derive)));
                     let mut helper_attrs = Vec::new();
                     let mut has_copy = false;
                     for ext in exts {
@@ -432,7 +433,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                     // can be in scope for all code produced by that container's expansion.
                     item.visit_with(&mut MarkAttrs(&helper_attrs));
                     if has_copy {
-                        self.cx.resolver.add_derives(invoc.expansion_data.id, SpecialDerives::COPY);
+                        self.cx.resolver.add_derive_copy(invoc.expansion_data.id);
                     }
 
                     let mut derive_placeholders = Vec::with_capacity(derives.len());
@@ -634,12 +635,13 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                         | Annotatable::Variant(..)
                             => panic!("unexpected annotatable"),
                     })), DUMMY_SP).into();
-                    let input = self.extract_proc_macro_attr_input(attr.item.tokens, span);
+                    let item = attr.unwrap_normal_item();
+                    let input = self.extract_proc_macro_attr_input(item.tokens, span);
                     let tok_result = expander.expand(self.cx, span, input, item_tok);
-                    self.parse_ast_fragment(tok_result, fragment_kind, &attr.item.path, span)
+                    self.parse_ast_fragment(tok_result, fragment_kind, &item.path, span)
                 }
                 SyntaxExtensionKind::LegacyAttr(expander) => {
-                    match attr.parse_meta(self.cx.parse_sess) {
+                    match validate_attr::parse_meta(self.cx.parse_sess, &attr) {
                         Ok(meta) => {
                             let item = expander.expand(self.cx, span, &meta, item);
                             fragment_kind.expect_from_annotatables(item)
@@ -974,7 +976,7 @@ impl<'a, 'b> InvocationCollector<'a, 'b> {
                        -> Option<ast::Attribute> {
         let attr = attrs.iter()
                         .position(|a| {
-                            if a.path == sym::derive {
+                            if a.has_name(sym::derive) {
                                 *after_derive = true;
                             }
                             !attr::is_known(a) && !is_builtin_attr(a)
@@ -982,7 +984,7 @@ impl<'a, 'b> InvocationCollector<'a, 'b> {
                         .map(|i| attrs.remove(i));
         if let Some(attr) = &attr {
             if !self.cx.ecfg.custom_inner_attributes() &&
-               attr.style == ast::AttrStyle::Inner && attr.path != sym::test {
+               attr.style == ast::AttrStyle::Inner && !attr.has_name(sym::test) {
                 emit_feature_err(&self.cx.parse_sess, sym::custom_inner_attributes,
                                  attr.span, GateIssue::Language,
                                  "non-builtin inner attributes are unstable");
@@ -1030,9 +1032,10 @@ impl<'a, 'b> InvocationCollector<'a, 'b> {
         let features = self.cx.ecfg.features.unwrap();
         for attr in attrs.iter() {
             feature_gate::check_attribute(attr, self.cx.parse_sess, features);
+            validate_attr::check_meta(self.cx.parse_sess, attr);
 
             // macros are expanded before any lint passes so this warning has to be hardcoded
-            if attr.path == sym::derive {
+            if attr.has_name(sym::derive) {
                 self.cx.struct_span_warn(attr.span, "`#[derive]` does nothing on macro invocations")
                     .note("this may become a hard error in a future release")
                     .emit();
@@ -1300,7 +1303,7 @@ impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> {
                         Some(_) => DirectoryOwnership::Owned {
                             relative: Some(item.ident),
                         },
-                        None => DirectoryOwnership::UnownedViaMod(false),
+                        None => DirectoryOwnership::UnownedViaMod,
                     };
                     path.pop();
                     module.directory = path;
@@ -1547,11 +1550,12 @@ impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> {
 
             let meta = attr::mk_list_item(Ident::with_dummy_span(sym::doc), items);
             *at = attr::Attribute {
-                item: AttrItem { path: meta.path, tokens: meta.kind.tokens(meta.span) },
+                kind: ast::AttrKind::Normal(
+                    AttrItem { path: meta.path, tokens: meta.kind.tokens(meta.span) },
+                ),
                 span: at.span,
                 id: at.id,
                 style: at.style,
-                is_sugared_doc: false,
             };
         } else {
             noop_visit_attribute(at, self)
diff --git a/src/libsyntax_expand/lib.rs b/src/libsyntax_expand/lib.rs
index 10eb3ecb20b..46d59dd249c 100644
--- a/src/libsyntax_expand/lib.rs
+++ b/src/libsyntax_expand/lib.rs
@@ -33,6 +33,35 @@ pub use mbe::macro_rules::compile_declarative_macro;
 pub mod base;
 pub mod build;
 pub mod expand;
+#[macro_use] pub mod config;
 pub mod proc_macro;
 
 crate mod mbe;
+
+// HACK(Centril, #64197): These shouldn't really be here.
+// Rather, they should be with their respective modules which are defined in other crates.
+// However, since for now constructing a `ParseSess` sorta requires `config` from this crate,
+// these tests will need to live here in the iterim.
+
+#[cfg(test)]
+mod tests;
+#[cfg(test)]
+mod parse {
+    #[cfg(test)]
+    mod tests;
+    #[cfg(test)]
+    mod lexer {
+        #[cfg(test)]
+        mod tests;
+    }
+}
+#[cfg(test)]
+mod tokenstream {
+    #[cfg(test)]
+    mod tests;
+}
+#[cfg(test)]
+mod mut_visit {
+    #[cfg(test)]
+    mod tests;
+}
diff --git a/src/libsyntax_expand/mbe.rs b/src/libsyntax_expand/mbe.rs
index d0f790638ef..6964d01b719 100644
--- a/src/libsyntax_expand/mbe.rs
+++ b/src/libsyntax_expand/mbe.rs
@@ -10,10 +10,10 @@ crate mod macro_rules;
 crate mod quoted;
 
 use syntax::ast;
-use syntax::parse::token::{self, Token, TokenKind};
+use syntax::token::{self, Token, TokenKind};
 use syntax::tokenstream::{DelimSpan};
 
-use syntax_pos::{BytePos, Span};
+use syntax_pos::Span;
 
 use rustc_data_structures::sync::Lrc;
 
@@ -27,23 +27,13 @@ struct Delimited {
 
 impl Delimited {
     /// Returns a `self::TokenTree` with a `Span` corresponding to the opening delimiter.
-    fn open_tt(&self, span: Span) -> TokenTree {
-        let open_span = if span.is_dummy() {
-            span
-        } else {
-            span.with_hi(span.lo() + BytePos(self.delim.len() as u32))
-        };
-        TokenTree::token(token::OpenDelim(self.delim), open_span)
+    fn open_tt(&self, span: DelimSpan) -> TokenTree {
+        TokenTree::token(token::OpenDelim(self.delim), span.open)
     }
 
     /// Returns a `self::TokenTree` with a `Span` corresponding to the closing delimiter.
-    fn close_tt(&self, span: Span) -> TokenTree {
-        let close_span = if span.is_dummy() {
-            span
-        } else {
-            span.with_lo(span.hi() - BytePos(self.delim.len() as u32))
-        };
-        TokenTree::token(token::CloseDelim(self.delim), close_span)
+    fn close_tt(&self, span: DelimSpan) -> TokenTree {
+        TokenTree::token(token::CloseDelim(self.delim), span.close)
     }
 }
 
@@ -138,10 +128,10 @@ impl TokenTree {
             }
             (&TokenTree::Delimited(span, ref delimed), _) => {
                 if index == 0 {
-                    return delimed.open_tt(span.open);
+                    return delimed.open_tt(span);
                 }
                 if index == delimed.tts.len() + 1 {
-                    return delimed.close_tt(span.close);
+                    return delimed.close_tt(span);
                 }
                 delimed.tts[index - 1].clone()
             }
diff --git a/src/libsyntax_expand/mbe/macro_check.rs b/src/libsyntax_expand/mbe/macro_check.rs
index 50abda8d45e..25754ed4217 100644
--- a/src/libsyntax_expand/mbe/macro_check.rs
+++ b/src/libsyntax_expand/mbe/macro_check.rs
@@ -108,7 +108,7 @@ use crate::mbe::{KleeneToken, TokenTree};
 
 use syntax::ast::NodeId;
 use syntax::early_buffered_lints::BufferedEarlyLintId;
-use syntax::parse::token::{DelimToken, Token, TokenKind};
+use syntax::token::{DelimToken, Token, TokenKind};
 use syntax::sess::ParseSess;
 use syntax::symbol::{kw, sym};
 
diff --git a/src/libsyntax_expand/mbe/macro_parser.rs b/src/libsyntax_expand/mbe/macro_parser.rs
index 3efe22626a9..0dcb2c39fdc 100644
--- a/src/libsyntax_expand/mbe/macro_parser.rs
+++ b/src/libsyntax_expand/mbe/macro_parser.rs
@@ -76,16 +76,16 @@ use TokenTreeOrTokenTreeSlice::*;
 
 use crate::mbe::{self, TokenTree};
 
+use rustc_parse::Directory;
+use rustc_parse::parser::{Parser, PathStyle};
 use syntax::ast::{Ident, Name};
-use syntax::parse::{Directory, PResult};
-use syntax::parse::parser::{Parser, PathStyle};
-use syntax::parse::token::{self, DocComment, Nonterminal, Token};
 use syntax::print::pprust;
 use syntax::sess::ParseSess;
 use syntax::symbol::{kw, sym, Symbol};
+use syntax::token::{self, DocComment, Nonterminal, Token};
 use syntax::tokenstream::{DelimSpan, TokenStream};
 
-use errors::FatalError;
+use errors::{PResult, FatalError};
 use smallvec::{smallvec, SmallVec};
 use syntax_pos::Span;
 
@@ -652,7 +652,7 @@ pub(super) fn parse(
         directory,
         recurse_into_modules,
         true,
-        syntax::MACRO_ARGUMENTS,
+        rustc_parse::MACRO_ARGUMENTS,
     );
 
     // A queue of possible matcher positions. We initialize it with the matcher position in which
diff --git a/src/libsyntax_expand/mbe/macro_rules.rs b/src/libsyntax_expand/mbe/macro_rules.rs
index 9a4130b2d8d..72b7996628a 100644
--- a/src/libsyntax_expand/mbe/macro_rules.rs
+++ b/src/libsyntax_expand/mbe/macro_rules.rs
@@ -8,31 +8,30 @@ use crate::mbe::macro_parser::{Error, Failure, Success};
 use crate::mbe::macro_parser::{MatchedNonterminal, MatchedSeq, NamedParseResult};
 use crate::mbe::transcribe::transcribe;
 
+use rustc_parse::parser::Parser;
+use rustc_parse::Directory;
 use syntax::ast;
 use syntax::attr::{self, TransparencyError};
 use syntax::edition::Edition;
 use syntax::feature_gate::Features;
-use syntax::parse::parser::Parser;
-use syntax::parse::token::TokenKind::*;
-use syntax::parse::token::{self, NtTT, Token};
-use syntax::parse::Directory;
 use syntax::print::pprust;
 use syntax::sess::ParseSess;
 use syntax::symbol::{kw, sym, Symbol};
+use syntax::token::{self, NtTT, Token, TokenKind::*};
 use syntax::tokenstream::{DelimSpan, TokenStream};
+use syntax_pos::hygiene::Transparency;
+use syntax_pos::Span;
 
 use errors::{DiagnosticBuilder, FatalError};
 use log::debug;
-use syntax_pos::hygiene::Transparency;
-use syntax_pos::Span;
 
 use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::sync::Lrc;
 use std::borrow::Cow;
 use std::collections::hash_map::Entry;
-use std::slice;
+use std::{mem, slice};
 
 use errors::Applicability;
-use rustc_data_structures::sync::Lrc;
 
 const VALID_FRAGMENT_NAMES_MSG: &str = "valid fragment specifiers are \
                                         `ident`, `block`, `stmt`, `expr`, `pat`, `ty`, `lifetime`, \
@@ -182,7 +181,6 @@ fn generic_extension<'cx>(
 
     // Which arm's failure should we report? (the one furthest along)
     let mut best_failure: Option<(Token, &str)> = None;
-
     for (i, lhs) in lhses.iter().enumerate() {
         // try each arm's matchers
         let lhs_tt = match *lhs {
@@ -190,8 +188,18 @@ fn generic_extension<'cx>(
             _ => cx.span_bug(sp, "malformed macro lhs"),
         };
 
+        // Take a snapshot of the state of pre-expansion gating at this point.
+        // This is used so that if a matcher is not `Success(..)`ful,
+        // then the spans which became gated when parsing the unsucessful matcher
+        // are not recorded. On the first `Success(..)`ful matcher, the spans are merged.
+        let mut gated_spans_snaphot = mem::take(&mut *cx.parse_sess.gated_spans.spans.borrow_mut());
+
         match parse_tt(cx, lhs_tt, arg.clone()) {
             Success(named_matches) => {
+                // The matcher was `Success(..)`ful.
+                // Merge the gated spans from parsing the matcher with the pre-existing ones.
+                cx.parse_sess.gated_spans.merge(gated_spans_snaphot);
+
                 let rhs = match rhses[i] {
                     // ignore delimiters
                     mbe::TokenTree::Delimited(_, ref delimed) => delimed.tts.clone(),
@@ -225,7 +233,7 @@ fn generic_extension<'cx>(
                 };
                 let mut p = Parser::new(cx.parse_sess(), tts, Some(directory), true, false, None);
                 p.root_module_name =
-                    cx.current_expansion.module.mod_path.last().map(|id| id.as_str().to_string());
+                    cx.current_expansion.module.mod_path.last().map(|id| id.to_string());
                 p.last_type_ascription = cx.current_expansion.prior_type_ascription;
 
                 p.process_potential_macro_variable();
@@ -248,6 +256,10 @@ fn generic_extension<'cx>(
             },
             Error(err_sp, ref msg) => cx.span_fatal(err_sp.substitute_dummy(sp), &msg[..]),
         }
+
+        // The matcher was not `Success(..)`ful.
+        // Restore to the state before snapshotting and maybe try again.
+        mem::swap(&mut gated_spans_snaphot, &mut cx.parse_sess.gated_spans.spans.borrow_mut());
     }
 
     let (token, label) = best_failure.expect("ran no matchers");
@@ -566,7 +578,7 @@ impl FirstSets {
                     }
                     TokenTree::Delimited(span, ref delimited) => {
                         build_recur(sets, &delimited.tts[..]);
-                        first.replace_with(delimited.open_tt(span.open));
+                        first.replace_with(delimited.open_tt(span));
                     }
                     TokenTree::Sequence(sp, ref seq_rep) => {
                         let subfirst = build_recur(sets, &seq_rep.tts[..]);
@@ -628,7 +640,7 @@ impl FirstSets {
                     return first;
                 }
                 TokenTree::Delimited(span, ref delimited) => {
-                    first.add_one(delimited.open_tt(span.open));
+                    first.add_one(delimited.open_tt(span));
                     return first;
                 }
                 TokenTree::Sequence(sp, ref seq_rep) => {
@@ -826,7 +838,7 @@ fn check_matcher_core(
                 }
             }
             TokenTree::Delimited(span, ref d) => {
-                let my_suffix = TokenSet::singleton(d.close_tt(span.close));
+                let my_suffix = TokenSet::singleton(d.close_tt(span));
                 check_matcher_core(sess, features, attrs, first_sets, &d.tts, &my_suffix);
                 // don't track non NT tokens
                 last.replace_with_irrelevant();
diff --git a/src/libsyntax_expand/mbe/quoted.rs b/src/libsyntax_expand/mbe/quoted.rs
index cedd59233ad..dec504c0d97 100644
--- a/src/libsyntax_expand/mbe/quoted.rs
+++ b/src/libsyntax_expand/mbe/quoted.rs
@@ -2,10 +2,10 @@ use crate::mbe::macro_parser;
 use crate::mbe::{TokenTree, KleeneOp, KleeneToken, SequenceRepetition, Delimited};
 
 use syntax::ast;
-use syntax::parse::token::{self, Token};
 use syntax::print::pprust;
 use syntax::sess::ParseSess;
 use syntax::symbol::kw;
+use syntax::token::{self, Token};
 use syntax::tokenstream;
 
 use syntax_pos::Span;
diff --git a/src/libsyntax_expand/mbe/transcribe.rs b/src/libsyntax_expand/mbe/transcribe.rs
index 94523bbf91b..4092d4b97de 100644
--- a/src/libsyntax_expand/mbe/transcribe.rs
+++ b/src/libsyntax_expand/mbe/transcribe.rs
@@ -4,12 +4,12 @@ use crate::mbe::macro_parser::{MatchedNonterminal, MatchedSeq, NamedMatch};
 
 use syntax::ast::{Ident, Mac};
 use syntax::mut_visit::{self, MutVisitor};
-use syntax::parse::token::{self, NtTT, Token};
+use syntax::token::{self, NtTT, Token};
 use syntax::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndJoint};
 
 use smallvec::{smallvec, SmallVec};
 
-use errors::pluralise;
+use errors::pluralize;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::sync::Lrc;
 use syntax_pos::hygiene::{ExpnId, Transparency};
@@ -350,10 +350,10 @@ impl LockstepIterSize {
                         "meta-variable `{}` repeats {} time{}, but `{}` repeats {} time{}",
                         l_id,
                         l_len,
-                        pluralise!(l_len),
+                        pluralize!(l_len),
                         r_id,
                         r_len,
-                        pluralise!(r_len),
+                        pluralize!(r_len),
                     );
                     LockstepIterSize::Contradiction(msg)
                 }
diff --git a/src/libsyntax_expand/mut_visit/tests.rs b/src/libsyntax_expand/mut_visit/tests.rs
new file mode 100644
index 00000000000..30e812a1179
--- /dev/null
+++ b/src/libsyntax_expand/mut_visit/tests.rs
@@ -0,0 +1,70 @@
+use crate::tests::{string_to_crate, matches_codepattern};
+
+use syntax::ast::{self, Ident};
+use syntax::print::pprust;
+use syntax::mut_visit::{self, MutVisitor};
+use syntax::with_default_globals;
+
+// This version doesn't care about getting comments or doc-strings in.
+fn fake_print_crate(s: &mut pprust::State<'_>,
+                    krate: &ast::Crate) {
+    s.print_mod(&krate.module, &krate.attrs)
+}
+
+// Change every identifier to "zz".
+struct ToZzIdentMutVisitor;
+
+impl MutVisitor for ToZzIdentMutVisitor {
+    fn visit_ident(&mut self, ident: &mut ast::Ident) {
+        *ident = Ident::from_str("zz");
+    }
+    fn visit_mac(&mut self, mac: &mut ast::Mac) {
+        mut_visit::noop_visit_mac(mac, self)
+    }
+}
+
+// Maybe add to `expand.rs`.
+macro_rules! assert_pred {
+    ($pred:expr, $predname:expr, $a:expr , $b:expr) => (
+        {
+            let pred_val = $pred;
+            let a_val = $a;
+            let b_val = $b;
+            if !(pred_val(&a_val, &b_val)) {
+                panic!("expected args satisfying {}, got {} and {}",
+                        $predname, a_val, b_val);
+            }
+        }
+    )
+}
+
+// Make sure idents get transformed everywhere.
+#[test] fn ident_transformation () {
+    with_default_globals(|| {
+        let mut zz_visitor = ToZzIdentMutVisitor;
+        let mut krate = string_to_crate(
+            "#[a] mod b {fn c (d : e, f : g) {h!(i,j,k);l;m}}".to_string());
+        zz_visitor.visit_crate(&mut krate);
+        assert_pred!(
+            matches_codepattern,
+            "matches_codepattern",
+            pprust::to_string(|s| fake_print_crate(s, &krate)),
+            "#[zz]mod zz{fn zz(zz:zz,zz:zz){zz!(zz,zz,zz);zz;zz}}".to_string());
+    })
+}
+
+// Make sure idents get transformed even inside macro defs.
+#[test] fn ident_transformation_in_defs () {
+    with_default_globals(|| {
+        let mut zz_visitor = ToZzIdentMutVisitor;
+        let mut krate = string_to_crate(
+            "macro_rules! a {(b $c:expr $(d $e:token)f+ => \
+            (g $(d $d $e)+))} ".to_string());
+        zz_visitor.visit_crate(&mut krate);
+        assert_pred!(
+            matches_codepattern,
+            "matches_codepattern",
+            pprust::to_string(|s| fake_print_crate(s, &krate)),
+            "macro_rules! zz{(zz$zz:zz$(zz $zz:zz)zz+=>(zz$(zz$zz$zz)+))}".to_string());
+    })
+}
diff --git a/src/libsyntax_expand/parse/lexer/tests.rs b/src/libsyntax_expand/parse/lexer/tests.rs
new file mode 100644
index 00000000000..d8a1c359c95
--- /dev/null
+++ b/src/libsyntax_expand/parse/lexer/tests.rs
@@ -0,0 +1,277 @@
+use crate::config::process_configure_mod;
+
+use rustc_data_structures::sync::Lrc;
+use rustc_parse::lexer::StringReader;
+use syntax::token::{self, Token, TokenKind};
+use syntax::sess::ParseSess;
+use syntax::source_map::{SourceMap, FilePathMapping};
+use syntax::util::comments::is_doc_comment;
+use syntax::with_default_globals;
+use syntax_pos::symbol::Symbol;
+use syntax_pos::{BytePos, Span};
+
+use errors::{Handler, emitter::EmitterWriter};
+use std::io;
+use std::path::PathBuf;
+
+fn mk_sess(sm: Lrc<SourceMap>) -> ParseSess {
+    let emitter = EmitterWriter::new(
+        Box::new(io::sink()),
+        Some(sm.clone()),
+        false,
+        false,
+        false,
+        None,
+        false,
+    );
+    ParseSess::with_span_handler(
+        Handler::with_emitter(true, None, Box::new(emitter)),
+        sm,
+        process_configure_mod,
+    )
+}
+
+// Creates a string reader for the given string.
+fn setup<'a>(sm: &SourceMap,
+                sess: &'a ParseSess,
+                teststr: String)
+                -> StringReader<'a> {
+    let sf = sm.new_source_file(PathBuf::from(teststr.clone()).into(), teststr);
+    StringReader::new(sess, sf, None)
+}
+
+#[test]
+fn t1() {
+    with_default_globals(|| {
+        let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
+        let sh = mk_sess(sm.clone());
+        let mut string_reader = setup(
+            &sm,
+            &sh,
+            "/* my source file */ fn main() { println!(\"zebra\"); }\n".to_string(),
+        );
+        assert_eq!(string_reader.next_token(), token::Comment);
+        assert_eq!(string_reader.next_token(), token::Whitespace);
+        let tok1 = string_reader.next_token();
+        let tok2 = Token::new(
+            mk_ident("fn"),
+            Span::with_root_ctxt(BytePos(21), BytePos(23)),
+        );
+        assert_eq!(tok1.kind, tok2.kind);
+        assert_eq!(tok1.span, tok2.span);
+        assert_eq!(string_reader.next_token(), token::Whitespace);
+        // Read another token.
+        let tok3 = string_reader.next_token();
+        assert_eq!(string_reader.pos.clone(), BytePos(28));
+        let tok4 = Token::new(
+            mk_ident("main"),
+            Span::with_root_ctxt(BytePos(24), BytePos(28)),
+        );
+        assert_eq!(tok3.kind, tok4.kind);
+        assert_eq!(tok3.span, tok4.span);
+
+        assert_eq!(string_reader.next_token(), token::OpenDelim(token::Paren));
+        assert_eq!(string_reader.pos.clone(), BytePos(29))
+    })
+}
+
+// Checks that the given reader produces the desired stream
+// of tokens (stop checking after exhausting `expected`).
+fn check_tokenization(mut string_reader: StringReader<'_>, expected: Vec<TokenKind>) {
+    for expected_tok in &expected {
+        assert_eq!(&string_reader.next_token(), expected_tok);
+    }
+}
+
+// Makes the identifier by looking up the string in the interner.
+fn mk_ident(id: &str) -> TokenKind {
+    token::Ident(Symbol::intern(id), false)
+}
+
+fn mk_lit(kind: token::LitKind, symbol: &str, suffix: Option<&str>) -> TokenKind {
+    TokenKind::lit(kind, Symbol::intern(symbol), suffix.map(Symbol::intern))
+}
+
+#[test]
+fn doublecolon_parsing() {
+    with_default_globals(|| {
+        let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
+        let sh = mk_sess(sm.clone());
+        check_tokenization(
+            setup(&sm, &sh, "a b".to_string()),
+            vec![mk_ident("a"), token::Whitespace, mk_ident("b")],
+        );
+    })
+}
+
+#[test]
+fn doublecolon_parsing_2() {
+    with_default_globals(|| {
+        let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
+        let sh = mk_sess(sm.clone());
+        check_tokenization(
+            setup(&sm, &sh, "a::b".to_string()),
+            vec![mk_ident("a"), token::Colon, token::Colon, mk_ident("b")],
+        );
+    })
+}
+
+#[test]
+fn doublecolon_parsing_3() {
+    with_default_globals(|| {
+        let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
+        let sh = mk_sess(sm.clone());
+        check_tokenization(
+            setup(&sm, &sh, "a ::b".to_string()),
+            vec![mk_ident("a"), token::Whitespace, token::Colon, token::Colon, mk_ident("b")],
+        );
+    })
+}
+
+#[test]
+fn doublecolon_parsing_4() {
+    with_default_globals(|| {
+        let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
+        let sh = mk_sess(sm.clone());
+        check_tokenization(
+            setup(&sm, &sh, "a:: b".to_string()),
+            vec![mk_ident("a"), token::Colon, token::Colon, token::Whitespace, mk_ident("b")],
+        );
+    })
+}
+
+#[test]
+fn character_a() {
+    with_default_globals(|| {
+        let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
+        let sh = mk_sess(sm.clone());
+        assert_eq!(
+            setup(&sm, &sh, "'a'".to_string()).next_token(),
+            mk_lit(token::Char, "a", None),
+        );
+    })
+}
+
+#[test]
+fn character_space() {
+    with_default_globals(|| {
+        let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
+        let sh = mk_sess(sm.clone());
+        assert_eq!(
+            setup(&sm, &sh, "' '".to_string()).next_token(),
+            mk_lit(token::Char, " ", None),
+        );
+    })
+}
+
+#[test]
+fn character_escaped() {
+    with_default_globals(|| {
+        let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
+        let sh = mk_sess(sm.clone());
+        assert_eq!(
+            setup(&sm, &sh, "'\\n'".to_string()).next_token(),
+            mk_lit(token::Char, "\\n", None),
+        );
+    })
+}
+
+#[test]
+fn lifetime_name() {
+    with_default_globals(|| {
+        let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
+        let sh = mk_sess(sm.clone());
+        assert_eq!(
+            setup(&sm, &sh, "'abc".to_string()).next_token(),
+            token::Lifetime(Symbol::intern("'abc")),
+        );
+    })
+}
+
+#[test]
+fn raw_string() {
+    with_default_globals(|| {
+        let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
+        let sh = mk_sess(sm.clone());
+        assert_eq!(
+            setup(&sm, &sh, "r###\"\"#a\\b\x00c\"\"###".to_string()).next_token(),
+            mk_lit(token::StrRaw(3), "\"#a\\b\x00c\"", None),
+        );
+    })
+}
+
+#[test]
+fn literal_suffixes() {
+    with_default_globals(|| {
+        let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
+        let sh = mk_sess(sm.clone());
+        macro_rules! test {
+            ($input: expr, $tok_type: ident, $tok_contents: expr) => {{
+                assert_eq!(
+                    setup(&sm, &sh, format!("{}suffix", $input)).next_token(),
+                    mk_lit(token::$tok_type, $tok_contents, Some("suffix")),
+                );
+                // with a whitespace separator
+                assert_eq!(
+                    setup(&sm, &sh, format!("{} suffix", $input)).next_token(),
+                    mk_lit(token::$tok_type, $tok_contents, None),
+                );
+            }}
+        }
+
+        test!("'a'", Char, "a");
+        test!("b'a'", Byte, "a");
+        test!("\"a\"", Str, "a");
+        test!("b\"a\"", ByteStr, "a");
+        test!("1234", Integer, "1234");
+        test!("0b101", Integer, "0b101");
+        test!("0xABC", Integer, "0xABC");
+        test!("1.0", Float, "1.0");
+        test!("1.0e10", Float, "1.0e10");
+
+        assert_eq!(
+            setup(&sm, &sh, "2us".to_string()).next_token(),
+            mk_lit(token::Integer, "2", Some("us")),
+        );
+        assert_eq!(
+            setup(&sm, &sh, "r###\"raw\"###suffix".to_string()).next_token(),
+            mk_lit(token::StrRaw(3), "raw", Some("suffix")),
+        );
+        assert_eq!(
+            setup(&sm, &sh, "br###\"raw\"###suffix".to_string()).next_token(),
+            mk_lit(token::ByteStrRaw(3), "raw", Some("suffix")),
+        );
+    })
+}
+
+#[test]
+fn line_doc_comments() {
+    assert!(is_doc_comment("///"));
+    assert!(is_doc_comment("/// blah"));
+    assert!(!is_doc_comment("////"));
+}
+
+#[test]
+fn nested_block_comments() {
+    with_default_globals(|| {
+        let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
+        let sh = mk_sess(sm.clone());
+        let mut lexer = setup(&sm, &sh, "/* /* */ */'a'".to_string());
+        assert_eq!(lexer.next_token(), token::Comment);
+        assert_eq!(lexer.next_token(), mk_lit(token::Char, "a", None));
+    })
+}
+
+#[test]
+fn crlf_comments() {
+    with_default_globals(|| {
+        let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
+        let sh = mk_sess(sm.clone());
+        let mut lexer = setup(&sm, &sh, "// test\r\n/// test\r\n".to_string());
+        let comment = lexer.next_token();
+        assert_eq!(comment.kind, token::Comment);
+        assert_eq!((comment.span.lo(), comment.span.hi()), (BytePos(0), BytePos(7)));
+        assert_eq!(lexer.next_token(), token::Whitespace);
+        assert_eq!(lexer.next_token(), token::DocComment(Symbol::intern("/// test")));
+    })
+}
diff --git a/src/libsyntax_expand/parse/tests.rs b/src/libsyntax_expand/parse/tests.rs
new file mode 100644
index 00000000000..60911617995
--- /dev/null
+++ b/src/libsyntax_expand/parse/tests.rs
@@ -0,0 +1,338 @@
+use crate::config::process_configure_mod;
+use crate::tests::{matches_codepattern, string_to_stream, with_error_checking_parse};
+
+use rustc_parse::new_parser_from_source_str;
+use syntax::ast::{self, Name, PatKind};
+use syntax::attr::first_attr_value_str_by_name;
+use syntax::sess::ParseSess;
+use syntax::token::{self, Token};
+use syntax::print::pprust::item_to_string;
+use syntax::ptr::P;
+use syntax::source_map::FilePathMapping;
+use syntax::symbol::{kw, sym};
+use syntax::tokenstream::{DelimSpan, TokenTree, TokenStream};
+use syntax::visit;
+use syntax::with_default_globals;
+use syntax_pos::{Span, BytePos, Pos, FileName};
+use errors::PResult;
+
+use std::path::PathBuf;
+
+fn sess() -> ParseSess {
+    ParseSess::new(FilePathMapping::empty(), process_configure_mod)
+}
+
+/// Parses an item.
+///
+/// Returns `Ok(Some(item))` when successful, `Ok(None)` when no item was found, and `Err`
+/// when a syntax error occurred.
+fn parse_item_from_source_str(name: FileName, source: String, sess: &ParseSess)
+                                    -> PResult<'_, Option<P<ast::Item>>> {
+    new_parser_from_source_str(sess, name, source).parse_item()
+}
+
+// Produces a `syntax_pos::span`.
+fn sp(a: u32, b: u32) -> Span {
+    Span::with_root_ctxt(BytePos(a), BytePos(b))
+}
+
+/// Parses a string, return an expression.
+fn string_to_expr(source_str : String) -> P<ast::Expr> {
+    with_error_checking_parse(source_str, &sess(), |p| p.parse_expr())
+}
+
+/// Parses a string, returns an item.
+fn string_to_item(source_str : String) -> Option<P<ast::Item>> {
+    with_error_checking_parse(source_str, &sess(), |p| p.parse_item())
+}
+
+#[should_panic]
+#[test] fn bad_path_expr_1() {
+    with_default_globals(|| {
+        string_to_expr("::abc::def::return".to_string());
+    })
+}
+
+// Checks the token-tree-ization of macros.
+#[test]
+fn string_to_tts_macro () {
+    with_default_globals(|| {
+        let tts: Vec<_> =
+            string_to_stream("macro_rules! zip (($a)=>($a))".to_string()).trees().collect();
+        let tts: &[TokenTree] = &tts[..];
+
+        match tts {
+            [
+                TokenTree::Token(Token { kind: token::Ident(name_macro_rules, false), .. }),
+                TokenTree::Token(Token { kind: token::Not, .. }),
+                TokenTree::Token(Token { kind: token::Ident(name_zip, false), .. }),
+                TokenTree::Delimited(_, macro_delim,  macro_tts)
+            ]
+            if name_macro_rules == &sym::macro_rules && name_zip.as_str() == "zip" => {
+                let tts = &macro_tts.trees().collect::<Vec<_>>();
+                match &tts[..] {
+                    [
+                        TokenTree::Delimited(_, first_delim, first_tts),
+                        TokenTree::Token(Token { kind: token::FatArrow, .. }),
+                        TokenTree::Delimited(_, second_delim, second_tts),
+                    ]
+                    if macro_delim == &token::Paren => {
+                        let tts = &first_tts.trees().collect::<Vec<_>>();
+                        match &tts[..] {
+                            [
+                                TokenTree::Token(Token { kind: token::Dollar, .. }),
+                                TokenTree::Token(Token { kind: token::Ident(name, false), .. }),
+                            ]
+                            if first_delim == &token::Paren && name.as_str() == "a" => {},
+                            _ => panic!("value 3: {:?} {:?}", first_delim, first_tts),
+                        }
+                        let tts = &second_tts.trees().collect::<Vec<_>>();
+                        match &tts[..] {
+                            [
+                                TokenTree::Token(Token { kind: token::Dollar, .. }),
+                                TokenTree::Token(Token { kind: token::Ident(name, false), .. }),
+                            ]
+                            if second_delim == &token::Paren && name.as_str() == "a" => {},
+                            _ => panic!("value 4: {:?} {:?}", second_delim, second_tts),
+                        }
+                    },
+                    _ => panic!("value 2: {:?} {:?}", macro_delim, macro_tts),
+                }
+            },
+            _ => panic!("value: {:?}",tts),
+        }
+    })
+}
+
+#[test]
+fn string_to_tts_1() {
+    with_default_globals(|| {
+        let tts = string_to_stream("fn a (b : i32) { b; }".to_string());
+
+        let expected = TokenStream::new(vec![
+            TokenTree::token(token::Ident(kw::Fn, false), sp(0, 2)).into(),
+            TokenTree::token(token::Ident(Name::intern("a"), false), sp(3, 4)).into(),
+            TokenTree::Delimited(
+                DelimSpan::from_pair(sp(5, 6), sp(13, 14)),
+                token::DelimToken::Paren,
+                TokenStream::new(vec![
+                    TokenTree::token(token::Ident(Name::intern("b"), false), sp(6, 7)).into(),
+                    TokenTree::token(token::Colon, sp(8, 9)).into(),
+                    TokenTree::token(token::Ident(sym::i32, false), sp(10, 13)).into(),
+                ]).into(),
+            ).into(),
+            TokenTree::Delimited(
+                DelimSpan::from_pair(sp(15, 16), sp(20, 21)),
+                token::DelimToken::Brace,
+                TokenStream::new(vec![
+                    TokenTree::token(token::Ident(Name::intern("b"), false), sp(17, 18)).into(),
+                    TokenTree::token(token::Semi, sp(18, 19)).into(),
+                ]).into(),
+            ).into()
+        ]);
+
+        assert_eq!(tts, expected);
+    })
+}
+
+#[test] fn parse_use() {
+    with_default_globals(|| {
+        let use_s = "use foo::bar::baz;";
+        let vitem = string_to_item(use_s.to_string()).unwrap();
+        let vitem_s = item_to_string(&vitem);
+        assert_eq!(&vitem_s[..], use_s);
+
+        let use_s = "use foo::bar as baz;";
+        let vitem = string_to_item(use_s.to_string()).unwrap();
+        let vitem_s = item_to_string(&vitem);
+        assert_eq!(&vitem_s[..], use_s);
+    })
+}
+
+#[test] fn parse_extern_crate() {
+    with_default_globals(|| {
+        let ex_s = "extern crate foo;";
+        let vitem = string_to_item(ex_s.to_string()).unwrap();
+        let vitem_s = item_to_string(&vitem);
+        assert_eq!(&vitem_s[..], ex_s);
+
+        let ex_s = "extern crate foo as bar;";
+        let vitem = string_to_item(ex_s.to_string()).unwrap();
+        let vitem_s = item_to_string(&vitem);
+        assert_eq!(&vitem_s[..], ex_s);
+    })
+}
+
+fn get_spans_of_pat_idents(src: &str) -> Vec<Span> {
+    let item = string_to_item(src.to_string()).unwrap();
+
+    struct PatIdentVisitor {
+        spans: Vec<Span>
+    }
+    impl<'a> visit::Visitor<'a> for PatIdentVisitor {
+        fn visit_pat(&mut self, p: &'a ast::Pat) {
+            match p.kind {
+                PatKind::Ident(_ , ref ident, _) => {
+                    self.spans.push(ident.span.clone());
+                }
+                _ => {
+                    visit::walk_pat(self, p);
+                }
+            }
+        }
+    }
+    let mut v = PatIdentVisitor { spans: Vec::new() };
+    visit::walk_item(&mut v, &item);
+    return v.spans;
+}
+
+#[test] fn span_of_self_arg_pat_idents_are_correct() {
+    with_default_globals(|| {
+
+        let srcs = ["impl z { fn a (&self, &myarg: i32) {} }",
+                    "impl z { fn a (&mut self, &myarg: i32) {} }",
+                    "impl z { fn a (&'a self, &myarg: i32) {} }",
+                    "impl z { fn a (self, &myarg: i32) {} }",
+                    "impl z { fn a (self: Foo, &myarg: i32) {} }",
+                    ];
+
+        for &src in &srcs {
+            let spans = get_spans_of_pat_idents(src);
+            let (lo, hi) = (spans[0].lo(), spans[0].hi());
+            assert!("self" == &src[lo.to_usize()..hi.to_usize()],
+                    "\"{}\" != \"self\". src=\"{}\"",
+                    &src[lo.to_usize()..hi.to_usize()], src)
+        }
+    })
+}
+
+#[test] fn parse_exprs () {
+    with_default_globals(|| {
+        // just make sure that they parse....
+        string_to_expr("3 + 4".to_string());
+        string_to_expr("a::z.froob(b,&(987+3))".to_string());
+    })
+}
+
+#[test] fn attrs_fix_bug () {
+    with_default_globals(|| {
+        string_to_item("pub fn mk_file_writer(path: &Path, flags: &[FileFlag])
+                -> Result<Box<Writer>, String> {
+#[cfg(windows)]
+fn wb() -> c_int {
+    (O_WRONLY | libc::consts::os::extra::O_BINARY) as c_int
+}
+
+#[cfg(unix)]
+fn wb() -> c_int { O_WRONLY as c_int }
+
+let mut fflags: c_int = wb();
+}".to_string());
+    })
+}
+
+#[test] fn crlf_doc_comments() {
+    with_default_globals(|| {
+        let sess = sess();
+
+        let name_1 = FileName::Custom("crlf_source_1".to_string());
+        let source = "/// doc comment\r\nfn foo() {}".to_string();
+        let item = parse_item_from_source_str(name_1, source, &sess)
+            .unwrap().unwrap();
+        let doc = first_attr_value_str_by_name(&item.attrs, sym::doc).unwrap();
+        assert_eq!(doc.as_str(), "/// doc comment");
+
+        let name_2 = FileName::Custom("crlf_source_2".to_string());
+        let source = "/// doc comment\r\n/// line 2\r\nfn foo() {}".to_string();
+        let item = parse_item_from_source_str(name_2, source, &sess)
+            .unwrap().unwrap();
+        let docs = item.attrs.iter().filter(|a| a.has_name(sym::doc))
+                    .map(|a| a.value_str().unwrap().to_string()).collect::<Vec<_>>();
+        let b: &[_] = &["/// doc comment".to_string(), "/// line 2".to_string()];
+        assert_eq!(&docs[..], b);
+
+        let name_3 = FileName::Custom("clrf_source_3".to_string());
+        let source = "/** doc comment\r\n *  with CRLF */\r\nfn foo() {}".to_string();
+        let item = parse_item_from_source_str(name_3, source, &sess).unwrap().unwrap();
+        let doc = first_attr_value_str_by_name(&item.attrs, sym::doc).unwrap();
+        assert_eq!(doc.as_str(), "/** doc comment\n *  with CRLF */");
+    });
+}
+
+#[test]
+fn ttdelim_span() {
+    fn parse_expr_from_source_str(
+        name: FileName, source: String, sess: &ParseSess
+    ) -> PResult<'_, P<ast::Expr>> {
+        new_parser_from_source_str(sess, name, source).parse_expr()
+    }
+
+    with_default_globals(|| {
+        let sess = sess();
+        let expr = parse_expr_from_source_str(PathBuf::from("foo").into(),
+            "foo!( fn main() { body } )".to_string(), &sess).unwrap();
+
+        let tts: Vec<_> = match expr.kind {
+            ast::ExprKind::Mac(ref mac) => mac.stream().trees().collect(),
+            _ => panic!("not a macro"),
+        };
+
+        let span = tts.iter().rev().next().unwrap().span();
+
+        match sess.source_map().span_to_snippet(span) {
+            Ok(s) => assert_eq!(&s[..], "{ body }"),
+            Err(_) => panic!("could not get snippet"),
+        }
+    });
+}
+
+// This tests that when parsing a string (rather than a file) we don't try
+// and read in a file for a module declaration and just parse a stub.
+// See `recurse_into_file_modules` in the parser.
+#[test]
+fn out_of_line_mod() {
+    with_default_globals(|| {
+        let item = parse_item_from_source_str(
+            PathBuf::from("foo").into(),
+            "mod foo { struct S; mod this_does_not_exist; }".to_owned(),
+            &sess(),
+        ).unwrap().unwrap();
+
+        if let ast::ItemKind::Mod(ref m) = item.kind {
+            assert!(m.items.len() == 2);
+        } else {
+            panic!();
+        }
+    });
+}
+
+#[test]
+fn eqmodws() {
+    assert_eq!(matches_codepattern("",""),true);
+    assert_eq!(matches_codepattern("","a"),false);
+    assert_eq!(matches_codepattern("a",""),false);
+    assert_eq!(matches_codepattern("a","a"),true);
+    assert_eq!(matches_codepattern("a b","a   \n\t\r  b"),true);
+    assert_eq!(matches_codepattern("a b ","a   \n\t\r  b"),true);
+    assert_eq!(matches_codepattern("a b","a   \n\t\r  b "),false);
+    assert_eq!(matches_codepattern("a   b","a b"),true);
+    assert_eq!(matches_codepattern("ab","a b"),false);
+    assert_eq!(matches_codepattern("a   b","ab"),true);
+    assert_eq!(matches_codepattern(" a   b","ab"),true);
+}
+
+#[test]
+fn pattern_whitespace() {
+    assert_eq!(matches_codepattern("","\x0C"), false);
+    assert_eq!(matches_codepattern("a b ","a   \u{0085}\n\t\r  b"),true);
+    assert_eq!(matches_codepattern("a b","a   \u{0085}\n\t\r  b "),false);
+}
+
+#[test]
+fn non_pattern_whitespace() {
+    // These have the property 'White_Space' but not 'Pattern_White_Space'
+    assert_eq!(matches_codepattern("a b","a\u{2002}b"), false);
+    assert_eq!(matches_codepattern("a   b","a\u{2002}b"), false);
+    assert_eq!(matches_codepattern("\u{205F}a   b","ab"), false);
+    assert_eq!(matches_codepattern("a  \u{3000}b","ab"), false);
+}
diff --git a/src/libsyntax_expand/proc_macro.rs b/src/libsyntax_expand/proc_macro.rs
index bda9478ce96..db981fae5c2 100644
--- a/src/libsyntax_expand/proc_macro.rs
+++ b/src/libsyntax_expand/proc_macro.rs
@@ -4,8 +4,8 @@ use crate::proc_macro_server;
 use syntax::ast::{self, ItemKind, Attribute, Mac};
 use syntax::attr::{mark_used, mark_known};
 use syntax::errors::{Applicability, FatalError};
-use syntax::parse::{self, token};
 use syntax::symbol::sym;
+use syntax::token;
 use syntax::tokenstream::{self, TokenStream};
 use syntax::visit::Visitor;
 
@@ -134,7 +134,11 @@ impl MultiItemModifier for ProcMacroDerive {
         let error_count_before = ecx.parse_sess.span_diagnostic.err_count();
         let msg = "proc-macro derive produced unparseable tokens";
 
-        let mut parser = parse::stream_to_parser(ecx.parse_sess, stream, Some("proc-macro derive"));
+        let mut parser = rustc_parse::stream_to_parser(
+            ecx.parse_sess,
+            stream,
+            Some("proc-macro derive"),
+        );
         let mut items = vec![];
 
         loop {
@@ -181,7 +185,7 @@ impl<'a> Visitor<'a> for MarkAttrs<'a> {
 crate fn collect_derives(cx: &mut ExtCtxt<'_>, attrs: &mut Vec<ast::Attribute>) -> Vec<ast::Path> {
     let mut result = Vec::new();
     attrs.retain(|attr| {
-        if attr.path != sym::derive {
+        if !attr.has_name(sym::derive) {
             return true;
         }
         if !attr.is_meta_item_list() {
@@ -196,10 +200,10 @@ crate fn collect_derives(cx: &mut ExtCtxt<'_>, attrs: &mut Vec<ast::Attribute>)
         }
 
         let parse_derive_paths = |attr: &ast::Attribute| {
-            if attr.tokens.is_empty() {
+            if attr.get_normal_item().tokens.is_empty() {
                 return Ok(Vec::new());
             }
-            parse::parse_in_attr(cx.parse_sess, attr, |p| p.parse_derive_paths())
+            rustc_parse::parse_in_attr(cx.parse_sess, attr, |p| p.parse_derive_paths())
         };
 
         match parse_derive_paths(attr) {
diff --git a/src/libsyntax_expand/proc_macro_server.rs b/src/libsyntax_expand/proc_macro_server.rs
index 4ce99cfe73b..43a9d4169f6 100644
--- a/src/libsyntax_expand/proc_macro_server.rs
+++ b/src/libsyntax_expand/proc_macro_server.rs
@@ -1,16 +1,17 @@
 use crate::base::ExtCtxt;
 
+use rustc_parse::{parse_stream_from_source_str, nt_to_tokenstream};
 use syntax::ast;
-use syntax::parse::{self, token};
-use syntax::parse::lexer::comments;
+use syntax::util::comments;
 use syntax::print::pprust;
 use syntax::sess::ParseSess;
+use syntax::token;
 use syntax::tokenstream::{self, DelimSpan, IsJoint::*, TokenStream, TreeAndJoint};
+use syntax_pos::{BytePos, FileName, MultiSpan, Pos, SourceFile, Span};
+use syntax_pos::symbol::{kw, sym, Symbol};
 
 use errors::Diagnostic;
 use rustc_data_structures::sync::Lrc;
-use syntax_pos::{BytePos, FileName, MultiSpan, Pos, SourceFile, Span};
-use syntax_pos::symbol::{kw, sym, Symbol};
 
 use pm::{Delimiter, Level, LineColumn, Spacing};
 use pm::bridge::{server, TokenTree};
@@ -52,7 +53,7 @@ impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec<Self>)>
 {
     fn from_internal(((tree, is_joint), sess, stack): (TreeAndJoint, &ParseSess, &mut Vec<Self>))
                     -> Self {
-        use syntax::parse::token::*;
+        use syntax::token::*;
 
         let joint = is_joint == Joint;
         let Token { kind, span } = match tree {
@@ -177,7 +178,7 @@ impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec<Self>)>
             }
 
             Interpolated(nt) => {
-                let stream = parse::nt_to_tokenstream(&nt, sess, span);
+                let stream = nt_to_tokenstream(&nt, sess, span);
                 TokenTree::Group(Group {
                     delimiter: Delimiter::None,
                     stream,
@@ -193,7 +194,7 @@ impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec<Self>)>
 
 impl ToInternal<TokenStream> for TokenTree<Group, Punct, Ident, Literal> {
     fn to_internal(self) -> TokenStream {
-        use syntax::parse::token::*;
+        use syntax::token::*;
 
         let (ch, joint, span) = match self {
             TokenTree::Punct(Punct { ch, joint, span }) => (ch, joint, span),
@@ -401,7 +402,7 @@ impl server::TokenStream for Rustc<'_> {
         stream.is_empty()
     }
     fn from_str(&mut self, src: &str) -> Self::TokenStream {
-        parse::parse_stream_from_source_str(
+        parse_stream_from_source_str(
             FileName::proc_macro_source_code(src),
             src.to_string(),
             self.sess,
diff --git a/src/libsyntax_expand/tests.rs b/src/libsyntax_expand/tests.rs
new file mode 100644
index 00000000000..2683620e845
--- /dev/null
+++ b/src/libsyntax_expand/tests.rs
@@ -0,0 +1,1254 @@
+use crate::config::process_configure_mod;
+use rustc_parse::{source_file_to_stream, new_parser_from_source_str, parser::Parser};
+use syntax::ast;
+use syntax::tokenstream::TokenStream;
+use syntax::sess::ParseSess;
+use syntax::source_map::{SourceMap, FilePathMapping};
+use syntax::with_default_globals;
+use syntax_pos::{BytePos, Span, MultiSpan};
+
+use errors::emitter::EmitterWriter;
+use errors::{PResult, Handler};
+use rustc_data_structures::sync::Lrc;
+
+use std::io;
+use std::io::prelude::*;
+use std::iter::Peekable;
+use std::path::{Path, PathBuf};
+use std::str;
+use std::sync::{Arc, Mutex};
+
+/// Map string to parser (via tts).
+fn string_to_parser(ps: &ParseSess, source_str: String) -> Parser<'_> {
+    new_parser_from_source_str(ps, PathBuf::from("bogofile").into(), source_str)
+}
+
+crate fn with_error_checking_parse<'a, T, F>(s: String, ps: &'a ParseSess, f: F) -> T where
+    F: FnOnce(&mut Parser<'a>) -> PResult<'a, T>,
+{
+    let mut p = string_to_parser(&ps, s);
+    let x = f(&mut p).unwrap();
+    p.sess.span_diagnostic.abort_if_errors();
+    x
+}
+
+/// Maps a string to tts, using a made-up filename.
+crate fn string_to_stream(source_str: String) -> TokenStream {
+    let ps = ParseSess::new(FilePathMapping::empty(), process_configure_mod);
+    source_file_to_stream(
+        &ps,
+        ps.source_map().new_source_file(PathBuf::from("bogofile").into(),
+        source_str,
+    ), None).0
+}
+
+/// Parses a string, returns a crate.
+crate fn string_to_crate(source_str : String) -> ast::Crate {
+    let ps = ParseSess::new(FilePathMapping::empty(), process_configure_mod);
+    with_error_checking_parse(source_str, &ps, |p| {
+        p.parse_crate_mod()
+    })
+}
+
+/// Does the given string match the pattern? whitespace in the first string
+/// may be deleted or replaced with other whitespace to match the pattern.
+/// This function is relatively Unicode-ignorant; fortunately, the careful design
+/// of UTF-8 mitigates this ignorance. It doesn't do NKF-normalization(?).
+crate fn matches_codepattern(a : &str, b : &str) -> bool {
+    let mut a_iter = a.chars().peekable();
+    let mut b_iter = b.chars().peekable();
+
+    loop {
+        let (a, b) = match (a_iter.peek(), b_iter.peek()) {
+            (None, None) => return true,
+            (None, _) => return false,
+            (Some(&a), None) => {
+                if rustc_lexer::is_whitespace(a) {
+                    break // Trailing whitespace check is out of loop for borrowck.
+                } else {
+                    return false
+                }
+            }
+            (Some(&a), Some(&b)) => (a, b)
+        };
+
+        if rustc_lexer::is_whitespace(a) && rustc_lexer::is_whitespace(b) {
+            // Skip whitespace for `a` and `b`.
+            scan_for_non_ws_or_end(&mut a_iter);
+            scan_for_non_ws_or_end(&mut b_iter);
+        } else if rustc_lexer::is_whitespace(a) {
+            // Skip whitespace for `a`.
+            scan_for_non_ws_or_end(&mut a_iter);
+        } else if a == b {
+            a_iter.next();
+            b_iter.next();
+        } else {
+            return false
+        }
+    }
+
+    // Check if a has *only* trailing whitespace.
+    a_iter.all(rustc_lexer::is_whitespace)
+}
+
+/// Advances the given peekable `Iterator` until it reaches a non-whitespace character.
+fn scan_for_non_ws_or_end<I: Iterator<Item = char>>(iter: &mut Peekable<I>) {
+    while iter.peek().copied().map(|c| rustc_lexer::is_whitespace(c)) == Some(true) {
+        iter.next();
+    }
+}
+
+/// Identifies a position in the text by the n'th occurrence of a string.
+struct Position {
+    string: &'static str,
+    count: usize,
+}
+
+struct SpanLabel {
+    start: Position,
+    end: Position,
+    label: &'static str,
+}
+
+crate struct Shared<T: Write> {
+    pub data: Arc<Mutex<T>>,
+}
+
+impl<T: Write> Write for Shared<T> {
+    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
+        self.data.lock().unwrap().write(buf)
+    }
+
+    fn flush(&mut self) -> io::Result<()> {
+        self.data.lock().unwrap().flush()
+    }
+}
+
+fn test_harness(file_text: &str, span_labels: Vec<SpanLabel>, expected_output: &str) {
+    with_default_globals(|| {
+        let output = Arc::new(Mutex::new(Vec::new()));
+
+        let source_map = Lrc::new(SourceMap::new(FilePathMapping::empty()));
+        source_map.new_source_file(Path::new("test.rs").to_owned().into(), file_text.to_owned());
+
+        let primary_span = make_span(&file_text, &span_labels[0].start, &span_labels[0].end);
+        let mut msp = MultiSpan::from_span(primary_span);
+        for span_label in span_labels {
+            let span = make_span(&file_text, &span_label.start, &span_label.end);
+            msp.push_span_label(span, span_label.label.to_string());
+            println!("span: {:?} label: {:?}", span, span_label.label);
+            println!("text: {:?}", source_map.span_to_snippet(span));
+        }
+
+        let emitter = EmitterWriter::new(
+            Box::new(Shared { data: output.clone() }),
+            Some(source_map.clone()),
+            false,
+            false,
+            false,
+            None,
+            false,
+        );
+        let handler = Handler::with_emitter(true, None, Box::new(emitter));
+        handler.span_err(msp, "foo");
+
+        assert!(expected_output.chars().next() == Some('\n'),
+                "expected output should begin with newline");
+        let expected_output = &expected_output[1..];
+
+        let bytes = output.lock().unwrap();
+        let actual_output = str::from_utf8(&bytes).unwrap();
+        println!("expected output:\n------\n{}------", expected_output);
+        println!("actual output:\n------\n{}------", actual_output);
+
+        assert!(expected_output == actual_output)
+    })
+}
+
+fn make_span(file_text: &str, start: &Position, end: &Position) -> Span {
+    let start = make_pos(file_text, start);
+    let end = make_pos(file_text, end) + end.string.len(); // just after matching thing ends
+    assert!(start <= end);
+    Span::with_root_ctxt(BytePos(start as u32), BytePos(end as u32))
+}
+
+fn make_pos(file_text: &str, pos: &Position) -> usize {
+    let mut remainder = file_text;
+    let mut offset = 0;
+    for _ in 0..pos.count {
+        if let Some(n) = remainder.find(&pos.string) {
+            offset += n;
+            remainder = &remainder[n + 1..];
+        } else {
+            panic!("failed to find {} instances of {:?} in {:?}",
+                   pos.count,
+                   pos.string,
+                   file_text);
+        }
+    }
+    offset
+}
+
+#[test]
+fn ends_on_col0() {
+    test_harness(r#"
+fn foo() {
+}
+"#,
+    vec![
+        SpanLabel {
+           start: Position {
+               string: "{",
+               count: 1,
+           },
+           end: Position {
+               string: "}",
+               count: 1,
+           },
+           label: "test",
+       },
+    ],
+    r#"
+error: foo
+ --> test.rs:2:10
+  |
+2 |   fn foo() {
+  |  __________^
+3 | | }
+  | |_^ test
+
+"#);
+}
+
+#[test]
+fn ends_on_col2() {
+    test_harness(r#"
+fn foo() {
+
+
+  }
+"#,
+     vec![
+        SpanLabel {
+            start: Position {
+                string: "{",
+                count: 1,
+            },
+            end: Position {
+                string: "}",
+                count: 1,
+            },
+            label: "test",
+        },
+     ],
+     r#"
+error: foo
+ --> test.rs:2:10
+  |
+2 |   fn foo() {
+  |  __________^
+3 | |
+4 | |
+5 | |   }
+  | |___^ test
+
+"#);
+}
+#[test]
+fn non_nested() {
+    test_harness(r#"
+fn foo() {
+  X0 Y0
+  X1 Y1
+  X2 Y2
+}
+"#,
+    vec![
+        SpanLabel {
+            start: Position {
+                string: "X0",
+                count: 1,
+            },
+            end: Position {
+                string: "X2",
+                count: 1,
+            },
+            label: "`X` is a good letter",
+        },
+        SpanLabel {
+            start: Position {
+                string: "Y0",
+                count: 1,
+            },
+            end: Position {
+                string: "Y2",
+                count: 1,
+            },
+            label: "`Y` is a good letter too",
+        },
+    ],
+    r#"
+error: foo
+ --> test.rs:3:3
+  |
+3 |      X0 Y0
+  |  ____^__-
+  | | ___|
+  | ||
+4 | ||   X1 Y1
+5 | ||   X2 Y2
+  | ||____^__- `Y` is a good letter too
+  |  |____|
+  |       `X` is a good letter
+
+"#);
+}
+
+#[test]
+fn nested() {
+    test_harness(r#"
+fn foo() {
+  X0 Y0
+  Y1 X1
+}
+"#,
+    vec![
+        SpanLabel {
+            start: Position {
+                string: "X0",
+                count: 1,
+            },
+            end: Position {
+                string: "X1",
+                count: 1,
+            },
+            label: "`X` is a good letter",
+        },
+        SpanLabel {
+            start: Position {
+                string: "Y0",
+                count: 1,
+            },
+            end: Position {
+                string: "Y1",
+                count: 1,
+            },
+            label: "`Y` is a good letter too",
+        },
+    ],
+r#"
+error: foo
+ --> test.rs:3:3
+  |
+3 |      X0 Y0
+  |  ____^__-
+  | | ___|
+  | ||
+4 | ||   Y1 X1
+  | ||____-__^ `X` is a good letter
+  | |_____|
+  |       `Y` is a good letter too
+
+"#);
+}
+
+#[test]
+fn different_overlap() {
+    test_harness(r#"
+fn foo() {
+  X0 Y0 Z0
+  X1 Y1 Z1
+  X2 Y2 Z2
+  X3 Y3 Z3
+}
+"#,
+    vec![
+        SpanLabel {
+            start: Position {
+                string: "Y0",
+                count: 1,
+            },
+            end: Position {
+                string: "X2",
+                count: 1,
+            },
+            label: "`X` is a good letter",
+        },
+        SpanLabel {
+            start: Position {
+                string: "Z1",
+                count: 1,
+            },
+            end: Position {
+                string: "X3",
+                count: 1,
+            },
+            label: "`Y` is a good letter too",
+        },
+    ],
+    r#"
+error: foo
+ --> test.rs:3:6
+  |
+3 |      X0 Y0 Z0
+  |   ______^
+4 |  |   X1 Y1 Z1
+  |  |_________-
+5 | ||   X2 Y2 Z2
+  | ||____^ `X` is a good letter
+6 | |    X3 Y3 Z3
+  | |_____- `Y` is a good letter too
+
+"#);
+}
+
+#[test]
+fn triple_overlap() {
+    test_harness(r#"
+fn foo() {
+  X0 Y0 Z0
+  X1 Y1 Z1
+  X2 Y2 Z2
+}
+"#,
+    vec![
+        SpanLabel {
+            start: Position {
+                string: "X0",
+                count: 1,
+            },
+            end: Position {
+                string: "X2",
+                count: 1,
+            },
+            label: "`X` is a good letter",
+        },
+        SpanLabel {
+            start: Position {
+                string: "Y0",
+                count: 1,
+            },
+            end: Position {
+                string: "Y2",
+                count: 1,
+            },
+            label: "`Y` is a good letter too",
+        },
+        SpanLabel {
+            start: Position {
+                string: "Z0",
+                count: 1,
+            },
+            end: Position {
+                string: "Z2",
+                count: 1,
+            },
+            label: "`Z` label",
+        },
+    ],
+    r#"
+error: foo
+ --> test.rs:3:3
+  |
+3 |       X0 Y0 Z0
+  |  _____^__-__-
+  | | ____|__|
+  | || ___|
+  | |||
+4 | |||   X1 Y1 Z1
+5 | |||   X2 Y2 Z2
+  | |||____^__-__- `Z` label
+  |  ||____|__|
+  |   |____|  `Y` is a good letter too
+  |        `X` is a good letter
+
+"#);
+}
+
+#[test]
+fn triple_exact_overlap() {
+    test_harness(r#"
+fn foo() {
+  X0 Y0 Z0
+  X1 Y1 Z1
+  X2 Y2 Z2
+}
+"#,
+    vec![
+        SpanLabel {
+            start: Position {
+                string: "X0",
+                count: 1,
+            },
+            end: Position {
+                string: "X2",
+                count: 1,
+            },
+            label: "`X` is a good letter",
+        },
+        SpanLabel {
+            start: Position {
+                string: "X0",
+                count: 1,
+            },
+            end: Position {
+                string: "X2",
+                count: 1,
+            },
+            label: "`Y` is a good letter too",
+        },
+        SpanLabel {
+            start: Position {
+                string: "X0",
+                count: 1,
+            },
+            end: Position {
+                string: "X2",
+                count: 1,
+            },
+            label: "`Z` label",
+        },
+    ],
+    r#"
+error: foo
+ --> test.rs:3:3
+  |
+3 | /   X0 Y0 Z0
+4 | |   X1 Y1 Z1
+5 | |   X2 Y2 Z2
+  | |    ^
+  | |    |
+  | |    `X` is a good letter
+  | |____`Y` is a good letter too
+  |      `Z` label
+
+"#);
+}
+
+#[test]
+fn minimum_depth() {
+    test_harness(r#"
+fn foo() {
+  X0 Y0 Z0
+  X1 Y1 Z1
+  X2 Y2 Z2
+  X3 Y3 Z3
+}
+"#,
+    vec![
+        SpanLabel {
+            start: Position {
+                string: "Y0",
+                count: 1,
+            },
+            end: Position {
+                string: "X1",
+                count: 1,
+            },
+            label: "`X` is a good letter",
+        },
+        SpanLabel {
+            start: Position {
+                string: "Y1",
+                count: 1,
+            },
+            end: Position {
+                string: "Z2",
+                count: 1,
+            },
+            label: "`Y` is a good letter too",
+        },
+        SpanLabel {
+            start: Position {
+                string: "X2",
+                count: 1,
+            },
+            end: Position {
+                string: "Y3",
+                count: 1,
+            },
+            label: "`Z`",
+        },
+    ],
+    r#"
+error: foo
+ --> test.rs:3:6
+  |
+3 |      X0 Y0 Z0
+  |   ______^
+4 |  |   X1 Y1 Z1
+  |  |____^_-
+  | ||____|
+  | |     `X` is a good letter
+5 | |    X2 Y2 Z2
+  | |____-______- `Y` is a good letter too
+  |  ____|
+  | |
+6 | |    X3 Y3 Z3
+  | |________- `Z`
+
+"#);
+}
+
+#[test]
+fn non_overlaping() {
+    test_harness(r#"
+fn foo() {
+  X0 Y0 Z0
+  X1 Y1 Z1
+  X2 Y2 Z2
+  X3 Y3 Z3
+}
+"#,
+    vec![
+        SpanLabel {
+            start: Position {
+                string: "X0",
+                count: 1,
+            },
+            end: Position {
+                string: "X1",
+                count: 1,
+            },
+            label: "`X` is a good letter",
+        },
+        SpanLabel {
+            start: Position {
+                string: "Y2",
+                count: 1,
+            },
+            end: Position {
+                string: "Z3",
+                count: 1,
+            },
+            label: "`Y` is a good letter too",
+        },
+    ],
+    r#"
+error: foo
+ --> test.rs:3:3
+  |
+3 | /   X0 Y0 Z0
+4 | |   X1 Y1 Z1
+  | |____^ `X` is a good letter
+5 |     X2 Y2 Z2
+  |  ______-
+6 | |   X3 Y3 Z3
+  | |__________- `Y` is a good letter too
+
+"#);
+}
+
+#[test]
+fn overlaping_start_and_end() {
+    test_harness(r#"
+fn foo() {
+  X0 Y0 Z0
+  X1 Y1 Z1
+  X2 Y2 Z2
+  X3 Y3 Z3
+}
+"#,
+    vec![
+        SpanLabel {
+            start: Position {
+                string: "Y0",
+                count: 1,
+            },
+            end: Position {
+                string: "X1",
+                count: 1,
+            },
+            label: "`X` is a good letter",
+        },
+        SpanLabel {
+            start: Position {
+                string: "Z1",
+                count: 1,
+            },
+            end: Position {
+                string: "Z3",
+                count: 1,
+            },
+            label: "`Y` is a good letter too",
+        },
+    ],
+    r#"
+error: foo
+ --> test.rs:3:6
+  |
+3 |      X0 Y0 Z0
+  |   ______^
+4 |  |   X1 Y1 Z1
+  |  |____^____-
+  | ||____|
+  | |     `X` is a good letter
+5 | |    X2 Y2 Z2
+6 | |    X3 Y3 Z3
+  | |___________- `Y` is a good letter too
+
+"#);
+}
+
+#[test]
+fn multiple_labels_primary_without_message() {
+    test_harness(r#"
+fn foo() {
+  a { b { c } d }
+}
+"#,
+    vec![
+        SpanLabel {
+            start: Position {
+                string: "b",
+                count: 1,
+            },
+            end: Position {
+                string: "}",
+                count: 1,
+            },
+            label: "",
+        },
+        SpanLabel {
+            start: Position {
+                string: "a",
+                count: 1,
+            },
+            end: Position {
+                string: "d",
+                count: 1,
+            },
+            label: "`a` is a good letter",
+        },
+        SpanLabel {
+            start: Position {
+                string: "c",
+                count: 1,
+            },
+            end: Position {
+                string: "c",
+                count: 1,
+            },
+            label: "",
+        },
+    ],
+    r#"
+error: foo
+ --> test.rs:3:7
+  |
+3 |   a { b { c } d }
+  |   ----^^^^-^^-- `a` is a good letter
+
+"#);
+}
+
+#[test]
+fn multiple_labels_secondary_without_message() {
+    test_harness(r#"
+fn foo() {
+  a { b { c } d }
+}
+"#,
+    vec![
+        SpanLabel {
+            start: Position {
+                string: "a",
+                count: 1,
+            },
+            end: Position {
+                string: "d",
+                count: 1,
+            },
+            label: "`a` is a good letter",
+        },
+        SpanLabel {
+            start: Position {
+                string: "b",
+                count: 1,
+            },
+            end: Position {
+                string: "}",
+                count: 1,
+            },
+            label: "",
+        },
+    ],
+    r#"
+error: foo
+ --> test.rs:3:3
+  |
+3 |   a { b { c } d }
+  |   ^^^^-------^^ `a` is a good letter
+
+"#);
+}
+
+#[test]
+fn multiple_labels_primary_without_message_2() {
+    test_harness(r#"
+fn foo() {
+  a { b { c } d }
+}
+"#,
+    vec![
+        SpanLabel {
+            start: Position {
+                string: "b",
+                count: 1,
+            },
+            end: Position {
+                string: "}",
+                count: 1,
+            },
+            label: "`b` is a good letter",
+        },
+        SpanLabel {
+            start: Position {
+                string: "a",
+                count: 1,
+            },
+            end: Position {
+                string: "d",
+                count: 1,
+            },
+            label: "",
+        },
+        SpanLabel {
+            start: Position {
+                string: "c",
+                count: 1,
+            },
+            end: Position {
+                string: "c",
+                count: 1,
+            },
+            label: "",
+        },
+    ],
+    r#"
+error: foo
+ --> test.rs:3:7
+  |
+3 |   a { b { c } d }
+  |   ----^^^^-^^--
+  |       |
+  |       `b` is a good letter
+
+"#);
+}
+
+#[test]
+fn multiple_labels_secondary_without_message_2() {
+    test_harness(r#"
+fn foo() {
+  a { b { c } d }
+}
+"#,
+    vec![
+        SpanLabel {
+            start: Position {
+                string: "a",
+                count: 1,
+            },
+            end: Position {
+                string: "d",
+                count: 1,
+            },
+            label: "",
+        },
+        SpanLabel {
+            start: Position {
+                string: "b",
+                count: 1,
+            },
+            end: Position {
+                string: "}",
+                count: 1,
+            },
+            label: "`b` is a good letter",
+        },
+    ],
+    r#"
+error: foo
+ --> test.rs:3:3
+  |
+3 |   a { b { c } d }
+  |   ^^^^-------^^
+  |       |
+  |       `b` is a good letter
+
+"#);
+}
+
+#[test]
+fn multiple_labels_secondary_without_message_3() {
+    test_harness(r#"
+fn foo() {
+  a  bc  d
+}
+"#,
+    vec![
+        SpanLabel {
+            start: Position {
+                string: "a",
+                count: 1,
+            },
+            end: Position {
+                string: "b",
+                count: 1,
+            },
+            label: "`a` is a good letter",
+        },
+        SpanLabel {
+            start: Position {
+                string: "c",
+                count: 1,
+            },
+            end: Position {
+                string: "d",
+                count: 1,
+            },
+            label: "",
+        },
+    ],
+    r#"
+error: foo
+ --> test.rs:3:3
+  |
+3 |   a  bc  d
+  |   ^^^^----
+  |   |
+  |   `a` is a good letter
+
+"#);
+}
+
+#[test]
+fn multiple_labels_without_message() {
+    test_harness(r#"
+fn foo() {
+  a { b { c } d }
+}
+"#,
+    vec![
+        SpanLabel {
+            start: Position {
+                string: "a",
+                count: 1,
+            },
+            end: Position {
+                string: "d",
+                count: 1,
+            },
+            label: "",
+        },
+        SpanLabel {
+            start: Position {
+                string: "b",
+                count: 1,
+            },
+            end: Position {
+                string: "}",
+                count: 1,
+            },
+            label: "",
+        },
+    ],
+    r#"
+error: foo
+ --> test.rs:3:3
+  |
+3 |   a { b { c } d }
+  |   ^^^^-------^^
+
+"#);
+}
+
+#[test]
+fn multiple_labels_without_message_2() {
+    test_harness(r#"
+fn foo() {
+  a { b { c } d }
+}
+"#,
+    vec![
+        SpanLabel {
+            start: Position {
+                string: "b",
+                count: 1,
+            },
+            end: Position {
+                string: "}",
+                count: 1,
+            },
+            label: "",
+        },
+        SpanLabel {
+            start: Position {
+                string: "a",
+                count: 1,
+            },
+            end: Position {
+                string: "d",
+                count: 1,
+            },
+            label: "",
+        },
+        SpanLabel {
+            start: Position {
+                string: "c",
+                count: 1,
+            },
+            end: Position {
+                string: "c",
+                count: 1,
+            },
+            label: "",
+        },
+    ],
+    r#"
+error: foo
+ --> test.rs:3:7
+  |
+3 |   a { b { c } d }
+  |   ----^^^^-^^--
+
+"#);
+}
+
+#[test]
+fn multiple_labels_with_message() {
+    test_harness(r#"
+fn foo() {
+  a { b { c } d }
+}
+"#,
+    vec![
+        SpanLabel {
+            start: Position {
+                string: "a",
+                count: 1,
+            },
+            end: Position {
+                string: "d",
+                count: 1,
+            },
+            label: "`a` is a good letter",
+        },
+        SpanLabel {
+            start: Position {
+                string: "b",
+                count: 1,
+            },
+            end: Position {
+                string: "}",
+                count: 1,
+            },
+            label: "`b` is a good letter",
+        },
+    ],
+    r#"
+error: foo
+ --> test.rs:3:3
+  |
+3 |   a { b { c } d }
+  |   ^^^^-------^^
+  |   |   |
+  |   |   `b` is a good letter
+  |   `a` is a good letter
+
+"#);
+}
+
+#[test]
+fn single_label_with_message() {
+    test_harness(r#"
+fn foo() {
+  a { b { c } d }
+}
+"#,
+    vec![
+        SpanLabel {
+            start: Position {
+                string: "a",
+                count: 1,
+            },
+            end: Position {
+                string: "d",
+                count: 1,
+            },
+            label: "`a` is a good letter",
+        },
+    ],
+    r#"
+error: foo
+ --> test.rs:3:3
+  |
+3 |   a { b { c } d }
+  |   ^^^^^^^^^^^^^ `a` is a good letter
+
+"#);
+}
+
+#[test]
+fn single_label_without_message() {
+    test_harness(r#"
+fn foo() {
+  a { b { c } d }
+}
+"#,
+    vec![
+        SpanLabel {
+            start: Position {
+                string: "a",
+                count: 1,
+            },
+            end: Position {
+                string: "d",
+                count: 1,
+            },
+            label: "",
+        },
+    ],
+    r#"
+error: foo
+ --> test.rs:3:3
+  |
+3 |   a { b { c } d }
+  |   ^^^^^^^^^^^^^
+
+"#);
+}
+
+#[test]
+fn long_snippet() {
+    test_harness(r#"
+fn foo() {
+  X0 Y0 Z0
+  X1 Y1 Z1
+1
+2
+3
+4
+5
+6
+7
+8
+9
+10
+  X2 Y2 Z2
+  X3 Y3 Z3
+}
+"#,
+    vec![
+        SpanLabel {
+            start: Position {
+                string: "Y0",
+                count: 1,
+            },
+            end: Position {
+                string: "X1",
+                count: 1,
+            },
+            label: "`X` is a good letter",
+        },
+        SpanLabel {
+            start: Position {
+                string: "Z1",
+                count: 1,
+            },
+            end: Position {
+                string: "Z3",
+                count: 1,
+            },
+            label: "`Y` is a good letter too",
+        },
+    ],
+    r#"
+error: foo
+  --> test.rs:3:6
+   |
+3  |      X0 Y0 Z0
+   |   ______^
+4  |  |   X1 Y1 Z1
+   |  |____^____-
+   | ||____|
+   | |     `X` is a good letter
+5  | |  1
+6  | |  2
+7  | |  3
+...  |
+15 | |    X2 Y2 Z2
+16 | |    X3 Y3 Z3
+   | |___________- `Y` is a good letter too
+
+"#);
+}
+
+#[test]
+fn long_snippet_multiple_spans() {
+    test_harness(r#"
+fn foo() {
+  X0 Y0 Z0
+1
+2
+3
+  X1 Y1 Z1
+4
+5
+6
+  X2 Y2 Z2
+7
+8
+9
+10
+  X3 Y3 Z3
+}
+"#,
+    vec![
+        SpanLabel {
+            start: Position {
+                string: "Y0",
+                count: 1,
+            },
+            end: Position {
+                string: "Y3",
+                count: 1,
+            },
+            label: "`Y` is a good letter",
+        },
+        SpanLabel {
+            start: Position {
+                string: "Z1",
+                count: 1,
+            },
+            end: Position {
+                string: "Z2",
+                count: 1,
+            },
+            label: "`Z` is a good letter too",
+        },
+    ],
+    r#"
+error: foo
+  --> test.rs:3:6
+   |
+3  |      X0 Y0 Z0
+   |   ______^
+4  |  | 1
+5  |  | 2
+6  |  | 3
+7  |  |   X1 Y1 Z1
+   |  |_________-
+8  | || 4
+9  | || 5
+10 | || 6
+11 | ||   X2 Y2 Z2
+   | ||__________- `Z` is a good letter too
+...   |
+15 |  | 10
+16 |  |   X3 Y3 Z3
+   |  |_______^ `Y` is a good letter
+
+"#);
+}
diff --git a/src/libsyntax_expand/tokenstream/tests.rs b/src/libsyntax_expand/tokenstream/tests.rs
new file mode 100644
index 00000000000..cf9fead638e
--- /dev/null
+++ b/src/libsyntax_expand/tokenstream/tests.rs
@@ -0,0 +1,110 @@
+use crate::tests::string_to_stream;
+
+use syntax::ast::Name;
+use syntax::token;
+use syntax::tokenstream::{TokenStream, TokenStreamBuilder, TokenTree};
+use syntax::with_default_globals;
+use syntax_pos::{Span, BytePos};
+use smallvec::smallvec;
+
+fn string_to_ts(string: &str) -> TokenStream {
+    string_to_stream(string.to_owned())
+}
+
+fn sp(a: u32, b: u32) -> Span {
+    Span::with_root_ctxt(BytePos(a), BytePos(b))
+}
+
+#[test]
+fn test_concat() {
+    with_default_globals(|| {
+        let test_res = string_to_ts("foo::bar::baz");
+        let test_fst = string_to_ts("foo::bar");
+        let test_snd = string_to_ts("::baz");
+        let eq_res = TokenStream::from_streams(smallvec![test_fst, test_snd]);
+        assert_eq!(test_res.trees().count(), 5);
+        assert_eq!(eq_res.trees().count(), 5);
+        assert_eq!(test_res.eq_unspanned(&eq_res), true);
+    })
+}
+
+#[test]
+fn test_to_from_bijection() {
+    with_default_globals(|| {
+        let test_start = string_to_ts("foo::bar(baz)");
+        let test_end = test_start.trees().collect();
+        assert_eq!(test_start, test_end)
+    })
+}
+
+#[test]
+fn test_eq_0() {
+    with_default_globals(|| {
+        let test_res = string_to_ts("foo");
+        let test_eqs = string_to_ts("foo");
+        assert_eq!(test_res, test_eqs)
+    })
+}
+
+#[test]
+fn test_eq_1() {
+    with_default_globals(|| {
+        let test_res = string_to_ts("::bar::baz");
+        let test_eqs = string_to_ts("::bar::baz");
+        assert_eq!(test_res, test_eqs)
+    })
+}
+
+#[test]
+fn test_eq_3() {
+    with_default_globals(|| {
+        let test_res = string_to_ts("");
+        let test_eqs = string_to_ts("");
+        assert_eq!(test_res, test_eqs)
+    })
+}
+
+#[test]
+fn test_diseq_0() {
+    with_default_globals(|| {
+        let test_res = string_to_ts("::bar::baz");
+        let test_eqs = string_to_ts("bar::baz");
+        assert_eq!(test_res == test_eqs, false)
+    })
+}
+
+#[test]
+fn test_diseq_1() {
+    with_default_globals(|| {
+        let test_res = string_to_ts("(bar,baz)");
+        let test_eqs = string_to_ts("bar,baz");
+        assert_eq!(test_res == test_eqs, false)
+    })
+}
+
+#[test]
+fn test_is_empty() {
+    with_default_globals(|| {
+        let test0: TokenStream = Vec::<TokenTree>::new().into_iter().collect();
+        let test1: TokenStream =
+            TokenTree::token(token::Ident(Name::intern("a"), false), sp(0, 1)).into();
+        let test2 = string_to_ts("foo(bar::baz)");
+
+        assert_eq!(test0.is_empty(), true);
+        assert_eq!(test1.is_empty(), false);
+        assert_eq!(test2.is_empty(), false);
+    })
+}
+
+#[test]
+fn test_dotdotdot() {
+    with_default_globals(|| {
+        let mut builder = TokenStreamBuilder::new();
+        builder.push(TokenTree::token(token::Dot, sp(0, 1)).joint());
+        builder.push(TokenTree::token(token::Dot, sp(1, 2)).joint());
+        builder.push(TokenTree::token(token::Dot, sp(2, 3)));
+        let stream = builder.build();
+        assert!(stream.eq_unspanned(&string_to_ts("...")));
+        assert_eq!(stream.trees().count(), 1);
+    })
+}