Merge branch 'master' into copied

9 files changed, 159 insertions, 148 deletions

diff --git a/src/libsyntax/ext/base.rs b/src/libsyntax/ext/base.rs
index bb927b62a18..b807a65f6ae 100644
--- a/src/libsyntax/ext/base.rs
+++ b/src/libsyntax/ext/base.rs
@@ -670,7 +670,7 @@ pub enum SyntaxExtension {
     /// An attribute-like procedural macro that derives a builtin trait.
     BuiltinDerive(BuiltinDeriveFn),
 
-    /// A declarative macro, e.g. `macro m() {}`.
+    /// A declarative macro, e.g., `macro m() {}`.
     DeclMacro {
         expander: Box<dyn TTMacroExpander + sync::Sync + sync::Send>,
         def_info: Option<(ast::NodeId, Span)>,
@@ -732,17 +732,12 @@ pub type NamedSyntaxExtension = (Name, SyntaxExtension);
 pub trait Resolver {
     fn next_node_id(&mut self) -> ast::NodeId;
     fn get_module_scope(&mut self, id: ast::NodeId) -> Mark;
-    fn eliminate_crate_var(&mut self, item: P<ast::Item>) -> P<ast::Item>;
-    fn is_whitelisted_legacy_custom_derive(&self, name: Name) -> bool;
 
     fn visit_ast_fragment_with_placeholders(&mut self, mark: Mark, fragment: &AstFragment,
                                             derives: &[Mark]);
     fn add_builtin(&mut self, ident: ast::Ident, ext: Lrc<SyntaxExtension>);
 
     fn resolve_imports(&mut self);
-    // Resolves attribute and derive legacy macros from `#![plugin(..)]`.
-    fn find_legacy_attr_invoc(&mut self, attrs: &mut Vec<Attribute>, allow_derive: bool)
-                              -> Option<Attribute>;
 
     fn resolve_macro_invocation(&mut self, invoc: &Invocation, invoc_id: Mark, force: bool)
                                 -> Result<Option<Lrc<SyntaxExtension>>, Determinacy>;
@@ -770,16 +765,12 @@ pub struct DummyResolver;
 impl Resolver for DummyResolver {
     fn next_node_id(&mut self) -> ast::NodeId { ast::DUMMY_NODE_ID }
     fn get_module_scope(&mut self, _id: ast::NodeId) -> Mark { Mark::root() }
-    fn eliminate_crate_var(&mut self, item: P<ast::Item>) -> P<ast::Item> { item }
-    fn is_whitelisted_legacy_custom_derive(&self, _name: Name) -> bool { false }
 
     fn visit_ast_fragment_with_placeholders(&mut self, _invoc: Mark, _fragment: &AstFragment,
                                             _derives: &[Mark]) {}
     fn add_builtin(&mut self, _ident: ast::Ident, _ext: Lrc<SyntaxExtension>) {}
 
     fn resolve_imports(&mut self) {}
-    fn find_legacy_attr_invoc(&mut self, _attrs: &mut Vec<Attribute>, _allow_derive: bool)
-                              -> Option<Attribute> { None }
     fn resolve_macro_invocation(&mut self, _invoc: &Invocation, _invoc_id: Mark, _force: bool)
                                 -> Result<Option<Lrc<SyntaxExtension>>, Determinacy> {
         Err(Determinacy::Determined)
@@ -915,7 +906,7 @@ impl<'a> ExtCtxt<'a> {
     /// `span_err` should be strongly preferred where-ever possible:
     /// this should *only* be used when:
     ///
-    /// - continuing has a high risk of flow-on errors (e.g. errors in
+    /// - continuing has a high risk of flow-on errors (e.g., errors in
     ///   declaring a macro would cause all uses of that macro to
     ///   complain about "undefined macro"), or
     /// - there is literally nothing else that can be done (however,
@@ -1002,7 +993,7 @@ pub fn expr_to_spanned_string<'a>(
         expr
     });
 
-    // we want to be able to handle e.g. `concat!("foo", "bar")`
+    // we want to be able to handle e.g., `concat!("foo", "bar")`
     let expr = cx.expander().fold_expr(expr);
     Err(match expr.node {
         ast::ExprKind::Lit(ref l) => match l.node {
diff --git a/src/libsyntax/ext/build.rs b/src/libsyntax/ext/build.rs
index cacec867cf1..5770f6bb8a2 100644
--- a/src/libsyntax/ext/build.rs
+++ b/src/libsyntax/ext/build.rs
@@ -318,9 +318,13 @@ impl<'a> AstBuilder for ExtCtxt<'a> {
                 args: Vec<ast::GenericArg>,
                 bindings: Vec<ast::TypeBinding> )
                 -> ast::Path {
+        assert!(!idents.is_empty());
+        let add_root = global && !idents[0].is_path_segment_keyword();
+        let mut segments = Vec::with_capacity(idents.len() + add_root as usize);
+        if add_root {
+            segments.push(ast::PathSegment::path_root(span));
+        }
         let last_ident = idents.pop().unwrap();
-        let mut segments: Vec<ast::PathSegment> = vec![];
-
         segments.extend(idents.into_iter().map(|ident| {
             ast::PathSegment::from_ident(ident.with_span_pos(span))
         }));
@@ -334,13 +338,7 @@ impl<'a> AstBuilder for ExtCtxt<'a> {
             id: ast::DUMMY_NODE_ID,
             args,
         });
-        let mut path = ast::Path { span, segments };
-        if global {
-            if let Some(seg) = path.make_root() {
-                path.segments.insert(0, seg);
-            }
-        }
-        path
+        ast::Path { span, segments }
     }
 
     /// Constructs a qualified path.
@@ -625,7 +623,7 @@ impl<'a> AstBuilder for ExtCtxt<'a> {
         self.expr_path(self.path_ident(span, id))
     }
     fn expr_self(&self, span: Span) -> P<ast::Expr> {
-        self.expr_ident(span, keywords::SelfValue.ident())
+        self.expr_ident(span, keywords::SelfLower.ident())
     }
 
     fn expr_binary(&self, sp: Span, op: ast::BinOpKind,
diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index 68a96293891..57ccc3e9817 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -8,7 +8,7 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use ast::{self, Block, Ident, NodeId, PatKind, Path};
+use ast::{self, Block, Ident, LitKind, NodeId, PatKind, Path};
 use ast::{MacStmtStyle, StmtKind, ItemKind};
 use attr::{self, HasAttrs};
 use source_map::{ExpnInfo, MacroBang, MacroAttribute, dummy_spanned, respan};
@@ -34,8 +34,8 @@ use tokenstream::{TokenStream, TokenTree};
 use visit::{self, Visitor};
 
 use rustc_data_structures::fx::FxHashMap;
-use std::fs::File;
-use std::io::Read;
+use std::fs;
+use std::io::ErrorKind;
 use std::{iter, mem};
 use std::rc::Rc;
 use std::path::PathBuf;
@@ -203,10 +203,7 @@ fn macro_bang_format(path: &ast::Path) -> ExpnFormat {
         if i != 0 {
             path_str.push_str("::");
         }
-
-        if segment.ident.name != keywords::CrateRoot.name() &&
-            segment.ident.name != keywords::DollarCrate.name()
-        {
+        if segment.ident.name != keywords::PathRoot.name() {
             path_str.push_str(&segment.ident.as_str())
         }
     }
@@ -622,9 +619,9 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
     fn extract_proc_macro_attr_input(&self, tokens: TokenStream, span: Span) -> TokenStream {
         let mut trees = tokens.trees();
         match trees.next() {
-            Some(TokenTree::Delimited(_, delim)) => {
+            Some(TokenTree::Delimited(_, _, tts)) => {
                 if trees.next().is_none() {
-                    return delim.tts.into()
+                    return tts.into()
                 }
             }
             Some(TokenTree::Token(..)) => {}
@@ -1134,12 +1131,6 @@ impl<'a, 'b> InvocationCollector<'a, 'b> {
         let (mut attr, mut traits, mut after_derive) = (None, Vec::new(), false);
 
         item = item.map_attrs(|mut attrs| {
-            if let Some(legacy_attr_invoc) = self.cx.resolver.find_legacy_attr_invoc(&mut attrs,
-                                                                                     true) {
-                attr = Some(legacy_attr_invoc);
-                return attrs;
-            }
-
             attr = self.find_attr_invoc(&mut attrs, &mut after_derive);
             traits = collect_derives(&mut self.cx, &mut attrs);
             attrs
@@ -1156,12 +1147,6 @@ impl<'a, 'b> InvocationCollector<'a, 'b> {
         let (mut attr, mut after_derive) = (None, false);
 
         item = item.map_attrs(|mut attrs| {
-            if let Some(legacy_attr_invoc) = self.cx.resolver.find_legacy_attr_invoc(&mut attrs,
-                                                                                     false) {
-                attr = Some(legacy_attr_invoc);
-                return attrs;
-            }
-
             attr = self.find_attr_invoc(&mut attrs, &mut after_derive);
             attrs
         });
@@ -1363,7 +1348,7 @@ impl<'a, 'b> Folder for InvocationCollector<'a, 'b> {
                 module.mod_path.push(item.ident);
 
                 // Detect if this is an inline module (`mod m { ... }` as opposed to `mod m;`).
-                // In the non-inline case, `inner` is never the dummy span (c.f. `parse_item_mod`).
+                // In the non-inline case, `inner` is never the dummy span (cf. `parse_item_mod`).
                 // Thus, if `inner` is the dummy span, we know the module is inline.
                 let inline_module = item.span.contains(inner) || inner.is_dummy();
 
@@ -1519,20 +1504,8 @@ impl<'a, 'b> Folder for InvocationCollector<'a, 'b> {
                         return noop_fold_attribute(at, self);
                     }
 
-                    let mut buf = vec![];
                     let filename = self.cx.root_path.join(file.to_string());
-
-                    match File::open(&filename).and_then(|mut f| f.read_to_end(&mut buf)) {
-                        Ok(..) => {}
-                        Err(e) => {
-                            self.cx.span_err(at.span,
-                                             &format!("couldn't read {}: {}",
-                                                      filename.display(),
-                                                      e));
-                        }
-                    }
-
-                    match String::from_utf8(buf) {
+                    match fs::read_to_string(&filename) {
                         Ok(src) => {
                             let src_interned = Symbol::intern(&src);
 
@@ -1542,25 +1515,82 @@ impl<'a, 'b> Folder for InvocationCollector<'a, 'b> {
 
                             let include_info = vec![
                                 dummy_spanned(ast::NestedMetaItemKind::MetaItem(
-                                        attr::mk_name_value_item_str(Ident::from_str("file"),
-                                                                     dummy_spanned(file)))),
+                                    attr::mk_name_value_item_str(
+                                        Ident::from_str("file"),
+                                        dummy_spanned(file),
+                                    ),
+                                )),
                                 dummy_spanned(ast::NestedMetaItemKind::MetaItem(
-                                        attr::mk_name_value_item_str(Ident::from_str("contents"),
-                                                            dummy_spanned(src_interned)))),
+                                    attr::mk_name_value_item_str(
+                                        Ident::from_str("contents"),
+                                        dummy_spanned(src_interned),
+                                    ),
+                                )),
                             ];
 
                             let include_ident = Ident::from_str("include");
                             let item = attr::mk_list_item(DUMMY_SP, include_ident, include_info);
                             items.push(dummy_spanned(ast::NestedMetaItemKind::MetaItem(item)));
                         }
-                        Err(_) => {
-                            self.cx.span_err(at.span,
-                                             &format!("{} wasn't a utf-8 file",
-                                                      filename.display()));
+                        Err(e) => {
+                            let lit = it
+                                .meta_item()
+                                .and_then(|item| item.name_value_literal())
+                                .unwrap();
+
+                            if e.kind() == ErrorKind::InvalidData {
+                                self.cx
+                                    .struct_span_err(
+                                        lit.span,
+                                        &format!("{} wasn't a utf-8 file", filename.display()),
+                                    )
+                                    .span_label(lit.span, "contains invalid utf-8")
+                                    .emit();
+                            } else {
+                                let mut err = self.cx.struct_span_err(
+                                    lit.span,
+                                    &format!("couldn't read {}: {}", filename.display(), e),
+                                );
+                                err.span_label(lit.span, "couldn't read file");
+
+                                if e.kind() == ErrorKind::NotFound {
+                                    err.help("external doc paths are relative to the crate root");
+                                }
+
+                                err.emit();
+                            }
                         }
                     }
                 } else {
-                    items.push(noop_fold_meta_list_item(it, self));
+                    let mut err = self.cx.struct_span_err(
+                        it.span,
+                        &format!("expected path to external documentation"),
+                    );
+
+                    // Check if the user erroneously used `doc(include(...))` syntax.
+                    let literal = it.meta_item_list().and_then(|list| {
+                        if list.len() == 1 {
+                            list[0].literal().map(|literal| &literal.node)
+                        } else {
+                            None
+                        }
+                    });
+
+                    let (path, applicability) = match &literal {
+                        Some(LitKind::Str(path, ..)) => {
+                            (path.to_string(), Applicability::MachineApplicable)
+                        }
+                        _ => (String::from("<path>"), Applicability::HasPlaceholders),
+                    };
+
+                    err.span_suggestion_with_applicability(
+                        it.span,
+                        "provide a file path with `=`",
+                        format!("include = \"{}\"", path),
+                        applicability,
+                    );
+
+                    err.emit();
                 }
             }
 
@@ -1623,7 +1653,6 @@ impl<'feat> ExpansionConfig<'feat> {
     }
 
     feature_tests! {
-        fn enable_quotes = quote,
         fn enable_asm = asm,
         fn enable_custom_test_frameworks = custom_test_frameworks,
         fn enable_global_asm = global_asm,
@@ -1631,7 +1660,6 @@ impl<'feat> ExpansionConfig<'feat> {
         fn enable_concat_idents = concat_idents,
         fn enable_trace_macros = trace_macros,
         fn enable_allow_internal_unstable = allow_internal_unstable,
-        fn enable_custom_derive = custom_derive,
         fn enable_format_args_nl = format_args_nl,
         fn macros_in_extern_enabled = macros_in_extern,
         fn proc_macro_hygiene = proc_macro_hygiene,
diff --git a/src/libsyntax/ext/quote.rs b/src/libsyntax/ext/quote.rs
index 69ed318b049..5820b49ab62 100644
--- a/src/libsyntax/ext/quote.rs
+++ b/src/libsyntax/ext/quote.rs
@@ -36,7 +36,7 @@ pub mod rt {
     use symbol::Symbol;
     use ThinVec;
 
-    use tokenstream::{self, DelimSpan, TokenTree, TokenStream};
+    use tokenstream::{DelimSpan, TokenTree, TokenStream};
 
     pub use parse::new_parser_from_tts;
     pub use syntax_pos::{BytePos, Span, DUMMY_SP, FileName};
@@ -246,9 +246,9 @@ pub mod rt {
             inner.push(self.tokens.clone());
 
             let delim_span = DelimSpan::from_single(self.span);
-            r.push(TokenTree::Delimited(delim_span, tokenstream::Delimited {
-                delim: token::Bracket, tts: TokenStream::concat(inner).into()
-            }));
+            r.push(TokenTree::Delimited(
+                delim_span, token::Bracket, TokenStream::new(inner).into()
+            ));
             r
         }
     }
@@ -262,10 +262,9 @@ pub mod rt {
 
     impl ToTokens for () {
         fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
-            vec![TokenTree::Delimited(DelimSpan::dummy(), tokenstream::Delimited {
-                delim: token::Paren,
-                tts: TokenStream::empty().into(),
-            })]
+            vec![
+                TokenTree::Delimited(DelimSpan::dummy(), token::Paren, TokenStream::empty().into())
+            ]
         }
     }
 
@@ -353,27 +352,27 @@ pub mod rt {
     impl<'a> ExtParseUtils for ExtCtxt<'a> {
         fn parse_item(&self, s: String) -> P<ast::Item> {
             panictry!(parse::parse_item_from_source_str(
-                FileName::QuoteExpansion,
+                FileName::quote_expansion_source_code(&s),
                 s,
                 self.parse_sess())).expect("parse error")
         }
 
         fn parse_stmt(&self, s: String) -> ast::Stmt {
             panictry!(parse::parse_stmt_from_source_str(
-                FileName::QuoteExpansion,
+                FileName::quote_expansion_source_code(&s),
                 s,
                 self.parse_sess())).expect("parse error")
         }
 
         fn parse_expr(&self, s: String) -> P<ast::Expr> {
             panictry!(parse::parse_expr_from_source_str(
-                FileName::QuoteExpansion,
+                FileName::quote_expansion_source_code(&s),
                 s,
                 self.parse_sess()))
         }
 
         fn parse_tts(&self, s: String) -> Vec<TokenTree> {
-            let source_name = FileName::QuoteExpansion;
+            let source_name = FileName::quote_expansion_source_code(&s);
             parse::parse_stream_from_source_str(source_name, s, self.parse_sess(), None)
                 .into_trees().collect()
         }
@@ -382,8 +381,6 @@ pub mod rt {
 
 // Replaces `Token::OpenDelim .. Token::CloseDelim` with `TokenTree::Delimited(..)`.
 pub fn unflatten(tts: Vec<TokenTree>) -> Vec<TokenTree> {
-    use tokenstream::Delimited;
-
     let mut results = Vec::new();
     let mut result = Vec::new();
     let mut open_span = DUMMY_SP;
@@ -395,10 +392,11 @@ pub fn unflatten(tts: Vec<TokenTree>) -> Vec<TokenTree> {
             }
             TokenTree::Token(span, token::CloseDelim(delim)) => {
                 let delim_span = DelimSpan::from_pair(open_span, span);
-                let tree = TokenTree::Delimited(delim_span, Delimited {
+                let tree = TokenTree::Delimited(
+                    delim_span,
                     delim,
-                    tts: result.into_iter().map(TokenStream::from).collect::<TokenStream>().into(),
-                });
+                    result.into_iter().map(TokenStream::from).collect::<TokenStream>().into(),
+                );
                 result = results.pop().unwrap();
                 result.push(tree);
             }
@@ -758,10 +756,10 @@ fn statements_mk_tt(cx: &ExtCtxt, tt: &TokenTree, quoted: bool) -> Vec<ast::Stmt
                                     vec![e_tok]);
             vec![cx.stmt_expr(e_push)]
         },
-        TokenTree::Delimited(span, ref delimed) => {
-            let mut stmts = statements_mk_tt(cx, &delimed.open_tt(span.open), false);
-            stmts.extend(statements_mk_tts(cx, delimed.stream()));
-            stmts.extend(statements_mk_tt(cx, &delimed.close_tt(span.close), false));
+        TokenTree::Delimited(span, delim, ref tts) => {
+            let mut stmts = statements_mk_tt(cx, &TokenTree::open_tt(span.open, delim), false);
+            stmts.extend(statements_mk_tts(cx, tts.stream()));
+            stmts.extend(statements_mk_tt(cx, &TokenTree::close_tt(span.close, delim), false));
             stmts
         }
     }
diff --git a/src/libsyntax/ext/source_util.rs b/src/libsyntax/ext/source_util.rs
index e1ba8897a47..ec27ceeb74c 100644
--- a/src/libsyntax/ext/source_util.rs
+++ b/src/libsyntax/ext/source_util.rs
@@ -21,8 +21,8 @@ use smallvec::SmallVec;
 use symbol::Symbol;
 use tokenstream;
 
-use std::fs::File;
-use std::io::prelude::*;
+use std::fs;
+use std::io::ErrorKind;
 use std::path::PathBuf;
 use rustc_data_structures::sync::Lrc;
 
@@ -137,18 +137,7 @@ pub fn expand_include_str(cx: &mut ExtCtxt, sp: Span, tts: &[tokenstream::TokenT
         None => return DummyResult::expr(sp)
     };
     let file = res_rel_file(cx, sp, file);
-    let mut bytes = Vec::new();
-    match File::open(&file).and_then(|mut f| f.read_to_end(&mut bytes)) {
-        Ok(..) => {}
-        Err(e) => {
-            cx.span_err(sp,
-                        &format!("couldn't read {}: {}",
-                                file.display(),
-                                e));
-            return DummyResult::expr(sp);
-        }
-    };
-    match String::from_utf8(bytes) {
+    match fs::read_to_string(&file) {
         Ok(src) => {
             let interned_src = Symbol::intern(&src);
 
@@ -157,11 +146,13 @@ pub fn expand_include_str(cx: &mut ExtCtxt, sp: Span, tts: &[tokenstream::TokenT
             cx.source_map().new_source_file(file.into(), src);
 
             base::MacEager::expr(cx.expr_str(sp, interned_src))
+        },
+        Err(ref e) if e.kind() == ErrorKind::InvalidData => {
+            cx.span_err(sp, &format!("{} wasn't a utf-8 file", file.display()));
+            DummyResult::expr(sp)
         }
-        Err(_) => {
-            cx.span_err(sp,
-                        &format!("{} wasn't a utf-8 file",
-                                file.display()));
+        Err(e) => {
+            cx.span_err(sp, &format!("couldn't read {}: {}", file.display(), e));
             DummyResult::expr(sp)
         }
     }
@@ -174,19 +165,23 @@ pub fn expand_include_bytes(cx: &mut ExtCtxt, sp: Span, tts: &[tokenstream::Toke
         None => return DummyResult::expr(sp)
     };
     let file = res_rel_file(cx, sp, file);
-    let mut bytes = Vec::new();
-    match File::open(&file).and_then(|mut f| f.read_to_end(&mut bytes)) {
-        Err(e) => {
-            cx.span_err(sp,
-                        &format!("couldn't read {}: {}", file.display(), e));
-            DummyResult::expr(sp)
-        }
-        Ok(..) => {
-            // Add this input file to the code map to make it available as
-            // dependency information, but don't enter it's contents
-            cx.source_map().new_source_file(file.into(), String::new());
+    match fs::read(&file) {
+        Ok(bytes) => {
+            // Add the contents to the source map if it contains UTF-8.
+            let (contents, bytes) = match String::from_utf8(bytes) {
+                Ok(s) => {
+                    let bytes = s.as_bytes().to_owned();
+                    (s, bytes)
+                },
+                Err(e) => (String::new(), e.into_bytes()),
+            };
+            cx.source_map().new_source_file(file.into(), contents);
 
             base::MacEager::expr(cx.expr_lit(sp, ast::LitKind::ByteStr(Lrc::new(bytes))))
+        },
+        Err(e) => {
+            cx.span_err(sp, &format!("couldn't read {}: {}", file.display(), e));
+            DummyResult::expr(sp)
         }
     }
 }
@@ -201,6 +196,7 @@ fn res_rel_file(cx: &mut ExtCtxt, sp: syntax_pos::Span, arg: String) -> PathBuf
         let callsite = sp.source_callsite();
         let mut path = match cx.source_map().span_to_unmapped_path(callsite) {
             FileName::Real(path) => path,
+            FileName::DocTest(path, _) => path,
             other => panic!("cannot resolve relative path in non-file source `{}`", other),
         };
         path.pop();
diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs
index 8fd9590a664..26348d5ec82 100644
--- a/src/libsyntax/ext/tt/macro_parser.rs
+++ b/src/libsyntax/ext/tt/macro_parser.rs
@@ -214,10 +214,10 @@ struct MatcherPos<'root, 'tt: 'root> {
     up: Option<MatcherPosHandle<'root, 'tt>>,
 
     /// Specifically used to "unzip" token trees. By "unzip", we mean to unwrap the delimiters from
-    /// a delimited token tree (e.g. something wrapped in `(` `)`) or to get the contents of a doc
+    /// a delimited token tree (e.g., something wrapped in `(` `)`) or to get the contents of a doc
     /// comment...
     ///
-    /// When matching against matchers with nested delimited submatchers (e.g. `pat ( pat ( .. )
+    /// When matching against matchers with nested delimited submatchers (e.g., `pat ( pat ( .. )
     /// pat ) pat`), we need to keep track of the matchers we are descending into. This stack does
     /// that where the bottom of the stack is the outermost matcher.
     /// Also, throughout the comments, this "descent" is often referred to as "unzipping"...
@@ -309,7 +309,7 @@ fn create_matches(len: usize) -> Box<[Rc<NamedMatchVec>]> {
         vec![]
     } else {
         let empty_matches = Rc::new(SmallVec::new());
-        vec![empty_matches.clone(); len]
+        vec![empty_matches; len]
     }.into_boxed_slice()
 }
 
@@ -373,7 +373,7 @@ fn nameize<I: Iterator<Item = NamedMatch>>(
     ms: &[TokenTree],
     mut res: I,
 ) -> NamedParseResult {
-    // Recursively descend into each type of matcher (e.g. sequences, delimited, metavars) and make
+    // Recursively descend into each type of matcher (e.g., sequences, delimited, metavars) and make
     // sure that each metavar has _exactly one_ binding. If a metavar does not have exactly one
     // binding, then there is an error. If it does, then we insert the binding into the
     // `NamedParseResult`.
@@ -895,7 +895,7 @@ fn may_begin_with(name: &str, token: &Token) -> bool {
 ///
 /// - `p`: the "black-box" parser to use
 /// - `sp`: the `Span` we want to parse
-/// - `name`: the name of the metavar _matcher_ we want to match (e.g. `tt`, `ident`, `block`,
+/// - `name`: the name of the metavar _matcher_ we want to match (e.g., `tt`, `ident`, `block`,
 ///   etc...)
 ///
 /// # Returns
diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs
index ff622b0c18f..30322ff523d 100644
--- a/src/libsyntax/ext/tt/macro_rules.rs
+++ b/src/libsyntax/ext/tt/macro_rules.rs
@@ -77,9 +77,9 @@ impl<'a> ParserAnyMacro<'a> {
             e
         }));
 
-        // We allow semicolons at the end of expressions -- e.g. the semicolon in
+        // We allow semicolons at the end of expressions -- e.g., the semicolon in
         // `macro_rules! m { () => { panic!(); } }` isn't parsed by `.parse_expr()`,
-        // but `m!()` is allowed in expression positions (c.f. issue #34706).
+        // but `m!()` is allowed in expression positions (cf. issue #34706).
         if kind == AstFragmentKind::Expr && parser.token == token::Semi {
             parser.bump();
         }
@@ -482,15 +482,15 @@ fn check_matcher(sess: &ParseSess,
     err == sess.span_diagnostic.err_count()
 }
 
-// The FirstSets for a matcher is a mapping from subsequences in the
+// `The FirstSets` for a matcher is a mapping from subsequences in the
 // matcher to the FIRST set for that subsequence.
 //
 // This mapping is partially precomputed via a backwards scan over the
 // token trees of the matcher, which provides a mapping from each
-// repetition sequence to its FIRST set.
+// repetition sequence to its *first* set.
 //
-// (Hypothetically sequences should be uniquely identifiable via their
-// spans, though perhaps that is false e.g. for macro-generated macros
+// (Hypothetically, sequences should be uniquely identifiable via their
+// spans, though perhaps that is false, e.g., for macro-generated macros
 // that do not try to inject artificial span information. My plan is
 // to try to catch such cases ahead of time and not include them in
 // the precomputed mapping.)
diff --git a/src/libsyntax/ext/tt/quoted.rs b/src/libsyntax/ext/tt/quoted.rs
index edc431be369..b142f09cdbc 100644
--- a/src/libsyntax/ext/tt/quoted.rs
+++ b/src/libsyntax/ext/tt/quoted.rs
@@ -93,9 +93,9 @@ pub enum TokenTree {
     Delimited(DelimSpan, Lrc<Delimited>),
     /// A kleene-style repetition sequence
     Sequence(DelimSpan, Lrc<SequenceRepetition>),
-    /// E.g. `$var`
+    /// e.g., `$var`
     MetaVar(Span, ast::Ident),
-    /// E.g. `$var:expr`. This is only used in the left hand side of MBE macros.
+    /// e.g., `$var:expr`. This is only used in the left hand side of MBE macros.
     MetaVarDecl(
         Span,
         ast::Ident, /* name to bind */
@@ -199,7 +199,7 @@ pub fn parse(
     let mut trees = input.trees().peekable();
     while let Some(tree) = trees.next() {
         // Given the parsed tree, if there is a metavar and we are expecting matchers, actually
-        // parse out the matcher (i.e. in `$id:ident` this would parse the `:` and `ident`).
+        // parse out the matcher (i.e., in `$id:ident` this would parse the `:` and `ident`).
         let tree = parse_tree(
             tree,
             &mut trees,
@@ -281,16 +281,16 @@ where
         tokenstream::TokenTree::Token(span, token::Dollar) => match trees.next() {
             // `tree` is followed by a delimited set of token trees. This indicates the beginning
             // of a repetition sequence in the macro (e.g. `$(pat)*`).
-            Some(tokenstream::TokenTree::Delimited(span, delimited)) => {
+            Some(tokenstream::TokenTree::Delimited(span, delim, tts)) => {
                 // Must have `(` not `{` or `[`
-                if delimited.delim != token::Paren {
-                    let tok = pprust::token_to_string(&token::OpenDelim(delimited.delim));
+                if delim != token::Paren {
+                    let tok = pprust::token_to_string(&token::OpenDelim(delim));
                     let msg = format!("expected `(`, found `{}`", tok);
                     sess.span_diagnostic.span_err(span.entire(), &msg);
                 }
                 // Parse the contents of the sequence itself
                 let sequence = parse(
-                    delimited.tts.into(),
+                    tts.into(),
                     expect_matchers,
                     sess,
                     features,
@@ -309,7 +309,7 @@ where
                         edition,
                         macro_node_id,
                     );
-                // Count the number of captured "names" (i.e. named metavars)
+                // Count the number of captured "names" (i.e., named metavars)
                 let name_captures = macro_parser::count_names(&sequence);
                 TokenTree::Sequence(
                     span,
@@ -352,14 +352,14 @@ where
         // `tree` is an arbitrary token. Keep it.
         tokenstream::TokenTree::Token(span, tok) => TokenTree::Token(span, tok),
 
-        // `tree` is the beginning of a delimited set of tokens (e.g. `(` or `{`). We need to
+        // `tree` is the beginning of a delimited set of tokens (e.g., `(` or `{`). We need to
         // descend into the delimited set and further parse it.
-        tokenstream::TokenTree::Delimited(span, delimited) => TokenTree::Delimited(
+        tokenstream::TokenTree::Delimited(span, delim, tts) => TokenTree::Delimited(
             span,
             Lrc::new(Delimited {
-                delim: delimited.delim,
+                delim: delim,
                 tts: parse(
-                    delimited.tts.into(),
+                    tts.into(),
                     expect_matchers,
                     sess,
                     features,
@@ -444,7 +444,6 @@ where
             macro_node_id,
         ),
         Edition::Edition2018 => parse_sep_and_kleene_op_2018(input, span, sess, features, attrs),
-        _ => unimplemented!(),
     }
 }
 
diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/tt/transcribe.rs
index 3d897d17e0b..a63abd40495 100644
--- a/src/libsyntax/ext/tt/transcribe.rs
+++ b/src/libsyntax/ext/tt/transcribe.rs
@@ -17,7 +17,7 @@ use fold::noop_fold_tt;
 use parse::token::{self, Token, NtTT};
 use smallvec::SmallVec;
 use syntax_pos::DUMMY_SP;
-use tokenstream::{TokenStream, TokenTree, Delimited, DelimSpan};
+use tokenstream::{TokenStream, TokenTree, DelimSpan};
 
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::sync::Lrc;
@@ -103,12 +103,13 @@ pub fn transcribe(cx: &ExtCtxt,
                 }
                 Frame::Delimited { forest, span, .. } => {
                     if result_stack.is_empty() {
-                        return TokenStream::concat(result);
+                        return TokenStream::new(result);
                     }
-                    let tree = TokenTree::Delimited(span, Delimited {
-                        delim: forest.delim,
-                        tts: TokenStream::concat(result).into(),
-                    });
+                    let tree = TokenTree::Delimited(
+                        span,
+                        forest.delim,
+                        TokenStream::new(result).into(),
+                    );
                     result = result_stack.pop().unwrap();
                     result.push(tree.into());
                 }