Merge pull request #17 from rust-lang/master

sync with rust-lang/rust master branch

8 files changed, 74 insertions, 82 deletions

diff --git a/src/libsyntax/diagnostics/plugin.rs b/src/libsyntax/diagnostics/plugin.rs
index e9a55af52e8..5de39c8d14d 100644
--- a/src/libsyntax/diagnostics/plugin.rs
+++ b/src/libsyntax/diagnostics/plugin.rs
@@ -6,7 +6,7 @@ use crate::ext::base::{ExtCtxt, MacEager, MacResult};
 use crate::parse::token::{self, Token};
 use crate::ptr::P;
 use crate::symbol::kw;
-use crate::tokenstream::{TokenTree};
+use crate::tokenstream::{TokenTree, TokenStream};
 
 use smallvec::smallvec;
 use syntax_pos::Span;
@@ -27,12 +27,11 @@ pub type ErrorMap = BTreeMap<Name, ErrorInfo>;
 
 pub fn expand_diagnostic_used<'cx>(ecx: &'cx mut ExtCtxt<'_>,
                                    span: Span,
-                                   token_tree: &[TokenTree])
+                                   tts: TokenStream)
                                    -> Box<dyn MacResult+'cx> {
-    let code = match token_tree {
-        [
-            TokenTree::Token(Token { kind: token::Ident(code, _), .. })
-        ] => code,
+    assert_eq!(tts.len(), 1);
+    let code = match tts.into_trees().next() {
+        Some(TokenTree::Token(Token { kind: token::Ident(code, _), .. })) => code,
         _ => unreachable!()
     };
 
@@ -62,20 +61,21 @@ pub fn expand_diagnostic_used<'cx>(ecx: &'cx mut ExtCtxt<'_>,
 
 pub fn expand_register_diagnostic<'cx>(ecx: &'cx mut ExtCtxt<'_>,
                                        span: Span,
-                                       token_tree: &[TokenTree])
+                                       tts: TokenStream)
                                        -> Box<dyn MacResult+'cx> {
-    let (code, description) = match  token_tree {
-        [
-            TokenTree::Token(Token { kind: token::Ident(code, _), .. })
-        ] => {
-            (*code, None)
-        },
-        [
-            TokenTree::Token(Token { kind: token::Ident(code, _), .. }),
-            TokenTree::Token(Token { kind: token::Comma, .. }),
-            TokenTree::Token(Token { kind: token::Literal(token::Lit { symbol, .. }), ..})
-        ] => {
-            (*code, Some(*symbol))
+    assert!(tts.len() == 1 || tts.len() == 3);
+    let mut cursor = tts.into_trees();
+    let code = match cursor.next() {
+        Some(TokenTree::Token(Token { kind: token::Ident(code, _), .. })) => code,
+        _ => unreachable!()
+    };
+    let description = match  (cursor.next(), cursor.next()) {
+        (None, None) => None,
+        (
+            Some(TokenTree::Token(Token { kind: token::Comma, .. })),
+            Some(TokenTree::Token(Token { kind: token::Literal(token::Lit { symbol, .. }), ..}))
+        ) => {
+            Some(symbol)
         },
         _ => unreachable!()
     };
@@ -121,12 +121,12 @@ pub fn expand_register_diagnostic<'cx>(ecx: &'cx mut ExtCtxt<'_>,
 
 pub fn expand_build_diagnostic_array<'cx>(ecx: &'cx mut ExtCtxt<'_>,
                                           span: Span,
-                                          token_tree: &[TokenTree])
+                                          tts: TokenStream)
                                           -> Box<dyn MacResult+'cx> {
-    assert_eq!(token_tree.len(), 3);
-    let ident = match &token_tree[2] {
+    assert_eq!(tts.len(), 3);
+    let ident = match tts.into_trees().nth(2) {
         // DIAGNOSTICS ident.
-        &TokenTree::Token(Token { kind: token::Ident(name, _), span })
+        Some(TokenTree::Token(Token { kind: token::Ident(name, _), span }))
         => Ident::new(name, span),
         _ => unreachable!()
     };
diff --git a/src/libsyntax/ext/base.rs b/src/libsyntax/ext/base.rs
index 5d68983d7cb..109ba041016 100644
--- a/src/libsyntax/ext/base.rs
+++ b/src/libsyntax/ext/base.rs
@@ -10,7 +10,7 @@ use crate::parse::token;
 use crate::ptr::P;
 use crate::symbol::{kw, sym, Ident, Symbol};
 use crate::{ThinVec, MACRO_ARGUMENTS};
-use crate::tokenstream::{self, TokenStream, TokenTree};
+use crate::tokenstream::{self, TokenStream};
 use crate::visit::Visitor;
 
 use errors::{DiagnosticBuilder, DiagnosticId};
@@ -235,18 +235,18 @@ pub trait TTMacroExpander {
 }
 
 pub type MacroExpanderFn =
-    for<'cx> fn(&'cx mut ExtCtxt<'_>, Span, &[tokenstream::TokenTree])
+    for<'cx> fn(&'cx mut ExtCtxt<'_>, Span, TokenStream)
                 -> Box<dyn MacResult+'cx>;
 
 impl<F> TTMacroExpander for F
-    where F: for<'cx> Fn(&'cx mut ExtCtxt<'_>, Span, &[tokenstream::TokenTree])
+    where F: for<'cx> Fn(&'cx mut ExtCtxt<'_>, Span, TokenStream)
     -> Box<dyn MacResult+'cx>
 {
     fn expand<'cx>(
         &self,
         ecx: &'cx mut ExtCtxt<'_>,
         span: Span,
-        input: TokenStream,
+        mut input: TokenStream,
     ) -> Box<dyn MacResult+'cx> {
         struct AvoidInterpolatedIdents;
 
@@ -268,10 +268,8 @@ impl<F> TTMacroExpander for F
                 mut_visit::noop_visit_mac(mac, self)
             }
         }
-
-        let input: Vec<_> =
-            input.trees().map(|mut tt| { AvoidInterpolatedIdents.visit_tt(&mut tt); tt }).collect();
-        (*self)(ecx, span, &input)
+        AvoidInterpolatedIdents.visit_tts(&mut input);
+        (*self)(ecx, span, input)
     }
 }
 
@@ -677,7 +675,7 @@ impl SyntaxExtension {
     }
 
     pub fn dummy_bang(edition: Edition) -> SyntaxExtension {
-        fn expander<'cx>(_: &'cx mut ExtCtxt<'_>, span: Span, _: &[TokenTree])
+        fn expander<'cx>(_: &'cx mut ExtCtxt<'_>, span: Span, _: TokenStream)
                          -> Box<dyn MacResult + 'cx> {
             DummyResult::any(span)
         }
@@ -811,9 +809,8 @@ impl<'a> ExtCtxt<'a> {
     pub fn monotonic_expander<'b>(&'b mut self) -> expand::MacroExpander<'b, 'a> {
         expand::MacroExpander::new(self, true)
     }
-
-    pub fn new_parser_from_tts(&self, tts: &[tokenstream::TokenTree]) -> parser::Parser<'a> {
-        parse::stream_to_parser(self.parse_sess, tts.iter().cloned().collect(), MACRO_ARGUMENTS)
+    pub fn new_parser_from_tts(&self, stream: TokenStream) -> parser::Parser<'a> {
+        parse::stream_to_parser(self.parse_sess, stream, MACRO_ARGUMENTS)
     }
     pub fn source_map(&self) -> &'a SourceMap { self.parse_sess.source_map() }
     pub fn parse_sess(&self) -> &'a parse::ParseSess { self.parse_sess }
@@ -1019,7 +1016,7 @@ pub fn expr_to_string(cx: &mut ExtCtxt<'_>, expr: P<ast::Expr>, err_msg: &str)
 /// done as rarely as possible).
 pub fn check_zero_tts(cx: &ExtCtxt<'_>,
                       sp: Span,
-                      tts: &[tokenstream::TokenTree],
+                      tts: TokenStream,
                       name: &str) {
     if !tts.is_empty() {
         cx.span_err(sp, &format!("{} takes no arguments", name));
@@ -1030,7 +1027,7 @@ pub fn check_zero_tts(cx: &ExtCtxt<'_>,
 /// expect exactly one string literal, or emit an error and return `None`.
 pub fn get_single_str_from_tts(cx: &mut ExtCtxt<'_>,
                                sp: Span,
-                               tts: &[tokenstream::TokenTree],
+                               tts: TokenStream,
                                name: &str)
                                -> Option<String> {
     let mut p = cx.new_parser_from_tts(tts);
@@ -1053,7 +1050,7 @@ pub fn get_single_str_from_tts(cx: &mut ExtCtxt<'_>,
 /// parsing error, emit a non-fatal error and return `None`.
 pub fn get_exprs_from_tts(cx: &mut ExtCtxt<'_>,
                           sp: Span,
-                          tts: &[tokenstream::TokenTree]) -> Option<Vec<P<ast::Expr>>> {
+                          tts: TokenStream) -> Option<Vec<P<ast::Expr>>> {
     let mut p = cx.new_parser_from_tts(tts);
     let mut es = Vec::new();
     while p.token != token::Eof {
diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index 7b4a5167446..4fd0c367288 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -701,7 +701,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
         path: &Path,
         span: Span,
     ) -> AstFragment {
-        let mut parser = self.cx.new_parser_from_tts(&toks.into_trees().collect::<Vec<_>>());
+        let mut parser = self.cx.new_parser_from_tts(toks);
         match parser.parse_ast_fragment(kind, false) {
             Ok(fragment) => {
                 parser.ensure_complete_parse(path, kind.name(), span);
diff --git a/src/libsyntax/ext/proc_macro_server.rs b/src/libsyntax/ext/proc_macro_server.rs
index 1a26b17dac7..544ec789d80 100644
--- a/src/libsyntax/ext/proc_macro_server.rs
+++ b/src/libsyntax/ext/proc_macro_server.rs
@@ -322,8 +322,7 @@ impl Ident {
     fn is_valid(string: &str) -> bool {
         let mut chars = string.chars();
         if let Some(start) = chars.next() {
-            (start == '_' || start.is_xid_start())
-                && chars.all(|cont| cont == '_' || cont.is_xid_continue())
+            rustc_lexer::is_id_start(start) && chars.all(rustc_lexer::is_id_continue)
         } else {
             false
         }
diff --git a/src/libsyntax/parse/parser/expr.rs b/src/libsyntax/parse/parser/expr.rs
index 5b9f0f1df67..e502a08f4b2 100644
--- a/src/libsyntax/parse/parser/expr.rs
+++ b/src/libsyntax/parse/parser/expr.rs
@@ -889,6 +889,36 @@ impl<'a> Parser<'a> {
                     hi = path.span;
                     return Ok(self.mk_expr(lo.to(hi), ExprKind::Path(Some(qself), path), attrs));
                 }
+                if self.token.is_path_start() {
+                    let path = self.parse_path(PathStyle::Expr)?;
+
+                    // `!`, as an operator, is prefix, so we know this isn't that
+                    if self.eat(&token::Not) {
+                        // MACRO INVOCATION expression
+                        let (delim, tts) = self.expect_delimited_token_tree()?;
+                        hi = self.prev_span;
+                        ex = ExprKind::Mac(Mac {
+                            path,
+                            tts,
+                            delim,
+                            span: lo.to(hi),
+                            prior_type_ascription: self.last_type_ascription,
+                        });
+                    } else if self.check(&token::OpenDelim(token::Brace)) {
+                        if let Some(expr) = self.maybe_parse_struct_expr(lo, &path, &attrs) {
+                            return expr;
+                        } else {
+                            hi = path.span;
+                            ex = ExprKind::Path(None, path);
+                        }
+                    } else {
+                        hi = path.span;
+                        ex = ExprKind::Path(None, path);
+                    }
+
+                    let expr = self.mk_expr(lo.to(hi), ex, attrs);
+                    return self.maybe_recover_from_bad_qpath(expr, true);
+                }
                 if self.check_keyword(kw::Move) || self.check_keyword(kw::Static) {
                     return self.parse_lambda_expr(attrs);
                 }
@@ -1007,32 +1037,6 @@ impl<'a> Parser<'a> {
                     let (await_hi, e_kind) = self.parse_incorrect_await_syntax(lo, self.prev_span)?;
                     hi = await_hi;
                     ex = e_kind;
-                } else if self.token.is_path_start() {
-                    let path = self.parse_path(PathStyle::Expr)?;
-
-                    // `!`, as an operator, is prefix, so we know this isn't that
-                    if self.eat(&token::Not) {
-                        // MACRO INVOCATION expression
-                        let (delim, tts) = self.expect_delimited_token_tree()?;
-                        hi = self.prev_span;
-                        ex = ExprKind::Mac(Mac {
-                            path,
-                            tts,
-                            delim,
-                            span: lo.to(hi),
-                            prior_type_ascription: self.last_type_ascription,
-                        });
-                    } else if self.check(&token::OpenDelim(token::Brace)) {
-                        if let Some(expr) = self.maybe_parse_struct_expr(lo, &path, &attrs) {
-                            return expr;
-                        } else {
-                            hi = path.span;
-                            ex = ExprKind::Path(None, path);
-                        }
-                    } else {
-                        hi = path.span;
-                        ex = ExprKind::Path(None, path);
-                    }
                 } else {
                     if !self.unclosed_delims.is_empty() && self.check(&token::Semi) {
                         // Don't complain about bare semicolons after unclosed braces
diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs
index bead941b20d..37305055e62 100644
--- a/src/libsyntax/print/pprust.rs
+++ b/src/libsyntax/print/pprust.rs
@@ -356,11 +356,7 @@ pub fn tt_to_string(tt: tokenstream::TokenTree) -> String {
     to_string(|s| s.print_tt(tt, false))
 }
 
-pub fn tts_to_string(tts: &[tokenstream::TokenTree]) -> String {
-    tokens_to_string(tts.iter().cloned().collect())
-}
-
-pub fn tokens_to_string(tokens: TokenStream) -> String {
+pub fn tts_to_string(tokens: TokenStream) -> String {
     to_string(|s| s.print_tts(tokens, false))
 }
 
diff --git a/src/libsyntax/tests.rs b/src/libsyntax/tests.rs
index c472212bc20..9b90b31f2d2 100644
--- a/src/libsyntax/tests.rs
+++ b/src/libsyntax/tests.rs
@@ -63,7 +63,7 @@ crate fn matches_codepattern(a : &str, b : &str) -> bool {
             (None, None) => return true,
             (None, _) => return false,
             (Some(&a), None) => {
-                if is_pattern_whitespace(a) {
+                if rustc_lexer::is_whitespace(a) {
                     break // trailing whitespace check is out of loop for borrowck
                 } else {
                     return false
@@ -72,11 +72,11 @@ crate fn matches_codepattern(a : &str, b : &str) -> bool {
             (Some(&a), Some(&b)) => (a, b)
         };
 
-        if is_pattern_whitespace(a) && is_pattern_whitespace(b) {
+        if rustc_lexer::is_whitespace(a) && rustc_lexer::is_whitespace(b) {
             // skip whitespace for a and b
             scan_for_non_ws_or_end(&mut a_iter);
             scan_for_non_ws_or_end(&mut b_iter);
-        } else if is_pattern_whitespace(a) {
+        } else if rustc_lexer::is_whitespace(a) {
             // skip whitespace for a
             scan_for_non_ws_or_end(&mut a_iter);
         } else if a == b {
@@ -88,20 +88,16 @@ crate fn matches_codepattern(a : &str, b : &str) -> bool {
     }
 
     // check if a has *only* trailing whitespace
-    a_iter.all(is_pattern_whitespace)
+    a_iter.all(rustc_lexer::is_whitespace)
 }
 
 /// Advances the given peekable `Iterator` until it reaches a non-whitespace character
 fn scan_for_non_ws_or_end<I: Iterator<Item = char>>(iter: &mut Peekable<I>) {
-    while iter.peek().copied().map(|c| is_pattern_whitespace(c)) == Some(true) {
+    while iter.peek().copied().map(|c| rustc_lexer::is_whitespace(c)) == Some(true) {
         iter.next();
     }
 }
 
-fn is_pattern_whitespace(c: char) -> bool {
-    rustc_lexer::character_properties::is_whitespace(c)
-}
-
 /// Identify a position in the text by the Nth occurrence of a string.
 struct Position {
     string: &'static str,
diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs
index 0d9f3769ce9..48055358619 100644
--- a/src/libsyntax/tokenstream.rs
+++ b/src/libsyntax/tokenstream.rs
@@ -506,7 +506,7 @@ impl Cursor {
 
 impl fmt::Display for TokenStream {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.write_str(&pprust::tokens_to_string(self.clone()))
+        f.write_str(&pprust::tts_to_string(self.clone()))
     }
 }