Merge branch 'master' into issue-30961

2 files changed, 102 insertions, 154 deletions

diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs
index e4875b7c244..4656ba03e21 100644
--- a/src/libsyntax/parse/parser.rs
+++ b/src/libsyntax/parse/parser.rs
@@ -259,7 +259,6 @@ pub struct Parser<'a> {
     pub restrictions: Restrictions,
     pub quote_depth: usize, // not (yet) related to the quasiquoter
     pub reader: Box<Reader+'a>,
-    pub interner: Rc<token::IdentInterner>,
     /// The set of seen errors about obsolete syntax. Used to suppress
     /// extra detail when the same error is seen twice
     pub obsolete_set: HashSet<ObsoleteSyntax>,
@@ -356,7 +355,6 @@ impl<'a> Parser<'a> {
 
         Parser {
             reader: rdr,
-            interner: token::get_ident_interner(),
             sess: sess,
             cfg: cfg,
             token: tok0.tok,
@@ -3789,7 +3787,13 @@ impl<'a> Parser<'a> {
         self.span_err(self.last_span, message);
     }
 
-    /// Parse a statement. may include decl.
+    /// Parse a statement. This stops just before trailing semicolons on everything but items.
+    /// e.g. a `StmtKind::Semi` parses to a `StmtKind::Expr`, leaving the trailing `;` unconsumed.
+    ///
+    /// Also, if a macro begins an expression statement, this only parses the macro. For example,
+    /// ```rust
+    /// vec![1].into_iter(); //< `parse_stmt` only parses the "vec![1]"
+    /// ```
     pub fn parse_stmt(&mut self) -> PResult<'a, Option<Stmt>> {
         Ok(self.parse_stmt_())
     }
@@ -4038,36 +4042,14 @@ impl<'a> Parser<'a> {
         let mut stmts = vec![];
 
         while !self.eat(&token::CloseDelim(token::Brace)) {
-            let Stmt {node, span, ..} = if let Some(s) = self.parse_stmt_() {
-                s
+            if let Some(stmt) = self.parse_full_stmt(false)? {
+                stmts.push(stmt);
             } else if self.token == token::Eof {
                 break;
             } else {
                 // Found only `;` or `}`.
                 continue;
             };
-
-            match node {
-                StmtKind::Expr(e) => {
-                    self.handle_expression_like_statement(e, span, &mut stmts)?;
-                }
-                StmtKind::Mac(mac) => {
-                    self.handle_macro_in_block(mac.unwrap(), span, &mut stmts)?;
-                }
-                _ => { // all other kinds of statements:
-                    let mut hi = span.hi;
-                    if classify::stmt_ends_with_semi(&node) {
-                        self.expect(&token::Semi)?;
-                        hi = self.last_span.hi;
-                    }
-
-                    stmts.push(Stmt {
-                        id: ast::DUMMY_NODE_ID,
-                        node: node,
-                        span: mk_sp(span.lo, hi)
-                    });
-                }
-            }
         }
 
         Ok(P(ast::Block {
@@ -4078,93 +4060,88 @@ impl<'a> Parser<'a> {
         }))
     }
 
-    fn handle_macro_in_block(&mut self,
-                             (mac, style, attrs): (ast::Mac, MacStmtStyle, ThinVec<Attribute>),
-                             span: Span,
-                             stmts: &mut Vec<Stmt>)
-                             -> PResult<'a, ()> {
-        if style == MacStmtStyle::NoBraces {
-            // statement macro without braces; might be an
-            // expr depending on whether a semicolon follows
-            match self.token {
-                token::Semi => {
-                    stmts.push(Stmt {
-                        id: ast::DUMMY_NODE_ID,
-                        node: StmtKind::Mac(P((mac, MacStmtStyle::Semicolon, attrs))),
-                        span: mk_sp(span.lo, self.span.hi),
-                    });
-                    self.bump();
-                }
-                _ => {
-                    let e = self.mk_mac_expr(span.lo, span.hi, mac.node, ThinVec::new());
-                    let lo = e.span.lo;
-                    let e = self.parse_dot_or_call_expr_with(e, lo, attrs)?;
-                    let e = self.parse_assoc_expr_with(0, LhsExpr::AlreadyParsed(e))?;
-                    self.handle_expression_like_statement(e, span, stmts)?;
-                }
-            }
-        } else {
-            // statement macro; might be an expr
-            match self.token {
-                token::Semi => {
-                    stmts.push(Stmt {
-                        id: ast::DUMMY_NODE_ID,
-                        node: StmtKind::Mac(P((mac, MacStmtStyle::Semicolon, attrs))),
-                        span: mk_sp(span.lo, self.span.hi),
-                    });
-                    self.bump();
-                }
-                _ => {
-                    stmts.push(Stmt {
-                        id: ast::DUMMY_NODE_ID,
-                        node: StmtKind::Mac(P((mac, style, attrs))),
-                        span: span
-                    });
+    /// Parse a statement, including the trailing semicolon.
+    /// This parses expression statements that begin with macros correctly (c.f. `parse_stmt`).
+    pub fn parse_full_stmt(&mut self, macro_expanded: bool) -> PResult<'a, Option<Stmt>> {
+        let mut stmt = match self.parse_stmt_() {
+            Some(stmt) => stmt,
+            None => return Ok(None),
+        };
+
+        if let StmtKind::Mac(mac) = stmt.node {
+            if mac.1 != MacStmtStyle::NoBraces ||
+               self.token == token::Semi || self.token == token::Eof {
+                stmt.node = StmtKind::Mac(mac);
+            } else {
+                // We used to incorrectly stop parsing macro-expanded statements here.
+                // If the next token will be an error anyway but could have parsed with the
+                // earlier behavior, stop parsing here and emit a warning to avoid breakage.
+                if macro_expanded && self.token.can_begin_expr() && match self.token {
+                    // These tokens can continue an expression, so we can't stop parsing and warn.
+                    token::OpenDelim(token::Paren) | token::OpenDelim(token::Bracket) |
+                    token::BinOp(token::Minus) | token::BinOp(token::Star) |
+                    token::BinOp(token::And) | token::BinOp(token::Or) |
+                    token::AndAnd | token::OrOr |
+                    token::DotDot | token::DotDotDot => false,
+                    _ => true,
+                } {
+                    self.warn_missing_semicolon();
+                    stmt.node = StmtKind::Mac(mac);
+                    return Ok(Some(stmt));
                 }
+
+                let (mac, _style, attrs) = mac.unwrap();
+                let e = self.mk_mac_expr(stmt.span.lo, stmt.span.hi, mac.node, ThinVec::new());
+                let e = self.parse_dot_or_call_expr_with(e, stmt.span.lo, attrs)?;
+                let e = self.parse_assoc_expr_with(0, LhsExpr::AlreadyParsed(e))?;
+                stmt.node = StmtKind::Expr(e);
             }
         }
-        Ok(())
+
+        stmt = self.handle_trailing_semicolon(stmt, macro_expanded)?;
+        Ok(Some(stmt))
     }
 
-    fn handle_expression_like_statement(&mut self,
-                                        e: P<Expr>,
-                                        span: Span,
-                                        stmts: &mut Vec<Stmt>)
-                                        -> PResult<'a, ()> {
-        // expression without semicolon
-        if classify::expr_requires_semi_to_be_stmt(&e) {
-            // Just check for errors and recover; do not eat semicolon yet.
-            if let Err(mut e) =
-                self.expect_one_of(&[], &[token::Semi, token::CloseDelim(token::Brace)])
-            {
-                e.emit();
-                self.recover_stmt();
+    fn handle_trailing_semicolon(&mut self, mut stmt: Stmt, macro_expanded: bool)
+                                 -> PResult<'a, Stmt> {
+        match stmt.node {
+            StmtKind::Expr(ref expr) if self.token != token::Eof => {
+                // expression without semicolon
+                if classify::expr_requires_semi_to_be_stmt(expr) {
+                    // Just check for errors and recover; do not eat semicolon yet.
+                    if let Err(mut e) =
+                        self.expect_one_of(&[], &[token::Semi, token::CloseDelim(token::Brace)])
+                    {
+                        e.emit();
+                        self.recover_stmt();
+                    }
+                }
+            }
+            StmtKind::Local(..) => {
+                // We used to incorrectly allow a macro-expanded let statement to lack a semicolon.
+                if macro_expanded && self.token != token::Semi {
+                    self.warn_missing_semicolon();
+                } else {
+                    self.expect_one_of(&[token::Semi], &[])?;
+                }
             }
+            _ => {}
         }
 
-        match self.token {
-            token::Semi => {
-                self.bump();
-                let span_with_semi = Span {
-                    lo: span.lo,
-                    hi: self.last_span.hi,
-                    expn_id: span.expn_id,
-                };
-                stmts.push(Stmt {
-                    id: ast::DUMMY_NODE_ID,
-                    node: StmtKind::Semi(e),
-                    span: span_with_semi,
-                });
-            }
-            _ => {
-                stmts.push(Stmt {
-                    id: ast::DUMMY_NODE_ID,
-                    node: StmtKind::Expr(e),
-                    span: span
-                });
-            }
+        if self.eat(&token::Semi) {
+            stmt = stmt.add_trailing_semicolon();
         }
-        Ok(())
+
+        stmt.span.hi = self.last_span.hi;
+        Ok(stmt)
+    }
+
+    fn warn_missing_semicolon(&self) {
+        self.diagnostic().struct_span_warn(self.span, {
+            &format!("expected `;`, found `{}`", self.this_token_to_string())
+        }).note({
+            "This was erroneously allowed and will become a hard error in a future release"
+        }).emit();
     }
 
     // Parses a sequence of bounds if a `:` is found,
diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs
index 8376d28164d..fe9d3ef7c23 100644
--- a/src/libsyntax/parse/token.rs
+++ b/src/libsyntax/parse/token.rs
@@ -15,13 +15,12 @@ pub use self::Lit::*;
 pub use self::Token::*;
 
 use ast::{self, BinOpKind};
-use ext::mtwt;
 use ptr::P;
-use util::interner::{RcStr, StrInterner};
-use util::interner;
+use util::interner::Interner;
 use tokenstream;
 
 use serialize::{Decodable, Decoder, Encodable, Encoder};
+use std::cell::RefCell;
 use std::fmt;
 use std::ops::Deref;
 use std::rc::Rc;
@@ -313,17 +312,6 @@ impl Token {
             _ => false,
         }
     }
-
-    /// Hygienic identifier equality comparison.
-    ///
-    /// See `styntax::ext::mtwt`.
-    pub fn mtwt_eq(&self, other : &Token) -> bool {
-        match (self, other) {
-            (&Ident(id1), &Ident(id2)) | (&Lifetime(id1), &Lifetime(id2)) =>
-                mtwt::resolve(id1) == mtwt::resolve(id2),
-            _ => *self == *other
-        }
-    }
 }
 
 #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash)]
@@ -397,7 +385,7 @@ macro_rules! declare_keywords {(
     }
 
     fn mk_fresh_ident_interner() -> IdentInterner {
-        interner::StrInterner::prefill(&[$($string,)*])
+        Interner::prefill(&[$($string,)*])
     }
 }}
 
@@ -473,22 +461,25 @@ declare_keywords! {
 }
 
 // looks like we can get rid of this completely...
-pub type IdentInterner = StrInterner;
+pub type IdentInterner = Interner;
 
 // if an interner exists in TLS, return it. Otherwise, prepare a
 // fresh one.
 // FIXME(eddyb) #8726 This should probably use a thread-local reference.
-pub fn get_ident_interner() -> Rc<IdentInterner> {
-    thread_local!(static KEY: Rc<::parse::token::IdentInterner> = {
-        Rc::new(mk_fresh_ident_interner())
+pub fn with_ident_interner<T, F: FnOnce(&mut IdentInterner) -> T>(f: F) -> T {
+    thread_local!(static KEY: RefCell<IdentInterner> = {
+        RefCell::new(mk_fresh_ident_interner())
     });
-    KEY.with(|k| k.clone())
+    KEY.with(|interner| f(&mut *interner.borrow_mut()))
 }
 
 /// Reset the ident interner to its initial state.
 pub fn reset_ident_interner() {
-    let interner = get_ident_interner();
-    interner.reset(mk_fresh_ident_interner());
+    with_ident_interner(|interner| *interner = mk_fresh_ident_interner());
+}
+
+pub fn clear_ident_interner() {
+    with_ident_interner(|interner| *interner = IdentInterner::new());
 }
 
 /// Represents a string stored in the thread-local interner. Because the
@@ -502,19 +493,19 @@ pub fn reset_ident_interner() {
 /// somehow.
 #[derive(Clone, PartialEq, Hash, PartialOrd, Eq, Ord)]
 pub struct InternedString {
-    string: RcStr,
+    string: Rc<String>,
 }
 
 impl InternedString {
     #[inline]
     pub fn new(string: &'static str) -> InternedString {
         InternedString {
-            string: RcStr::new(string),
+            string: Rc::new(string.to_owned()),
         }
     }
 
     #[inline]
-    fn new_from_rc_str(string: RcStr) -> InternedString {
+    fn new_from_rc_str(string: Rc<String>) -> InternedString {
         InternedString {
             string: string,
         }
@@ -522,8 +513,7 @@ impl InternedString {
 
     #[inline]
     pub fn new_from_name(name: ast::Name) -> InternedString {
-        let interner = get_ident_interner();
-        InternedString::new_from_rc_str(interner.get(name))
+        with_ident_interner(|interner| InternedString::new_from_rc_str(interner.get(name)))
     }
 }
 
@@ -611,13 +601,13 @@ pub fn intern_and_get_ident(s: &str) -> InternedString {
 /// Maps a string to its interned representation.
 #[inline]
 pub fn intern(s: &str) -> ast::Name {
-    get_ident_interner().intern(s)
+    with_ident_interner(|interner| interner.intern(s))
 }
 
 /// gensym's a new usize, using the current interner.
 #[inline]
 pub fn gensym(s: &str) -> ast::Name {
-    get_ident_interner().gensym(s)
+    with_ident_interner(|interner| interner.gensym(s))
 }
 
 /// Maps a string to an identifier with an empty syntax context.
@@ -636,8 +626,7 @@ pub fn gensym_ident(s: &str) -> ast::Ident {
 // note that this guarantees that str_ptr_eq(ident_to_string(src),interner_get(fresh_name(src)));
 // that is, that the new name and the old one are connected to ptr_eq strings.
 pub fn fresh_name(src: ast::Ident) -> ast::Name {
-    let interner = get_ident_interner();
-    interner.gensym_copy(src.name)
+    with_ident_interner(|interner| interner.gensym_copy(src.name))
     // following: debug version. Could work in final except that it's incompatible with
     // good error messages and uses of struct names in ambiguous could-be-binding
     // locations. Also definitely destroys the guarantee given above about ptr_eq.
@@ -649,21 +638,3 @@ pub fn fresh_name(src: ast::Ident) -> ast::Name {
 pub fn fresh_mark() -> ast::Mrk {
     gensym("mark").0
 }
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use ast;
-    use ext::mtwt;
-
-    fn mark_ident(id : ast::Ident, m : ast::Mrk) -> ast::Ident {
-        ast::Ident::new(id.name, mtwt::apply_mark(m, id.ctxt))
-    }
-
-    #[test] fn mtwt_token_eq_test() {
-        assert!(Gt.mtwt_eq(&Gt));
-        let a = str_to_ident("bac");
-        let a1 = mark_ident(a,92);
-        assert!(Ident(a).mtwt_eq(&Ident(a1)));
-    }
-}