Merge branch 'master' into issue-30961

5 files changed, 102 insertions, 71 deletions

diff --git a/src/libsyntax/parse/attr.rs b/src/libsyntax/parse/attr.rs
index 15344cef1db..2ae3236cd5a 100644
--- a/src/libsyntax/parse/attr.rs
+++ b/src/libsyntax/parse/attr.rs
@@ -18,23 +18,43 @@ use parse::token;
 use parse::parser::{Parser, TokenType};
 use ptr::P;
 
+#[derive(PartialEq, Eq, Debug)]
+enum InnerAttributeParsePolicy<'a> {
+    Permitted,
+    NotPermitted { reason: &'a str },
+}
+
+const DEFAULT_UNEXPECTED_INNER_ATTR_ERR_MSG: &'static str = "an inner attribute is not \
+                                                             permitted in this context";
+
 impl<'a> Parser<'a> {
     /// Parse attributes that appear before an item
     pub fn parse_outer_attributes(&mut self) -> PResult<'a, Vec<ast::Attribute>> {
         let mut attrs: Vec<ast::Attribute> = Vec::new();
+        let mut just_parsed_doc_comment = false;
         loop {
             debug!("parse_outer_attributes: self.token={:?}", self.token);
             match self.token {
                 token::Pound => {
-                    attrs.push(self.parse_attribute(false)?);
+                    let inner_error_reason = if just_parsed_doc_comment {
+                        "an inner attribute is not permitted following an outer doc comment"
+                    } else if !attrs.is_empty() {
+                        "an inner attribute is not permitted following an outer attribute"
+                    } else {
+                        DEFAULT_UNEXPECTED_INNER_ATTR_ERR_MSG
+                    };
+                    let inner_parse_policy =
+                        InnerAttributeParsePolicy::NotPermitted { reason: inner_error_reason };
+                    attrs.push(self.parse_attribute_with_inner_parse_policy(inner_parse_policy)?);
+                    just_parsed_doc_comment = false;
                 }
                 token::DocComment(s) => {
                     let attr = ::attr::mk_sugared_doc_attr(
-                    attr::mk_attr_id(),
-                    self.id_to_interned_str(ast::Ident::with_empty_ctxt(s)),
-                    self.span.lo,
-                    self.span.hi
-                );
+                        attr::mk_attr_id(),
+                        self.id_to_interned_str(ast::Ident::with_empty_ctxt(s)),
+                        self.span.lo,
+                        self.span.hi
+                    );
                     if attr.node.style != ast::AttrStyle::Outer {
                         let mut err = self.fatal("expected outer doc comment");
                         err.note("inner doc comments like this (starting with \
@@ -43,6 +63,7 @@ impl<'a> Parser<'a> {
                     }
                     attrs.push(attr);
                     self.bump();
+                    just_parsed_doc_comment = true;
                 }
                 _ => break,
             }
@@ -55,26 +76,46 @@ impl<'a> Parser<'a> {
     /// If permit_inner is true, then a leading `!` indicates an inner
     /// attribute
     pub fn parse_attribute(&mut self, permit_inner: bool) -> PResult<'a, ast::Attribute> {
-        debug!("parse_attributes: permit_inner={:?} self.token={:?}",
+        debug!("parse_attribute: permit_inner={:?} self.token={:?}",
                permit_inner,
                self.token);
+        let inner_parse_policy = if permit_inner {
+            InnerAttributeParsePolicy::Permitted
+        } else {
+            InnerAttributeParsePolicy::NotPermitted
+                { reason: DEFAULT_UNEXPECTED_INNER_ATTR_ERR_MSG }
+        };
+        self.parse_attribute_with_inner_parse_policy(inner_parse_policy)
+    }
+
+    /// The same as `parse_attribute`, except it takes in an `InnerAttributeParsePolicy`
+    /// that prescribes how to handle inner attributes.
+    fn parse_attribute_with_inner_parse_policy(&mut self,
+                                               inner_parse_policy: InnerAttributeParsePolicy)
+                                               -> PResult<'a, ast::Attribute> {
+        debug!("parse_attribute_with_inner_parse_policy: inner_parse_policy={:?} self.token={:?}",
+               inner_parse_policy,
+               self.token);
         let (span, value, mut style) = match self.token {
             token::Pound => {
                 let lo = self.span.lo;
                 self.bump();
 
-                if permit_inner {
+                if inner_parse_policy == InnerAttributeParsePolicy::Permitted {
                     self.expected_tokens.push(TokenType::Token(token::Not));
                 }
                 let style = if self.token == token::Not {
                     self.bump();
-                    if !permit_inner {
+                    if let InnerAttributeParsePolicy::NotPermitted { reason } = inner_parse_policy
+                    {
                         let span = self.span;
                         self.diagnostic()
-                            .struct_span_err(span,
-                                             "an inner attribute is not permitted in this context")
-                            .help("place inner attribute at the top of the module or \
-                                   block")
+                            .struct_span_err(span, reason)
+                            .note("inner attributes and doc comments, like `#![no_std]` or \
+                                   `//! My crate`, annotate the item enclosing them, and are \
+                                   usually found at the beginning of source files. Outer \
+                                   attributes and doc comments, like `#[test]` and
+                                   `/// My function`, annotate the item following them.")
                             .emit()
                     }
                     ast::AttrStyle::Inner
@@ -95,7 +136,8 @@ impl<'a> Parser<'a> {
             }
         };
 
-        if permit_inner && self.token == token::Semi {
+        if inner_parse_policy == InnerAttributeParsePolicy::Permitted &&
+           self.token == token::Semi {
             self.bump();
             self.span_warn(span,
                            "this inner attribute syntax is deprecated. The new syntax is \
diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs
index 77b5c10899a..5ea1d6be9fe 100644
--- a/src/libsyntax/parse/lexer/mod.rs
+++ b/src/libsyntax/parse/lexer/mod.rs
@@ -1686,7 +1686,7 @@ mod tests {
         // FIXME (#22405): Replace `Box::new` with `box` here when/if possible.
         let emitter = errors::emitter::EmitterWriter::new(Box::new(io::sink()),
                                                 None,
-                                                cm,
+                                                Some(cm),
                                                 errors::snippet::FormatMode::EnvironmentSelected);
         errors::Handler::with_emitter(true, false, Box::new(emitter))
     }
diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs
index 9502bc48a3e..2147e8ec2eb 100644
--- a/src/libsyntax/parse/mod.rs
+++ b/src/libsyntax/parse/mod.rs
@@ -50,7 +50,11 @@ pub struct ParseSess {
 impl ParseSess {
     pub fn new() -> ParseSess {
         let cm = Rc::new(CodeMap::new());
-        let handler = Handler::with_tty_emitter(ColorConfig::Auto, None, true, false, cm.clone());
+        let handler = Handler::with_tty_emitter(ColorConfig::Auto,
+                                                None,
+                                                true,
+                                                false,
+                                                Some(cm.clone()));
         ParseSess::with_span_handler(handler, cm)
     }
 
@@ -224,10 +228,18 @@ pub fn filemap_to_parser<'a>(sess: &'a ParseSess,
 // compiler expands into it
 pub fn new_parser_from_tts<'a>(sess: &'a ParseSess,
                                cfg: ast::CrateConfig,
-                               tts: Vec<tokenstream::TokenTree>) -> Parser<'a> {
+                               tts: Vec<tokenstream::TokenTree>)
+                               -> Parser<'a> {
     tts_to_parser(sess, tts, cfg)
 }
 
+pub fn new_parser_from_ts<'a>(sess: &'a ParseSess,
+                              cfg: ast::CrateConfig,
+                              ts: tokenstream::TokenStream)
+                              -> Parser<'a> {
+    tts_to_parser(sess, ts.tts, cfg)
+}
+
 
 // base abstractions
 
diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs
index 4656ba03e21..125f1abb062 100644
--- a/src/libsyntax/parse/parser.rs
+++ b/src/libsyntax/parse/parser.rs
@@ -3789,13 +3789,8 @@ impl<'a> Parser<'a> {
 
     /// Parse a statement. This stops just before trailing semicolons on everything but items.
     /// e.g. a `StmtKind::Semi` parses to a `StmtKind::Expr`, leaving the trailing `;` unconsumed.
-    ///
-    /// Also, if a macro begins an expression statement, this only parses the macro. For example,
-    /// ```rust
-    /// vec![1].into_iter(); //< `parse_stmt` only parses the "vec![1]"
-    /// ```
     pub fn parse_stmt(&mut self) -> PResult<'a, Option<Stmt>> {
-        Ok(self.parse_stmt_())
+        Ok(self.parse_stmt_(true))
     }
 
     // Eat tokens until we can be relatively sure we reached the end of the
@@ -3859,15 +3854,15 @@ impl<'a> Parser<'a> {
         }
     }
 
-    fn parse_stmt_(&mut self) -> Option<Stmt> {
-        self.parse_stmt_without_recovery().unwrap_or_else(|mut e| {
+    fn parse_stmt_(&mut self, macro_expanded: bool) -> Option<Stmt> {
+        self.parse_stmt_without_recovery(macro_expanded).unwrap_or_else(|mut e| {
             e.emit();
             self.recover_stmt_(SemiColonMode::Break);
             None
         })
     }
 
-    fn parse_stmt_without_recovery(&mut self) -> PResult<'a, Option<Stmt>> {
+    fn parse_stmt_without_recovery(&mut self, macro_expanded: bool) -> PResult<'a, Option<Stmt>> {
         maybe_whole!(Some deref self, NtStmt);
 
         let attrs = self.parse_outer_attributes()?;
@@ -3930,10 +3925,34 @@ impl<'a> Parser<'a> {
 
             if id.name == keywords::Invalid.name() {
                 let mac = spanned(lo, hi, Mac_ { path: pth, tts: tts });
+                let node = if delim == token::Brace ||
+                              self.token == token::Semi || self.token == token::Eof {
+                    StmtKind::Mac(P((mac, style, attrs.into())))
+                }
+                // We used to incorrectly stop parsing macro-expanded statements here.
+                // If the next token will be an error anyway but could have parsed with the
+                // earlier behavior, stop parsing here and emit a warning to avoid breakage.
+                else if macro_expanded && self.token.can_begin_expr() && match self.token {
+                    // These can continue an expression, so we can't stop parsing and warn.
+                    token::OpenDelim(token::Paren) | token::OpenDelim(token::Bracket) |
+                    token::BinOp(token::Minus) | token::BinOp(token::Star) |
+                    token::BinOp(token::And) | token::BinOp(token::Or) |
+                    token::AndAnd | token::OrOr |
+                    token::DotDot | token::DotDotDot => false,
+                    _ => true,
+                } {
+                    self.warn_missing_semicolon();
+                    StmtKind::Mac(P((mac, style, attrs.into())))
+                } else {
+                    let e = self.mk_mac_expr(lo, hi, mac.node, ThinVec::new());
+                    let e = self.parse_dot_or_call_expr_with(e, lo, attrs.into())?;
+                    let e = self.parse_assoc_expr_with(0, LhsExpr::AlreadyParsed(e))?;
+                    StmtKind::Expr(e)
+                };
                 Stmt {
                     id: ast::DUMMY_NODE_ID,
-                    node: StmtKind::Mac(P((mac, style, attrs.into()))),
                     span: mk_sp(lo, hi),
+                    node: node,
                 }
             } else {
                 // if it has a special ident, it's definitely an item
@@ -4061,49 +4080,12 @@ impl<'a> Parser<'a> {
     }
 
     /// Parse a statement, including the trailing semicolon.
-    /// This parses expression statements that begin with macros correctly (c.f. `parse_stmt`).
     pub fn parse_full_stmt(&mut self, macro_expanded: bool) -> PResult<'a, Option<Stmt>> {
-        let mut stmt = match self.parse_stmt_() {
+        let mut stmt = match self.parse_stmt_(macro_expanded) {
             Some(stmt) => stmt,
             None => return Ok(None),
         };
 
-        if let StmtKind::Mac(mac) = stmt.node {
-            if mac.1 != MacStmtStyle::NoBraces ||
-               self.token == token::Semi || self.token == token::Eof {
-                stmt.node = StmtKind::Mac(mac);
-            } else {
-                // We used to incorrectly stop parsing macro-expanded statements here.
-                // If the next token will be an error anyway but could have parsed with the
-                // earlier behavior, stop parsing here and emit a warning to avoid breakage.
-                if macro_expanded && self.token.can_begin_expr() && match self.token {
-                    // These tokens can continue an expression, so we can't stop parsing and warn.
-                    token::OpenDelim(token::Paren) | token::OpenDelim(token::Bracket) |
-                    token::BinOp(token::Minus) | token::BinOp(token::Star) |
-                    token::BinOp(token::And) | token::BinOp(token::Or) |
-                    token::AndAnd | token::OrOr |
-                    token::DotDot | token::DotDotDot => false,
-                    _ => true,
-                } {
-                    self.warn_missing_semicolon();
-                    stmt.node = StmtKind::Mac(mac);
-                    return Ok(Some(stmt));
-                }
-
-                let (mac, _style, attrs) = mac.unwrap();
-                let e = self.mk_mac_expr(stmt.span.lo, stmt.span.hi, mac.node, ThinVec::new());
-                let e = self.parse_dot_or_call_expr_with(e, stmt.span.lo, attrs)?;
-                let e = self.parse_assoc_expr_with(0, LhsExpr::AlreadyParsed(e))?;
-                stmt.node = StmtKind::Expr(e);
-            }
-        }
-
-        stmt = self.handle_trailing_semicolon(stmt, macro_expanded)?;
-        Ok(Some(stmt))
-    }
-
-    fn handle_trailing_semicolon(&mut self, mut stmt: Stmt, macro_expanded: bool)
-                                 -> PResult<'a, Stmt> {
         match stmt.node {
             StmtKind::Expr(ref expr) if self.token != token::Eof => {
                 // expression without semicolon
@@ -4133,7 +4115,7 @@ impl<'a> Parser<'a> {
         }
 
         stmt.span.hi = self.last_span.hi;
-        Ok(stmt)
+        Ok(Some(stmt))
     }
 
     fn warn_missing_semicolon(&self) {
diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs
index fe9d3ef7c23..f0a6f8edeec 100644
--- a/src/libsyntax/parse/token.rs
+++ b/src/libsyntax/parse/token.rs
@@ -633,8 +633,3 @@ pub fn fresh_name(src: ast::Ident) -> ast::Name {
     /*let num = rand::thread_rng().gen_uint_range(0,0xffff);
     gensym(format!("{}_{}",ident_to_string(src),num))*/
 }
-
-// create a fresh mark.
-pub fn fresh_mark() -> ast::Mrk {
-    gensym("mark").0
-}