syntax: Enforce attribute grammar in the parser

5 files changed, 53 insertions, 27 deletions

diff --git a/src/libsyntax/attr/mod.rs b/src/libsyntax/attr/mod.rs
index 879f555ba03..5857bd282f0 100644
--- a/src/libsyntax/attr/mod.rs
+++ b/src/libsyntax/attr/mod.rs
@@ -607,7 +607,7 @@ impl NestedMetaItemKind {
 }
 
 impl Lit {
-    fn tokens(&self) -> TokenStream {
+    crate fn tokens(&self) -> TokenStream {
         TokenTree::Token(self.span, self.node.token()).into()
     }
 }
@@ -794,7 +794,7 @@ pub fn inject(mut krate: ast::Crate, parse_sess: &ParseSess, attrs: &[String]) -
         );
 
         let start_span = parser.span;
-        let (path, tokens) = panictry!(parser.parse_path_and_tokens());
+        let (path, tokens) = panictry!(parser.parse_meta_item_unrestricted());
         let end_span = parser.span;
         if parser.token != token::Eof {
             parse_sess.span_diagnostic
diff --git a/src/libsyntax/config.rs b/src/libsyntax/config.rs
index 4fe78bf829a..b4e35a9d564 100644
--- a/src/libsyntax/config.rs
+++ b/src/libsyntax/config.rs
@@ -90,7 +90,7 @@ impl<'a> StripUnconfigured<'a> {
             let cfg = parser.parse_meta_item()?;
             parser.expect(&token::Comma)?;
             let lo = parser.span.lo();
-            let (path, tokens) = parser.parse_path_and_tokens()?;
+            let (path, tokens) = parser.parse_meta_item_unrestricted()?;
             parser.expect(&token::CloseDelim(token::Paren))?;
             Ok((cfg, path, tokens, parser.prev_span.with_lo(lo)))
         }) {
diff --git a/src/libsyntax/feature_gate.rs b/src/libsyntax/feature_gate.rs
index 56e69b9df9e..f837bead6a0 100644
--- a/src/libsyntax/feature_gate.rs
+++ b/src/libsyntax/feature_gate.rs
@@ -1526,27 +1526,29 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
             }
         }
 
-        match attr.parse_meta(self.context.parse_sess) {
-            Ok(meta) => {
-                // allow attr_literals in #[repr(align(x))] and #[repr(packed(n))]
-                let mut allow_attr_literal = false;
-                if attr.path == "repr" {
-                    if let Some(content) = meta.meta_item_list() {
-                        allow_attr_literal = content.iter().any(
-                            |c| c.check_name("align") || c.check_name("packed"));
+        if !self.context.features.unrestricted_attribute_tokens {
+            // Unfortunately, `parse_meta` cannot be called speculatively because it can report
+            // errors by itself, so we have to call it only if the feature is disabled.
+            match attr.parse_meta(self.context.parse_sess) {
+                Ok(meta) => {
+                    // allow attr_literals in #[repr(align(x))] and #[repr(packed(n))]
+                    let mut allow_attr_literal = false;
+                    if attr.path == "repr" {
+                        if let Some(content) = meta.meta_item_list() {
+                            allow_attr_literal = content.iter().any(
+                                |c| c.check_name("align") || c.check_name("packed"));
+                        }
                     }
-                }
 
-                if !allow_attr_literal && contains_novel_literal(&meta) {
-                    gate_feature_post!(&self, attr_literals, attr.span,
-                                    "non-string literals in attributes, or string \
-                                    literals in top-level positions, are experimental");
+                    if !allow_attr_literal && contains_novel_literal(&meta) {
+                        gate_feature_post!(&self, attr_literals, attr.span,
+                                        "non-string literals in attributes, or string \
+                                        literals in top-level positions, are experimental");
+                    }
+                }
+                Err(mut err) => {
+                    err.help("try enabling `#![feature(unrestricted_attribute_tokens)]`").emit()
                 }
-            }
-            Err(mut err) => {
-                err.cancel();
-                gate_feature_post!(&self, unrestricted_attribute_tokens, attr.span,
-                                    "arbitrary tokens in non-macro attributes are unstable");
             }
         }
     }
diff --git a/src/libsyntax/parse/attr.rs b/src/libsyntax/parse/attr.rs
index 4d59f64bb6b..b0136c3e18b 100644
--- a/src/libsyntax/parse/attr.rs
+++ b/src/libsyntax/parse/attr.rs
@@ -12,9 +12,9 @@ use attr;
 use ast;
 use codemap::respan;
 use parse::{SeqSep, PResult};
-use parse::token::{self, Nonterminal};
+use parse::token::{self, Nonterminal, DelimToken};
 use parse::parser::{Parser, TokenType, PathStyle};
-use tokenstream::TokenStream;
+use tokenstream::{TokenStream, TokenTree};
 
 #[derive(Debug)]
 enum InnerAttributeParsePolicy<'a> {
@@ -116,7 +116,7 @@ impl<'a> Parser<'a> {
                 };
 
                 self.expect(&token::OpenDelim(token::Bracket))?;
-                let (path, tokens) = self.parse_path_and_tokens()?;
+                let (path, tokens) = self.parse_meta_item_unrestricted()?;
                 self.expect(&token::CloseDelim(token::Bracket))?;
                 let hi = self.prev_span;
 
@@ -138,7 +138,16 @@ impl<'a> Parser<'a> {
         })
     }
 
-    crate fn parse_path_and_tokens(&mut self) -> PResult<'a, (ast::Path, TokenStream)> {
+    /// Parse an inner part of attribute - path and following tokens.
+    /// The tokens must be either a delimited token stream, or empty token stream,
+    /// or the "legacy" key-value form.
+    /// PATH `(` TOKEN_STREAM `)`
+    /// PATH `[` TOKEN_STREAM `]`
+    /// PATH `{` TOKEN_STREAM `}`
+    /// PATH
+    /// PATH `=` TOKEN_TREE
+    /// The delimiters or `=` are still put into the resulting token stream.
+    crate fn parse_meta_item_unrestricted(&mut self) -> PResult<'a, (ast::Path, TokenStream)> {
         let meta = match self.token {
             token::Interpolated(ref nt) => match nt.0 {
                 Nonterminal::NtMeta(ref meta) => Some(meta.clone()),
@@ -150,7 +159,22 @@ impl<'a> Parser<'a> {
             self.bump();
             (meta.ident, meta.node.tokens(meta.span))
         } else {
-            (self.parse_path(PathStyle::Mod)?, self.parse_tokens())
+            let path = self.parse_path(PathStyle::Mod)?;
+            let tokens = if self.check(&token::OpenDelim(DelimToken::Paren)) ||
+               self.check(&token::OpenDelim(DelimToken::Bracket)) ||
+               self.check(&token::OpenDelim(DelimToken::Brace)) {
+                   self.parse_token_tree().into()
+            } else if self.eat(&token::Eq) {
+                let eq = TokenTree::Token(self.prev_span, token::Eq);
+                let tree = match self.token {
+                    token::CloseDelim(_) | token::Eof => self.unexpected()?,
+                    _ => self.parse_token_tree(),
+                };
+                TokenStream::concat(vec![eq.into(), tree.into()])
+            } else {
+                TokenStream::empty()
+            };
+            (path, tokens)
         })
     }
 
diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs
index 0e45cacaf38..b1e1cdee2ee 100644
--- a/src/libsyntax/parse/parser.rs
+++ b/src/libsyntax/parse/parser.rs
@@ -825,7 +825,7 @@ impl<'a> Parser<'a> {
     ///
     /// This method will automatically add `tok` to `expected_tokens` if `tok` is not
     /// encountered.
-    fn check(&mut self, tok: &token::Token) -> bool {
+    crate fn check(&mut self, tok: &token::Token) -> bool {
         let is_present = self.token == *tok;
         if !is_present { self.expected_tokens.push(TokenType::Token(tok.clone())); }
         is_present