Auto merge of #57367 - petrochenkov:unrestab, r=Centril

Stabilize `unrestricted_attribute_tokens`

In accordance with a plan described in https://internals.rust-lang.org/t/unrestricted-attribute-tokens-feature-status/8561/3.

Delimited non-macro non-builtin attributes now support the same syntax as macro attributes:
```
PATH
PATH `(` TOKEN_STREAM `)`
PATH `[` TOKEN_STREAM `]`
PATH `{` TOKEN_STREAM `}`
```
Such attributes mostly serve as inert proc macro helpers or tool attributes.
To some extent these attributes are de-facto stable due to a hole in feature gate checking (feature gating is done too late - after macro expansion.)
So if macro *removes* such helper attributes during expansion (and it must remove them, unless it's a derive macro), then the code will work on stable.

Key-value non-macro non-builtin attributes are now restricted to bare minimum required to support what we support on stable - unsuffixed literals (https://github.com/rust-lang/rust/issues/34981).
```
PATH `=` LITERAL
```
(Key-value macro attributes are not supported at all right now.)
Crater run in https://github.com/rust-lang/rust/pull/57321 found no regressions for this change.
There are multiple possible ways to extend key-value attributes (https://github.com/rust-lang/rust/pull/57321#issuecomment-451574065), but I'd expect an RFC for that and it's not a pressing enough issue to block stabilization of delimited attributes.

Built-in attributes are still restricted to the "classic" meta-item syntax, nothing changes here.
https://github.com/rust-lang/rust/pull/57321 goes further and adds some additional restrictions (more consistent input checking) to built-in attributes.

Closes https://github.com/rust-lang/rust/issues/55208

3 files changed, 22 insertions, 16 deletions

diff --git a/src/libsyntax/feature_gate.rs b/src/libsyntax/feature_gate.rs
index c9b441193b7..cc1953e69d4 100644
--- a/src/libsyntax/feature_gate.rs
+++ b/src/libsyntax/feature_gate.rs
@@ -21,8 +21,9 @@ use crate::early_buffered_lints::BufferedEarlyLintId;
 use crate::source_map::Spanned;
 use crate::edition::{ALL_EDITIONS, Edition};
 use crate::visit::{self, FnKind, Visitor};
-use crate::parse::ParseSess;
+use crate::parse::{token, ParseSess};
 use crate::symbol::Symbol;
+use crate::tokenstream::TokenTree;
 
 use errors::{DiagnosticBuilder, Handler};
 use rustc_data_structures::fx::FxHashMap;
@@ -431,9 +432,6 @@ declare_features! (
     // Added for testing E0705; perma-unstable.
     (active, test_2018_feature, "1.31.0", Some(0), Some(Edition::Edition2018)),
 
-    // support for arbitrary delimited token streams in non-macro attributes
-    (active, unrestricted_attribute_tokens, "1.30.0", Some(55208), None),
-
     // Allows unsized rvalues at arguments and parameters.
     (active, unsized_locals, "1.30.0", Some(48055), None),
 
@@ -700,6 +698,8 @@ declare_features! (
     (accepted, cfg_target_vendor, "1.33.0", Some(29718), None),
     // `extern crate self as foo;` puts local crate root into extern prelude under name `foo`.
     (accepted, extern_crate_self, "1.34.0", Some(56409), None),
+    // support for arbitrary delimited token streams in non-macro attributes
+    (accepted, unrestricted_attribute_tokens, "1.34.0", Some(55208), None),
 );
 
 // If you change this, please modify `src/doc/unstable-book` as well. You must
@@ -1660,13 +1660,9 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
 
         match BUILTIN_ATTRIBUTES.iter().find(|(name, ..)| attr.path == name) {
             Some(&(name, _, template, _)) => self.check_builtin_attribute(attr, name, template),
-            None => if !self.context.features.unrestricted_attribute_tokens {
-                // Unfortunately, `parse_meta` cannot be called speculatively
-                // because it can report errors by itself, so we have to call it
-                // only if the feature is disabled.
-                if let Err(mut err) = attr.parse_meta(self.context.parse_sess) {
-                    err.help("try enabling `#![feature(unrestricted_attribute_tokens)]`").emit()
-                }
+            None => if let Some(TokenTree::Token(_, token::Eq)) = attr.tokens.trees().next() {
+                // All key-value attributes are restricted to meta-item syntax.
+                attr.parse_meta(self.context.parse_sess).map_err(|mut err| err.emit()).ok();
             }
         }
     }
diff --git a/src/libsyntax/parse/attr.rs b/src/libsyntax/parse/attr.rs
index 9020c8c6a2d..e7937f57002 100644
--- a/src/libsyntax/parse/attr.rs
+++ b/src/libsyntax/parse/attr.rs
@@ -158,11 +158,21 @@ impl<'a> Parser<'a> {
                    self.parse_token_tree().into()
             } else if self.eat(&token::Eq) {
                 let eq = TokenTree::Token(self.prev_span, token::Eq);
-                let tree = match self.token {
-                    token::CloseDelim(_) | token::Eof => self.unexpected()?,
-                    _ => self.parse_token_tree(),
+                let mut is_interpolated_expr = false;
+                if let token::Interpolated(nt) = &self.token {
+                    if let token::NtExpr(..) = **nt {
+                        is_interpolated_expr = true;
+                    }
+                }
+                let tokens = if is_interpolated_expr {
+                    // We need to accept arbitrary interpolated expressions to continue
+                    // supporting things like `doc = $expr` that work on stable.
+                    // Non-literal interpolated expressions are rejected after expansion.
+                    self.parse_token_tree().into()
+                } else {
+                    self.parse_unsuffixed_lit()?.tokens()
                 };
-                TokenStream::new(vec![eq.into(), tree.into()])
+                TokenStream::from_streams(vec![eq.into(), tokens])
             } else {
                 TokenStream::empty()
             };
diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs
index 283679e758b..4ce308d015c 100644
--- a/src/libsyntax/tokenstream.rs
+++ b/src/libsyntax/tokenstream.rs
@@ -254,7 +254,7 @@ impl TokenStream {
         }
     }
 
-    fn from_streams(mut streams: Vec<TokenStream>) -> TokenStream {
+    pub(crate) fn from_streams(mut streams: Vec<TokenStream>) -> TokenStream {
         match streams.len() {
             0 => TokenStream::empty(),
             1 => streams.pop().unwrap(),