Auto merge of #49719 - mark-i-m:no_sep, r=petrochenkov

Update `?` repetition disambiguation.

**Do not merge** (yet)

This is a test implementation of some ideas from discussion in https://github.com/rust-lang/rust/issues/48075 . This PR
- disallows `?` repetition from taking a separator, since the separator is never used.
- disallows the use of `?` as a separator. This allows patterns like `$(a)?+` to match `+` and `a+` rather than `a?a?a`. This is a _breaking change_, but maybe that's ok? Perhaps a crater run is the right approach?

cc @durka @alexreg @nikomatsakis

1 files changed, 22 insertions, 67 deletions

diff --git a/src/libsyntax/ext/tt/quoted.rs b/src/libsyntax/ext/tt/quoted.rs
index 01b971976a7..77c6afa1c64 100644
--- a/src/libsyntax/ext/tt/quoted.rs
+++ b/src/libsyntax/ext/tt/quoted.rs
@@ -386,72 +386,26 @@ where
 {
     // We basically look at two token trees here, denoted as #1 and #2 below
     let span = match parse_kleene_op(input, span) {
-        // #1 is a `+` or `*` KleeneOp
-        //
-        // `?` is ambiguous: it could be a separator or a Kleene::ZeroOrOne, so we need to look
-        // ahead one more token to be sure.
-        Ok(Ok(op)) if op != KleeneOp::ZeroOrOne => return (None, op),
-
-        // #1 is `?` token, but it could be a Kleene::ZeroOrOne without a separator or it could
-        // be a `?` separator followed by any Kleene operator. We need to look ahead 1 token to
-        // find out which.
-        Ok(Ok(op)) => {
-            assert_eq!(op, KleeneOp::ZeroOrOne);
-
-            // Lookahead at #2. If it is a KleenOp, then #1 is a separator.
-            let is_1_sep = if let Some(&tokenstream::TokenTree::Token(_, ref tok2)) = input.peek() {
-                kleene_op(tok2).is_some()
-            } else {
-                false
-            };
-
-            if is_1_sep {
-                // #1 is a separator and #2 should be a KleepeOp::*
-                // (N.B. We need to advance the input iterator.)
-                match parse_kleene_op(input, span) {
-                    // #2 is a KleeneOp (this is the only valid option) :)
-                    Ok(Ok(op)) if op == KleeneOp::ZeroOrOne => {
-                        if !features.macro_at_most_once_rep
-                            && !attr::contains_name(attrs, "allow_internal_unstable")
-                        {
-                            let explain = feature_gate::EXPLAIN_MACRO_AT_MOST_ONCE_REP;
-                            emit_feature_err(
-                                sess,
-                                "macro_at_most_once_rep",
-                                span,
-                                GateIssue::Language,
-                                explain,
-                            );
-                        }
-                        return (Some(token::Question), op);
-                    }
-                    Ok(Ok(op)) => return (Some(token::Question), op),
-
-                    // #2 is a random token (this is an error) :(
-                    Ok(Err((_, span))) => span,
-
-                    // #2 is not even a token at all :(
-                    Err(span) => span,
-                }
-            } else {
-                if !features.macro_at_most_once_rep
-                    && !attr::contains_name(attrs, "allow_internal_unstable")
-                {
-                    let explain = feature_gate::EXPLAIN_MACRO_AT_MOST_ONCE_REP;
-                    emit_feature_err(
-                        sess,
-                        "macro_at_most_once_rep",
-                        span,
-                        GateIssue::Language,
-                        explain,
-                    );
-                }
-
-                // #2 is a random tree and #1 is KleeneOp::ZeroOrOne
-                return (None, op);
+        // #1 is any KleeneOp (`?`)
+        Ok(Ok(op)) if op == KleeneOp::ZeroOrOne => {
+            if !features.macro_at_most_once_rep
+                && !attr::contains_name(attrs, "allow_internal_unstable")
+            {
+                let explain = feature_gate::EXPLAIN_MACRO_AT_MOST_ONCE_REP;
+                emit_feature_err(
+                    sess,
+                    "macro_at_most_once_rep",
+                    span,
+                    GateIssue::Language,
+                    explain,
+                );
             }
+            return (None, op);
         }
 
+        // #1 is any KleeneOp (`+`, `*`)
+        Ok(Ok(op)) => return (None, op),
+
         // #1 is a separator followed by #2, a KleeneOp
         Ok(Err((tok, span))) => match parse_kleene_op(input, span) {
             // #2 is a KleeneOp :D
@@ -467,8 +421,11 @@ where
                         GateIssue::Language,
                         explain,
                     );
+                } else {
+                    sess.span_diagnostic
+                        .span_err(span, "`?` macro repetition does not allow a separator");
                 }
-                return (Some(tok), op);
+                return (None, op);
             }
             Ok(Ok(op)) => return (Some(tok), op),
 
@@ -483,9 +440,7 @@ where
         Err(span) => span,
     };
 
-    if !features.macro_at_most_once_rep
-        && !attr::contains_name(attrs, "allow_internal_unstable")
-    {
+    if !features.macro_at_most_once_rep && !attr::contains_name(attrs, "allow_internal_unstable") {
         sess.span_diagnostic
             .span_err(span, "expected one of: `*`, `+`, or `?`");
     } else {