From 39fbcfb277c45a6f1eeac31c171395173def4bdb Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Sat, 18 May 2019 08:23:52 +0200 Subject: Support ? Kleene operator in 2015. --- src/libsyntax/ext/tt/quoted.rs | 171 +---------------------------------------- 1 file changed, 4 insertions(+), 167 deletions(-) (limited to 'src/libsyntax') diff --git a/src/libsyntax/ext/tt/quoted.rs b/src/libsyntax/ext/tt/quoted.rs index 707fb65bcc5..61dc39c4459 100644 --- a/src/libsyntax/ext/tt/quoted.rs +++ b/src/libsyntax/ext/tt/quoted.rs @@ -1,5 +1,4 @@ use crate::ast::NodeId; -use crate::early_buffered_lints::BufferedEarlyLintId; use crate::ext::tt::macro_parser; use crate::feature_gate::Features; use crate::parse::token::{self, Token, TokenKind}; @@ -287,16 +286,7 @@ where macro_node_id, ); // Get the Kleene operator and optional separator - let (separator, op) = - parse_sep_and_kleene_op( - trees, - span.entire(), - sess, - features, - attrs, - edition, - macro_node_id, - ); + let (separator, op) = parse_sep_and_kleene_op(trees, span.entire(), sess); // Count the number of captured "names" (i.e., named metavars) let name_captures = macro_parser::count_names(&sequence); TokenTree::Sequence( @@ -403,164 +393,11 @@ where /// session `sess`. If the next one (or possibly two) tokens in `input` correspond to a Kleene /// operator and separator, then a tuple with `(separator, KleeneOp)` is returned. Otherwise, an /// error with the appropriate span is emitted to `sess` and a dummy value is returned. -/// -/// N.B., in the 2015 edition, `*` and `+` are the only Kleene operators, and `?` is a separator. -/// In the 2018 edition however, `?` is a Kleene operator, and not a separator. -fn parse_sep_and_kleene_op( - input: &mut Peekable, - span: Span, - sess: &ParseSess, - features: &Features, - attrs: &[ast::Attribute], - edition: Edition, - macro_node_id: NodeId, -) -> (Option, KleeneOp) -where - I: Iterator, -{ - match edition { - Edition::Edition2015 => parse_sep_and_kleene_op_2015( - input, - span, - sess, - features, - attrs, - macro_node_id, - ), - Edition::Edition2018 => parse_sep_and_kleene_op_2018(input, span, sess, features, attrs), - } -} - -// `?` is a separator (with a migration warning) and never a KleeneOp. -fn parse_sep_and_kleene_op_2015( - input: &mut Peekable, - span: Span, - sess: &ParseSess, - _features: &Features, - _attrs: &[ast::Attribute], - macro_node_id: NodeId, -) -> (Option, KleeneOp) -where - I: Iterator, -{ - // We basically look at two token trees here, denoted as #1 and #2 below - let span = match parse_kleene_op(input, span) { - // #1 is a `+` or `*` KleeneOp - // - // `?` is ambiguous: it could be a separator (warning) or a Kleene::ZeroOrOne (error), so - // we need to look ahead one more token to be sure. - Ok(Ok((op, _))) if op != KleeneOp::ZeroOrOne => return (None, op), - - // #1 is `?` token, but it could be a Kleene::ZeroOrOne (error in 2015) without a separator - // or it could be a `?` separator followed by any Kleene operator. We need to look ahead 1 - // token to find out which. - Ok(Ok((op, op1_span))) => { - assert_eq!(op, KleeneOp::ZeroOrOne); - - // Lookahead at #2. If it is a KleenOp, then #1 is a separator. - let is_1_sep = if let Some(tokenstream::TokenTree::Token(tok2)) = input.peek() { - kleene_op(tok2).is_some() - } else { - false - }; - - if is_1_sep { - // #1 is a separator and #2 should be a KleepeOp. - // (N.B. We need to advance the input iterator.) - match parse_kleene_op(input, span) { - // #2 is `?`, which is not allowed as a Kleene op in 2015 edition, - // but is allowed in the 2018 edition. - Ok(Ok((op, op2_span))) if op == KleeneOp::ZeroOrOne => { - sess.span_diagnostic - .struct_span_err(op2_span, "expected `*` or `+`") - .note("`?` is not a macro repetition operator in the 2015 edition, \ - but is accepted in the 2018 edition") - .emit(); - - // Return a dummy - return (None, KleeneOp::ZeroOrMore); - } - - // #2 is a Kleene op, which is the only valid option - Ok(Ok((op, _))) => { - // Warn that `?` as a separator will be deprecated - sess.buffer_lint( - BufferedEarlyLintId::QuestionMarkMacroSep, - op1_span, - macro_node_id, - "using `?` as a separator is deprecated and will be \ - a hard error in an upcoming edition", - ); - - return (Some(Token::new(token::Question, op1_span)), op); - } - - // #2 is a random token (this is an error) :( - Ok(Err(_)) => op1_span, - - // #2 is not even a token at all :( - Err(_) => op1_span, - } - } else { - // `?` is not allowed as a Kleene op in 2015, - // but is allowed in the 2018 edition - sess.span_diagnostic - .struct_span_err(op1_span, "expected `*` or `+`") - .note("`?` is not a macro repetition operator in the 2015 edition, \ - but is accepted in the 2018 edition") - .emit(); - - // Return a dummy - return (None, KleeneOp::ZeroOrMore); - } - } - - // #1 is a separator followed by #2, a KleeneOp - Ok(Err(token)) => match parse_kleene_op(input, token.span) { - // #2 is a `?`, which is not allowed as a Kleene op in 2015 edition, - // but is allowed in the 2018 edition - Ok(Ok((op, op2_span))) if op == KleeneOp::ZeroOrOne => { - sess.span_diagnostic - .struct_span_err(op2_span, "expected `*` or `+`") - .note("`?` is not a macro repetition operator in the 2015 edition, \ - but is accepted in the 2018 edition") - .emit(); - - // Return a dummy - return (None, KleeneOp::ZeroOrMore); - } - - // #2 is a KleeneOp :D - Ok(Ok((op, _))) => return (Some(token), op), - - // #2 is a random token :( - Ok(Err(token)) => token.span, - - // #2 is not a token at all :( - Err(span) => span, - }, - - // #1 is not a token - Err(span) => span, - }; - - sess.span_diagnostic.span_err(span, "expected `*` or `+`"); - - // Return a dummy - (None, KleeneOp::ZeroOrMore) -} - -// `?` is a Kleene op, not a separator -fn parse_sep_and_kleene_op_2018( - input: &mut Peekable, +fn parse_sep_and_kleene_op( + input: &mut Peekable>, span: Span, sess: &ParseSess, - _features: &Features, - _attrs: &[ast::Attribute], -) -> (Option, KleeneOp) -where - I: Iterator, -{ +) -> (Option, KleeneOp) { // We basically look at two token trees here, denoted as #1 and #2 below let span = match parse_kleene_op(input, span) { // #1 is a `?` (needs feature gate) -- cgit 1.4.1-3-g733a5 From ab7d75d89bbca8968c1edd5a69d921b306ca0cba Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Wed, 22 May 2019 02:38:19 +0200 Subject: Cleanups in parse_sep_and_kleene_op. --- src/libsyntax/ext/tt/quoted.rs | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) (limited to 'src/libsyntax') diff --git a/src/libsyntax/ext/tt/quoted.rs b/src/libsyntax/ext/tt/quoted.rs index 61dc39c4459..8475ea8000b 100644 --- a/src/libsyntax/ext/tt/quoted.rs +++ b/src/libsyntax/ext/tt/quoted.rs @@ -400,18 +400,13 @@ fn parse_sep_and_kleene_op( ) -> (Option, KleeneOp) { // We basically look at two token trees here, denoted as #1 and #2 below let span = match parse_kleene_op(input, span) { - // #1 is a `?` (needs feature gate) - Ok(Ok((op, _op1_span))) if op == KleeneOp::ZeroOrOne => { - return (None, op); - } - - // #1 is a `+` or `*` KleeneOp + // #1 is a `?`, `+`, or `*` KleeneOp Ok(Ok((op, _))) => return (None, op), // #1 is a separator followed by #2, a KleeneOp Ok(Err(token)) => match parse_kleene_op(input, token.span) { // #2 is the `?` Kleene op, which does not take a separator (error) - Ok(Ok((op, _op2_span))) if op == KleeneOp::ZeroOrOne => { + Ok(Ok((KleeneOp::ZeroOrOne, _))) => { // Error! sess.span_diagnostic.span_err( token.span, @@ -425,11 +420,8 @@ fn parse_sep_and_kleene_op( // #2 is a KleeneOp :D Ok(Ok((op, _))) => return (Some(token), op), - // #2 is a random token :( - Ok(Err(token)) => token.span, - - // #2 is not a token at all :( - Err(span) => span, + // #2 is a random token or not a token at all :( + Ok(Err(Token { span, .. })) | Err(span) => span, }, // #1 is not a token -- cgit 1.4.1-3-g733a5 From f86719a111ab4dff770914097af1d6ba76d2f97d Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Wed, 22 May 2019 02:43:46 +0200 Subject: Some more cleanup in libsyntax::ext::tt::quoted --- src/libsyntax/ext/tt/quoted.rs | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) (limited to 'src/libsyntax') diff --git a/src/libsyntax/ext/tt/quoted.rs b/src/libsyntax/ext/tt/quoted.rs index 8475ea8000b..4920ddfbfe5 100644 --- a/src/libsyntax/ext/tt/quoted.rs +++ b/src/libsyntax/ext/tt/quoted.rs @@ -249,19 +249,16 @@ pub fn parse( /// - `sess`: the parsing session. Any errors will be emitted to this session. /// - `features`, `attrs`: language feature flags and attributes so that we know whether to use /// unstable features or not. -fn parse_tree( +fn parse_tree( tree: tokenstream::TokenTree, - trees: &mut Peekable, + trees: &mut Peekable>, expect_matchers: bool, sess: &ParseSess, features: &Features, attrs: &[ast::Attribute], edition: Edition, macro_node_id: NodeId, -) -> TokenTree -where - I: Iterator, -{ +) -> TokenTree { // Depending on what `tree` is, we could be parsing different parts of a macro match tree { // `tree` is a `$` token. Look at the next token in `trees` @@ -365,10 +362,10 @@ fn kleene_op(token: &Token) -> Option { /// - Ok(Ok((op, span))) if the next token tree is a KleeneOp /// - Ok(Err(tok, span)) if the next token tree is a token but not a KleeneOp /// - Err(span) if the next token tree is not a token -fn parse_kleene_op(input: &mut I, span: Span) -> Result, Span> -where - I: Iterator, -{ +fn parse_kleene_op( + input: &mut impl Iterator, + span: Span, +) -> Result, Span> { match input.next() { Some(tokenstream::TokenTree::Token(token)) => match kleene_op(&token) { Some(op) => Ok(Ok((op, token.span))), @@ -378,7 +375,7 @@ where .as_ref() .map(tokenstream::TokenTree::span) .unwrap_or(span)), - } + } } /// Attempt to parse a single Kleene star, possibly with a separator. -- cgit 1.4.1-3-g733a5 From 3ba82f7cd8f74d73ebbdcd45aefc761f518f782d Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Sat, 8 Jun 2019 01:39:29 +0200 Subject: pacify tidy. --- src/libsyntax/ext/tt/quoted.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libsyntax') diff --git a/src/libsyntax/ext/tt/quoted.rs b/src/libsyntax/ext/tt/quoted.rs index 4920ddfbfe5..b52e3b71505 100644 --- a/src/libsyntax/ext/tt/quoted.rs +++ b/src/libsyntax/ext/tt/quoted.rs @@ -375,7 +375,7 @@ fn parse_kleene_op( .as_ref() .map(tokenstream::TokenTree::span) .unwrap_or(span)), - } + } } /// Attempt to parse a single Kleene star, possibly with a separator. -- cgit 1.4.1-3-g733a5