Auto merge of #53902 - dtolnay:group, r=petrochenkov

proc_macro::Group::span_open and span_close

Before this addition, every delimited group like `(`...`)` `[`...`]` `{`...`}` has only a single Span that covers the full source location from opening delimiter to closing delimiter. This makes it impossible for a procedural macro to trigger an error pointing to just the opening or closing delimiter. The Rust compiler does not seem to have the same limitation:

```rust
mod m {
    type T =
}
```

```console
error: expected type, found `}`
 --> src/main.rs:3:1
  |
3 | }
  | ^
```

On that same input, a procedural macro would be forced to trigger the error on the last token inside the block, on the entire block, or on the next token after the block, none of which is really what you want for an error like above.

This commit adds `group.span_open()` and `group.span_close()` which access the Span associated with just the opening delimiter and just the closing delimiter of the group. Relevant to Syn as we implement real error messages for when parsing fails in a procedural macro: https://github.com/dtolnay/syn/issues/476.

```diff
  impl Group {
      fn span(&self) -> Span;
+     fn span_open(&self) -> Span;
+     fn span_close(&self) -> Span;
  }
```

Fixes #48187
r? @alexcrichton

5 files changed, 57 insertions, 52 deletions

diff --git a/src/libsyntax/ext/quote.rs b/src/libsyntax/ext/quote.rs
index 13a139deea4..62bc9fae3b5 100644
--- a/src/libsyntax/ext/quote.rs
+++ b/src/libsyntax/ext/quote.rs
@@ -10,14 +10,14 @@
 
 use ast::{self, Arg, Arm, Block, Expr, Item, Pat, Stmt, Ty};
 use source_map::respan;
-use syntax_pos::Span;
+use syntax_pos::{Span, DUMMY_SP};
 use ext::base::ExtCtxt;
 use ext::base;
 use ext::build::AstBuilder;
 use parse::parser::{Parser, PathStyle};
 use parse::token;
 use ptr::P;
-use tokenstream::{TokenStream, TokenTree};
+use tokenstream::{DelimSpan, TokenStream, TokenTree};
 
 /// Quasiquoting works via token trees.
 ///
@@ -36,7 +36,7 @@ pub mod rt {
     use symbol::Symbol;
     use ThinVec;
 
-    use tokenstream::{self, TokenTree, TokenStream};
+    use tokenstream::{self, DelimSpan, TokenTree, TokenStream};
 
     pub use parse::new_parser_from_tts;
     pub use syntax_pos::{BytePos, Span, DUMMY_SP, FileName};
@@ -245,7 +245,8 @@ pub mod rt {
             }
             inner.push(self.tokens.clone());
 
-            r.push(TokenTree::Delimited(self.span, tokenstream::Delimited {
+            let delim_span = DelimSpan::from_single(self.span);
+            r.push(TokenTree::Delimited(delim_span, tokenstream::Delimited {
                 delim: token::Bracket, tts: TokenStream::concat(inner).into()
             }));
             r
@@ -261,7 +262,7 @@ pub mod rt {
 
     impl ToTokens for () {
         fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
-            vec![TokenTree::Delimited(DUMMY_SP, tokenstream::Delimited {
+            vec![TokenTree::Delimited(DelimSpan::dummy(), tokenstream::Delimited {
                 delim: token::Paren,
                 tts: TokenStream::empty().into(),
             })]
@@ -385,13 +386,16 @@ pub fn unflatten(tts: Vec<TokenTree>) -> Vec<TokenTree> {
 
     let mut results = Vec::new();
     let mut result = Vec::new();
+    let mut open_span = DUMMY_SP;
     for tree in tts {
         match tree {
-            TokenTree::Token(_, token::OpenDelim(..)) => {
+            TokenTree::Token(span, token::OpenDelim(..)) => {
+                open_span = span;
                 results.push(::std::mem::replace(&mut result, Vec::new()));
             }
             TokenTree::Token(span, token::CloseDelim(delim)) => {
-                let tree = TokenTree::Delimited(span, Delimited {
+                let delim_span = DelimSpan::from_pair(open_span, span);
+                let tree = TokenTree::Delimited(delim_span, Delimited {
                     delim,
                     tts: result.into_iter().map(TokenStream::from).collect::<TokenStream>().into(),
                 });
@@ -756,9 +760,9 @@ fn statements_mk_tt(cx: &ExtCtxt, tt: &TokenTree, quoted: bool) -> Vec<ast::Stmt
             vec![cx.stmt_expr(e_push)]
         },
         TokenTree::Delimited(span, ref delimed) => {
-            let mut stmts = statements_mk_tt(cx, &delimed.open_tt(span), false);
+            let mut stmts = statements_mk_tt(cx, &delimed.open_tt(span.open), false);
             stmts.extend(statements_mk_tts(cx, delimed.stream()));
-            stmts.extend(statements_mk_tt(cx, &delimed.close_tt(span), false));
+            stmts.extend(statements_mk_tt(cx, &delimed.close_tt(span.close), false));
             stmts
         }
     }
diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs
index c962e7fcbb4..941fd6f1000 100644
--- a/src/libsyntax/ext/tt/macro_parser.rs
+++ b/src/libsyntax/ext/tt/macro_parser.rs
@@ -85,7 +85,7 @@ pub use self::ParseResult::*;
 use self::TokenTreeOrTokenTreeSlice::*;
 
 use ast::Ident;
-use syntax_pos::{self, BytePos, Span};
+use syntax_pos::{self, Span};
 use errors::FatalError;
 use ext::tt::quoted::{self, TokenTree};
 use parse::{Directory, ParseSess};
@@ -94,7 +94,7 @@ use parse::token::{self, DocComment, Nonterminal, Token};
 use print::pprust;
 use OneVector;
 use symbol::keywords;
-use tokenstream::TokenStream;
+use tokenstream::{DelimSpan, TokenStream};
 
 use rustc_data_structures::fx::FxHashMap;
 use std::collections::hash_map::Entry::{Occupied, Vacant};
@@ -151,10 +151,10 @@ struct MatcherPos<'a> {
     top_elts: TokenTreeOrTokenTreeSlice<'a>,
     /// The position of the "dot" in this matcher
     idx: usize,
-    /// The beginning position in the source that the beginning of this matcher corresponds to. In
-    /// other words, the token in the source at `sp_lo` is matched against the first token of the
-    /// matcher.
-    sp_lo: BytePos,
+    /// The first span of source source that the beginning of this matcher corresponds to. In other
+    /// words, the token in the source whose span is `sp_open` is matched against the first token of
+    /// the matcher.
+    sp_open: Span,
 
     /// For each named metavar in the matcher, we keep track of token trees matched against the
     /// metavar by the black box parser. In particular, there may be more than one match per
@@ -284,8 +284,8 @@ fn create_matches(len: usize) -> Vec<Rc<Vec<NamedMatch>>> {
 }
 
 /// Generate the top-level matcher position in which the "dot" is before the first token of the
-/// matcher `ms` and we are going to start matching at position `lo` in the source.
-fn initial_matcher_pos(ms: &[TokenTree], lo: BytePos) -> MatcherPos {
+/// matcher `ms` and we are going to start matching at the span `open` in the source.
+fn initial_matcher_pos(ms: &[TokenTree], open: Span) -> MatcherPos {
     let match_idx_hi = count_names(ms);
     let matches = create_matches(match_idx_hi);
     MatcherPos {
@@ -293,8 +293,8 @@ fn initial_matcher_pos(ms: &[TokenTree], lo: BytePos) -> MatcherPos {
         top_elts: TtSeq(ms), // "elts" is an abbr. for "elements"
         // The "dot" is before the first token of the matcher
         idx: 0,
-        // We start matching with byte `lo` in the source code
-        sp_lo: lo,
+        // We start matching at the span `open` in the source code
+        sp_open: open,
 
         // Initialize `matches` to a bunch of empty `Vec`s -- one for each metavar in `top_elts`.
         // `match_lo` for `top_elts` is 0 and `match_hi` is `matches.len()`. `match_cur` is 0 since
@@ -332,7 +332,7 @@ fn initial_matcher_pos(ms: &[TokenTree], lo: BytePos) -> MatcherPos {
 /// token tree it was derived from.
 #[derive(Debug, Clone)]
 pub enum NamedMatch {
-    MatchedSeq(Rc<Vec<NamedMatch>>, syntax_pos::Span),
+    MatchedSeq(Rc<Vec<NamedMatch>>, DelimSpan),
     MatchedNonterminal(Rc<Nonterminal>),
 }
 
@@ -488,7 +488,7 @@ fn inner_parse_loop<'a>(
                     // Add matches from this repetition to the `matches` of `up`
                     for idx in item.match_lo..item.match_hi {
                         let sub = item.matches[idx].clone();
-                        let span = span.with_lo(item.sp_lo);
+                        let span = DelimSpan::from_pair(item.sp_open, span);
                         new_pos.push_match(idx, MatchedSeq(sub, span));
                     }
 
@@ -556,7 +556,7 @@ fn inner_parse_loop<'a>(
                         match_cur: item.match_cur,
                         match_hi: item.match_cur + seq.num_captures,
                         up: Some(item),
-                        sp_lo: sp.lo(),
+                        sp_open: sp.open,
                         top_elts: Tt(TokenTree::Sequence(sp, seq)),
                     })));
                 }
@@ -643,7 +643,7 @@ pub fn parse(
     //
     // This MatcherPos instance is allocated on the stack. All others -- and
     // there are frequently *no* others! -- are allocated on the heap.
-    let mut initial = initial_matcher_pos(ms, parser.span.lo());
+    let mut initial = initial_matcher_pos(ms, parser.span);
     let mut cur_items = smallvec![MatcherPosHandle::Ref(&mut initial)];
     let mut next_items = Vec::new();
 
diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs
index d09127d6b08..86247745c41 100644
--- a/src/libsyntax/ext/tt/macro_rules.rs
+++ b/src/libsyntax/ext/tt/macro_rules.rs
@@ -25,7 +25,7 @@ use parse::parser::Parser;
 use parse::token::{self, NtTT};
 use parse::token::Token::*;
 use symbol::Symbol;
-use tokenstream::{TokenStream, TokenTree};
+use tokenstream::{DelimSpan, TokenStream, TokenTree};
 
 use rustc_data_structures::fx::FxHashMap;
 use std::borrow::Cow;
@@ -226,7 +226,7 @@ pub fn compile(sess: &ParseSess, features: &Features, def: &ast::Item, edition:
     // ...quasiquoting this would be nice.
     // These spans won't matter, anyways
     let argument_gram = vec![
-        quoted::TokenTree::Sequence(DUMMY_SP, Lrc::new(quoted::SequenceRepetition {
+        quoted::TokenTree::Sequence(DelimSpan::dummy(), Lrc::new(quoted::SequenceRepetition {
             tts: vec![
                 quoted::TokenTree::MetaVarDecl(DUMMY_SP, lhs_nm, ast::Ident::from_str("tt")),
                 quoted::TokenTree::Token(DUMMY_SP, token::FatArrow),
@@ -237,7 +237,7 @@ pub fn compile(sess: &ParseSess, features: &Features, def: &ast::Item, edition:
             num_captures: 2,
         })),
         // to phase into semicolon-termination instead of semicolon-separation
-        quoted::TokenTree::Sequence(DUMMY_SP, Lrc::new(quoted::SequenceRepetition {
+        quoted::TokenTree::Sequence(DelimSpan::dummy(), Lrc::new(quoted::SequenceRepetition {
             tts: vec![quoted::TokenTree::Token(DUMMY_SP, token::Semi)],
             separator: None,
             op: quoted::KleeneOp::ZeroOrMore,
@@ -400,7 +400,8 @@ fn check_lhs_no_empty_seq(sess: &ParseSess, tts: &[quoted::TokenTree]) -> bool {
                         _ => false,
                     }
                 }) {
-                    sess.span_diagnostic.span_err(span, "repetition matches empty token tree");
+                    let sp = span.entire();
+                    sess.span_diagnostic.span_err(sp, "repetition matches empty token tree");
                     return false;
                 }
                 if !check_lhs_no_empty_seq(sess, &seq.tts) {
@@ -474,12 +475,12 @@ impl FirstSets {
                     }
                     TokenTree::Delimited(span, ref delimited) => {
                         build_recur(sets, &delimited.tts[..]);
-                        first.replace_with(delimited.open_tt(span));
+                        first.replace_with(delimited.open_tt(span.open));
                     }
                     TokenTree::Sequence(sp, ref seq_rep) => {
                         let subfirst = build_recur(sets, &seq_rep.tts[..]);
 
-                        match sets.first.entry(sp) {
+                        match sets.first.entry(sp.entire()) {
                             Entry::Vacant(vac) => {
                                 vac.insert(Some(subfirst.clone()));
                             }
@@ -499,7 +500,7 @@ impl FirstSets {
 
                         if let (Some(ref sep), true) = (seq_rep.separator.clone(),
                                                         subfirst.maybe_empty) {
-                            first.add_one_maybe(TokenTree::Token(sp, sep.clone()));
+                            first.add_one_maybe(TokenTree::Token(sp.entire(), sep.clone()));
                         }
 
                         // Reverse scan: Sequence comes before `first`.
@@ -534,11 +535,11 @@ impl FirstSets {
                     return first;
                 }
                 TokenTree::Delimited(span, ref delimited) => {
-                    first.add_one(delimited.open_tt(span));
+                    first.add_one(delimited.open_tt(span.open));
                     return first;
                 }
                 TokenTree::Sequence(sp, ref seq_rep) => {
-                    match self.first.get(&sp) {
+                    match self.first.get(&sp.entire()) {
                         Some(&Some(ref subfirst)) => {
 
                             // If the sequence contents can be empty, then the first
@@ -546,7 +547,7 @@ impl FirstSets {
 
                             if let (Some(ref sep), true) = (seq_rep.separator.clone(),
                                                             subfirst.maybe_empty) {
-                                first.add_one_maybe(TokenTree::Token(sp, sep.clone()));
+                                first.add_one_maybe(TokenTree::Token(sp.entire(), sep.clone()));
                             }
 
                             assert!(first.maybe_empty);
@@ -727,7 +728,7 @@ fn check_matcher_core(sess: &ParseSess,
                 }
             }
             TokenTree::Delimited(span, ref d) => {
-                let my_suffix = TokenSet::singleton(d.close_tt(span));
+                let my_suffix = TokenSet::singleton(d.close_tt(span.close));
                 check_matcher_core(sess, features, attrs, first_sets, &d.tts, &my_suffix);
                 // don't track non NT tokens
                 last.replace_with_irrelevant();
@@ -751,7 +752,7 @@ fn check_matcher_core(sess: &ParseSess,
                 let mut new;
                 let my_suffix = if let Some(ref u) = seq_rep.separator {
                     new = suffix_first.clone();
-                    new.add_one_maybe(TokenTree::Token(sp, u.clone()));
+                    new.add_one_maybe(TokenTree::Token(sp.entire(), u.clone()));
                     &new
                 } else {
                     &suffix_first
diff --git a/src/libsyntax/ext/tt/quoted.rs b/src/libsyntax/ext/tt/quoted.rs
index cc635a29275..74363f3e5f7 100644
--- a/src/libsyntax/ext/tt/quoted.rs
+++ b/src/libsyntax/ext/tt/quoted.rs
@@ -16,7 +16,7 @@ use parse::{token, ParseSess};
 use print::pprust;
 use symbol::keywords;
 use syntax_pos::{edition::Edition, BytePos, Span};
-use tokenstream;
+use tokenstream::{self, DelimSpan};
 use {ast, attr};
 
 use rustc_data_structures::sync::Lrc;
@@ -90,9 +90,9 @@ pub enum KleeneOp {
 #[derive(Debug, Clone, PartialEq, RustcEncodable, RustcDecodable)]
 pub enum TokenTree {
     Token(Span, token::Token),
-    Delimited(Span, Lrc<Delimited>),
+    Delimited(DelimSpan, Lrc<Delimited>),
     /// A kleene-style repetition sequence
-    Sequence(Span, Lrc<SequenceRepetition>),
+    Sequence(DelimSpan, Lrc<SequenceRepetition>),
     /// E.g. `$var`
     MetaVar(Span, ast::Ident),
     /// E.g. `$var:expr`. This is only used in the left hand side of MBE macros.
@@ -137,10 +137,10 @@ impl TokenTree {
             }
             (&TokenTree::Delimited(span, ref delimed), _) => {
                 if index == 0 {
-                    return delimed.open_tt(span);
+                    return delimed.open_tt(span.open);
                 }
                 if index == delimed.tts.len() + 1 {
-                    return delimed.close_tt(span);
+                    return delimed.close_tt(span.close);
                 }
                 delimed.tts[index - 1].clone()
             }
@@ -154,9 +154,9 @@ impl TokenTree {
         match *self {
             TokenTree::Token(sp, _)
             | TokenTree::MetaVar(sp, _)
-            | TokenTree::MetaVarDecl(sp, _, _)
-            | TokenTree::Delimited(sp, _)
-            | TokenTree::Sequence(sp, _) => sp,
+            | TokenTree::MetaVarDecl(sp, _, _) => sp,
+            TokenTree::Delimited(sp, _)
+            | TokenTree::Sequence(sp, _) => sp.entire(),
         }
     }
 }
@@ -286,7 +286,7 @@ where
                 if delimited.delim != token::Paren {
                     let tok = pprust::token_to_string(&token::OpenDelim(delimited.delim));
                     let msg = format!("expected `(`, found `{}`", tok);
-                    sess.span_diagnostic.span_err(span, &msg);
+                    sess.span_diagnostic.span_err(span.entire(), &msg);
                 }
                 // Parse the contents of the sequence itself
                 let sequence = parse(
@@ -302,7 +302,7 @@ where
                 let (separator, op) =
                     parse_sep_and_kleene_op(
                         trees,
-                        span,
+                        span.entire(),
                         sess,
                         features,
                         attrs,
diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/tt/transcribe.rs
index 549e5f00dce..2ed469e8e77 100644
--- a/src/libsyntax/ext/tt/transcribe.rs
+++ b/src/libsyntax/ext/tt/transcribe.rs
@@ -16,8 +16,8 @@ use ext::tt::quoted;
 use fold::noop_fold_tt;
 use parse::token::{self, Token, NtTT};
 use OneVector;
-use syntax_pos::{Span, DUMMY_SP};
-use tokenstream::{TokenStream, TokenTree, Delimited};
+use syntax_pos::DUMMY_SP;
+use tokenstream::{TokenStream, TokenTree, Delimited, DelimSpan};
 
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::sync::Lrc;
@@ -30,7 +30,7 @@ enum Frame {
     Delimited {
         forest: Lrc<quoted::Delimited>,
         idx: usize,
-        span: Span,
+        span: DelimSpan,
     },
     Sequence {
         forest: Lrc<quoted::SequenceRepetition>,
@@ -42,7 +42,7 @@ enum Frame {
 impl Frame {
     fn new(tts: Vec<quoted::TokenTree>) -> Frame {
         let forest = Lrc::new(quoted::Delimited { delim: token::NoDelim, tts: tts });
-        Frame::Delimited { forest: forest, idx: 0, span: DUMMY_SP }
+        Frame::Delimited { forest: forest, idx: 0, span: DelimSpan::dummy() }
     }
 }
 
@@ -123,20 +123,20 @@ pub fn transcribe(cx: &ExtCtxt,
                                          &interpolations,
                                          &repeats) {
                     LockstepIterSize::Unconstrained => {
-                        cx.span_fatal(sp, /* blame macro writer */
+                        cx.span_fatal(sp.entire(), /* blame macro writer */
                             "attempted to repeat an expression \
                              containing no syntax \
                              variables matched as repeating at this depth");
                     }
                     LockstepIterSize::Contradiction(ref msg) => {
                         // FIXME #2887 blame macro invoker instead
-                        cx.span_fatal(sp, &msg[..]);
+                        cx.span_fatal(sp.entire(), &msg[..]);
                     }
                     LockstepIterSize::Constraint(len, _) => {
                         if len == 0 {
                             if seq.op == quoted::KleeneOp::OneOrMore {
                                 // FIXME #2887 blame invoker
-                                cx.span_fatal(sp, "this must repeat at least once");
+                                cx.span_fatal(sp.entire(), "this must repeat at least once");
                             }
                         } else {
                             repeats.push((0, len));