Add Span and separate open/close delims to TTDelim

This came up when working [on the gl-rs generator extension](https://github.com/bjz/gl-rs/blob/990383de801bd2e233159d5be07c9b5622827620/src/gl_generator/lib.rs#L135-L146).

The new definition of  `TTDelim` adds an associated `Span` that covers the whole token tree and enforces the invariant that a delimited sequence of token trees must have an opening and closing delimiter.

A `get_span` method has also been added to `TokenTree` type to make it easier to implement better error messages for syntax extensions.

1 files changed, 21 insertions, 15 deletions

diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs
index 5abf79836f5..005ed2e7ed3 100644
--- a/src/libsyntax/parse/parser.rs
+++ b/src/libsyntax/parse/parser.rs
@@ -48,7 +48,7 @@ use ast::{StmtExpr, StmtSemi, StmtMac, StructDef, StructField};
 use ast::{StructVariantKind, BiSub};
 use ast::StrStyle;
 use ast::{SelfExplicit, SelfRegion, SelfStatic, SelfValue};
-use ast::{TokenTree, TraitItem, TraitRef, TTDelim, TTSeq, TTTok};
+use ast::{Delimiter, TokenTree, TraitItem, TraitRef, TTDelim, TTSeq, TTTok};
 use ast::{TTNonterminal, TupleVariantKind, Ty, Ty_, TyBot};
 use ast::{TypeField, TyFixedLengthVec, TyClosure, TyProc, TyBareFn};
 use ast::{TyTypeof, TyInfer, TypeMethod};
@@ -2574,16 +2574,11 @@ impl<'a> Parser<'a> {
                 }
               }
               _ => {
-                  parse_any_tt_tok(p)
+                  TTTok(p.span, p.bump_and_get())
               }
             }
         }
 
-        // turn the next token into a TTTok:
-        fn parse_any_tt_tok(p: &mut Parser) -> TokenTree {
-            TTTok(p.span, p.bump_and_get())
-        }
-
         match (&self.token, token::close_delimiter_for(&self.token)) {
             (&token::EOF, _) => {
                 let open_braces = self.open_braces.clone();
@@ -2595,21 +2590,32 @@ impl<'a> Parser<'a> {
                 self.fatal("this file contains an un-closed delimiter ");
             }
             (_, Some(close_delim)) => {
+                // The span for beginning of the delimited section
+                let pre_span = self.span;
+
                 // Parse the open delimiter.
                 self.open_braces.push(self.span);
-                let mut result = vec!(parse_any_tt_tok(self));
+                let open = Delimiter {
+                    span: self.span,
+                    token: self.bump_and_get(),
+                };
 
-                let trees =
-                    self.parse_seq_to_before_end(&close_delim,
-                                                 seq_sep_none(),
-                                                 |p| p.parse_token_tree());
-                result.extend(trees.into_iter());
+                // Parse the token trees within the delimeters
+                let tts = self.parse_seq_to_before_end(
+                    &close_delim, seq_sep_none(), |p| p.parse_token_tree()
+                );
 
                 // Parse the close delimiter.
-                result.push(parse_any_tt_tok(self));
+                let close = Delimiter {
+                    span: self.span,
+                    token: self.bump_and_get(),
+                };
                 self.open_braces.pop().unwrap();
 
-                TTDelim(Rc::new(result))
+                // Expand to cover the entire delimited token tree
+                let span = Span { hi: self.span.hi, ..pre_span };
+
+                TTDelim(span, open, Rc::new(tts), close)
             }
             _ => parse_non_delim_tt_tok(self)
         }