Add Span and separate open/close delims to TTDelim

This came up when working [on the gl-rs generator extension](https://github.com/bjz/gl-rs/blob/990383de801bd2e233159d5be07c9b5622827620/src/gl_generator/lib.rs#L135-L146).

The new definition of  `TTDelim` adds an associated `Span` that covers the whole token tree and enforces the invariant that a delimited sequence of token trees must have an opening and closing delimiter.

A `get_span` method has also been added to `TokenTree` type to make it easier to implement better error messages for syntax extensions.

2 files changed, 39 insertions, 34 deletions

diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs
index 2d7d32cd9ea..1c99b608f7a 100644
--- a/src/libsyntax/parse/mod.rs
+++ b/src/libsyntax/parse/mod.rs
@@ -788,35 +788,34 @@ mod test {
     }
 
     // check the token-tree-ization of macros
-    #[test] fn string_to_tts_macro () {
+    #[test]
+    fn string_to_tts_macro () {
         let tts = string_to_tts("macro_rules! zip (($a)=>($a))".to_string());
         let tts: &[ast::TokenTree] = tts.as_slice();
         match tts {
-            [ast::TTTok(_,_),
-             ast::TTTok(_,token::NOT),
-             ast::TTTok(_,_),
-             ast::TTDelim(ref delim_elts)] => {
+            [ast::TTTok(_, _),
+             ast::TTTok(_, token::NOT),
+             ast::TTTok(_, _),
+             ast::TTDelim(_, ast::TTTok(_, token::LPAREN),
+                          ref delim_elts,
+                          ast::TTTok(_, token::RPAREN))] => {
                 let delim_elts: &[ast::TokenTree] = delim_elts.as_slice();
                 match delim_elts {
-                    [ast::TTTok(_,token::LPAREN),
-                     ast::TTDelim(ref first_set),
-                     ast::TTTok(_,token::FAT_ARROW),
-                     ast::TTDelim(ref second_set),
-                     ast::TTTok(_,token::RPAREN)] => {
+                    [ast::TTDelim(_, ast::TTTok(_, token::LPAREN),
+                                  ref first_set,
+                                  ast::TTTok(_, token::RPAREN)),
+                     ast::TTTok(_, token::FAT_ARROW),
+                     ast::TTDelim(_, ast::TTTok(_, token::LPAREN),
+                                  ref second_set,
+                                  ast::TTTok(_, token::RPAREN))] => {
                         let first_set: &[ast::TokenTree] =
                             first_set.as_slice();
                         match first_set {
-                            [ast::TTTok(_,token::LPAREN),
-                             ast::TTTok(_,token::DOLLAR),
-                             ast::TTTok(_,_),
-                             ast::TTTok(_,token::RPAREN)] => {
+                            [ast::TTTok(_, token::DOLLAR), ast::TTTok(_, _)] => {
                                 let second_set: &[ast::TokenTree] =
                                     second_set.as_slice();
                                 match second_set {
-                                    [ast::TTTok(_,token::LPAREN),
-                                     ast::TTTok(_,token::DOLLAR),
-                                     ast::TTTok(_,_),
-                                     ast::TTTok(_,token::RPAREN)] => {
+                                    [ast::TTTok(_, token::DOLLAR), ast::TTTok(_, _)] => {
                                         assert_eq!("correct","correct")
                                     }
                                     _ => assert_eq!("wrong 4","correct")
@@ -837,7 +836,7 @@ mod test {
             _ => {
                 error!("failing value: {}",tts);
                 assert_eq!("wrong 1","correct");
-            }
+            },
         }
     }
 
diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs
index 5abf79836f5..005ed2e7ed3 100644
--- a/src/libsyntax/parse/parser.rs
+++ b/src/libsyntax/parse/parser.rs
@@ -48,7 +48,7 @@ use ast::{StmtExpr, StmtSemi, StmtMac, StructDef, StructField};
 use ast::{StructVariantKind, BiSub};
 use ast::StrStyle;
 use ast::{SelfExplicit, SelfRegion, SelfStatic, SelfValue};
-use ast::{TokenTree, TraitItem, TraitRef, TTDelim, TTSeq, TTTok};
+use ast::{Delimiter, TokenTree, TraitItem, TraitRef, TTDelim, TTSeq, TTTok};
 use ast::{TTNonterminal, TupleVariantKind, Ty, Ty_, TyBot};
 use ast::{TypeField, TyFixedLengthVec, TyClosure, TyProc, TyBareFn};
 use ast::{TyTypeof, TyInfer, TypeMethod};
@@ -2574,16 +2574,11 @@ impl<'a> Parser<'a> {
                 }
               }
               _ => {
-                  parse_any_tt_tok(p)
+                  TTTok(p.span, p.bump_and_get())
               }
             }
         }
 
-        // turn the next token into a TTTok:
-        fn parse_any_tt_tok(p: &mut Parser) -> TokenTree {
-            TTTok(p.span, p.bump_and_get())
-        }
-
         match (&self.token, token::close_delimiter_for(&self.token)) {
             (&token::EOF, _) => {
                 let open_braces = self.open_braces.clone();
@@ -2595,21 +2590,32 @@ impl<'a> Parser<'a> {
                 self.fatal("this file contains an un-closed delimiter ");
             }
             (_, Some(close_delim)) => {
+                // The span for beginning of the delimited section
+                let pre_span = self.span;
+
                 // Parse the open delimiter.
                 self.open_braces.push(self.span);
-                let mut result = vec!(parse_any_tt_tok(self));
+                let open = Delimiter {
+                    span: self.span,
+                    token: self.bump_and_get(),
+                };
 
-                let trees =
-                    self.parse_seq_to_before_end(&close_delim,
-                                                 seq_sep_none(),
-                                                 |p| p.parse_token_tree());
-                result.extend(trees.into_iter());
+                // Parse the token trees within the delimeters
+                let tts = self.parse_seq_to_before_end(
+                    &close_delim, seq_sep_none(), |p| p.parse_token_tree()
+                );
 
                 // Parse the close delimiter.
-                result.push(parse_any_tt_tok(self));
+                let close = Delimiter {
+                    span: self.span,
+                    token: self.bump_and_get(),
+                };
                 self.open_braces.pop().unwrap();
 
-                TTDelim(Rc::new(result))
+                // Expand to cover the entire delimited token tree
+                let span = Span { hi: self.span.hi, ..pre_span };
+
+                TTDelim(span, open, Rc::new(tts), close)
             }
             _ => parse_non_delim_tt_tok(self)
         }