Auto merge of #34424 - jseyfried:breaking_batch, r=Manishearth

Batch up libsyntax breaking changes

Batch of the following syntax-[breaking-change] changes:
 - #34213: Add a variant `Macro` to `TraitItemKind`
 - #34368: Merge the variant `QPath` of `PatKind` into the variant `PatKind::Path`
 - #34385: Move `syntax::ast::TokenTree` into a new module `syntax::tokenstream`
 - #33943:
  - Remove the type parameter from `visit::Visitor`
  - Remove `attr::WithAttrs` -- use `attr::HasAttrs` instead.
  - Change `fold_tt`/`fold_tts` to take token trees by value and avoid wrapping token trees in `Rc`.
  - Remove the field `ctxt` of `ast::Mac_`
  - Remove inherent method `attrs()` of types -- use the method `attrs` of `HasAttrs` instead.
 - #34316:
  - Remove `ast::Decl`/`ast::DeclKind` and add variants `Local` and `Item` to `StmtKind`.
  - Move the node id for statements from the `StmtKind` variants to a field of `Stmt` (making `Stmt` a struct instead of an alias for `Spanned<StmtKind>`)
  - Rename `ast::ExprKind::Again` to `Continue`.
 - #34339: Generalize and abstract `ThinAttributes` to `ThinVec<Attribute>`
  - Use `.into()` in convert between `Vec<Attribute>` and `ThinVec<Attribute>`
  - Use autoderef instead of `.as_attr_slice()`
 - #34436: Remove the optional expression from `ast::Block` and instead use a `StmtKind::Expr` at the end of the statement list.
 - #34403: Move errors into a separate crate (unlikely to cause breakage)

9 files changed, 373 insertions, 349 deletions

diff --git a/src/libsyntax/parse/attr.rs b/src/libsyntax/parse/attr.rs
index db643eb0df0..f6e94b7caea 100644
--- a/src/libsyntax/parse/attr.rs
+++ b/src/libsyntax/parse/attr.rs
@@ -10,7 +10,8 @@
 
 use attr;
 use ast;
-use codemap::{spanned, Spanned, mk_sp, Span};
+use syntax_pos::{mk_sp, Span};
+use codemap::{spanned, Spanned};
 use parse::common::SeqSep;
 use parse::PResult;
 use parse::token;
diff --git a/src/libsyntax/parse/classify.rs b/src/libsyntax/parse/classify.rs
index 89110f3160f..4fe4ec7e4c0 100644
--- a/src/libsyntax/parse/classify.rs
+++ b/src/libsyntax/parse/classify.rs
@@ -47,13 +47,9 @@ pub fn expr_is_simple_block(e: &ast::Expr) -> bool {
 /// seen the semicolon, and thus don't need another.
 pub fn stmt_ends_with_semi(stmt: &ast::StmtKind) -> bool {
     match *stmt {
-        ast::StmtKind::Decl(ref d, _) => {
-            match d.node {
-                ast::DeclKind::Local(_) => true,
-                ast::DeclKind::Item(_) => false,
-            }
-        }
-        ast::StmtKind::Expr(ref e, _) => expr_requires_semi_to_be_stmt(e),
+        ast::StmtKind::Local(_) => true,
+        ast::StmtKind::Item(_) => false,
+        ast::StmtKind::Expr(ref e) => expr_requires_semi_to_be_stmt(e),
         ast::StmtKind::Semi(..) => false,
         ast::StmtKind::Mac(..) => false,
     }
diff --git a/src/libsyntax/parse/lexer/comments.rs b/src/libsyntax/parse/lexer/comments.rs
index 06d255d5c0f..5eb5605ea71 100644
--- a/src/libsyntax/parse/lexer/comments.rs
+++ b/src/libsyntax/parse/lexer/comments.rs
@@ -11,7 +11,8 @@
 pub use self::CommentStyle::*;
 
 use ast;
-use codemap::{BytePos, CharPos, CodeMap, Pos};
+use codemap::CodeMap;
+use syntax_pos::{BytePos, CharPos, Pos};
 use errors;
 use parse::lexer::is_block_doc_comment;
 use parse::lexer::{StringReader, TokenAndSpan};
diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs
index d78a81dec83..809f4daa361 100644
--- a/src/libsyntax/parse/lexer/mod.rs
+++ b/src/libsyntax/parse/lexer/mod.rs
@@ -9,8 +9,8 @@
 // except according to those terms.
 
 use ast;
-use codemap::{BytePos, CharPos, CodeMap, Pos, Span};
-use codemap;
+use syntax_pos::{self, BytePos, CharPos, Pos, Span};
+use codemap::CodeMap;
 use errors::{FatalError, Handler, DiagnosticBuilder};
 use ext::tt::transcribe::tt_next_token;
 use parse::token::{self, keywords, str_to_ident};
@@ -84,7 +84,7 @@ pub struct StringReader<'a> {
     pub col: CharPos,
     /// The last character to be read
     pub curr: Option<char>,
-    pub filemap: Rc<codemap::FileMap>,
+    pub filemap: Rc<syntax_pos::FileMap>,
     // cached:
     pub peek_tok: token::Token,
     pub peek_span: Span,
@@ -162,7 +162,7 @@ impl<'a> Reader for TtReader<'a> {
 impl<'a> StringReader<'a> {
     /// For comments.rs, which hackily pokes into pos and curr
     pub fn new_raw<'b>(span_diagnostic: &'b Handler,
-                       filemap: Rc<codemap::FileMap>)
+                       filemap: Rc<syntax_pos::FileMap>)
                        -> StringReader<'b> {
         if filemap.src.is_none() {
             span_diagnostic.bug(&format!("Cannot lex filemap \
@@ -181,7 +181,7 @@ impl<'a> StringReader<'a> {
             filemap: filemap,
             // dummy values; not read
             peek_tok: token::Eof,
-            peek_span: codemap::DUMMY_SP,
+            peek_span: syntax_pos::DUMMY_SP,
             source_text: source_text,
             fatal_errs: Vec::new(),
         };
@@ -190,7 +190,7 @@ impl<'a> StringReader<'a> {
     }
 
     pub fn new<'b>(span_diagnostic: &'b Handler,
-                   filemap: Rc<codemap::FileMap>)
+                   filemap: Rc<syntax_pos::FileMap>)
                    -> StringReader<'b> {
         let mut sr = StringReader::new_raw(span_diagnostic, filemap);
         if let Err(_) = sr.advance_token() {
@@ -217,12 +217,12 @@ impl<'a> StringReader<'a> {
 
     /// Report a fatal error spanning [`from_pos`, `to_pos`).
     fn fatal_span_(&self, from_pos: BytePos, to_pos: BytePos, m: &str) -> FatalError {
-        self.fatal_span(codemap::mk_sp(from_pos, to_pos), m)
+        self.fatal_span(syntax_pos::mk_sp(from_pos, to_pos), m)
     }
 
     /// Report a lexical error spanning [`from_pos`, `to_pos`).
     fn err_span_(&self, from_pos: BytePos, to_pos: BytePos, m: &str) {
-        self.err_span(codemap::mk_sp(from_pos, to_pos), m)
+        self.err_span(syntax_pos::mk_sp(from_pos, to_pos), m)
     }
 
     /// Report a lexical error spanning [`from_pos`, `to_pos`), appending an
@@ -246,7 +246,7 @@ impl<'a> StringReader<'a> {
         for c in c.escape_default() {
             m.push(c)
         }
-        self.span_diagnostic.struct_span_fatal(codemap::mk_sp(from_pos, to_pos), &m[..])
+        self.span_diagnostic.struct_span_fatal(syntax_pos::mk_sp(from_pos, to_pos), &m[..])
     }
 
     /// Report a lexical error spanning [`from_pos`, `to_pos`), appending an
@@ -270,7 +270,7 @@ impl<'a> StringReader<'a> {
         for c in c.escape_default() {
             m.push(c)
         }
-        self.span_diagnostic.struct_span_err(codemap::mk_sp(from_pos, to_pos), &m[..])
+        self.span_diagnostic.struct_span_err(syntax_pos::mk_sp(from_pos, to_pos), &m[..])
     }
 
     /// Report a lexical error spanning [`from_pos`, `to_pos`), appending the
@@ -294,11 +294,11 @@ impl<'a> StringReader<'a> {
             None => {
                 if self.is_eof() {
                     self.peek_tok = token::Eof;
-                    self.peek_span = codemap::mk_sp(self.filemap.end_pos, self.filemap.end_pos);
+                    self.peek_span = syntax_pos::mk_sp(self.filemap.end_pos, self.filemap.end_pos);
                 } else {
                     let start_bytepos = self.last_pos;
                     self.peek_tok = self.next_token_inner()?;
-                    self.peek_span = codemap::mk_sp(start_bytepos, self.last_pos);
+                    self.peek_span = syntax_pos::mk_sp(start_bytepos, self.last_pos);
                 };
             }
         }
@@ -473,7 +473,7 @@ impl<'a> StringReader<'a> {
         match self.curr {
             Some(c) => {
                 if c.is_whitespace() {
-                    self.span_diagnostic.span_err(codemap::mk_sp(self.last_pos, self.last_pos),
+                    self.span_diagnostic.span_err(syntax_pos::mk_sp(self.last_pos, self.last_pos),
                                                   "called consume_any_line_comment, but there \
                                                    was whitespace");
                 }
@@ -524,13 +524,13 @@ impl<'a> StringReader<'a> {
 
                             Some(TokenAndSpan {
                                 tok: tok,
-                                sp: codemap::mk_sp(start_bpos, self.last_pos),
+                                sp: syntax_pos::mk_sp(start_bpos, self.last_pos),
                             })
                         })
                     } else {
                         Some(TokenAndSpan {
                             tok: token::Comment,
-                            sp: codemap::mk_sp(start_bpos, self.last_pos),
+                            sp: syntax_pos::mk_sp(start_bpos, self.last_pos),
                         })
                     };
                 }
@@ -563,7 +563,7 @@ impl<'a> StringReader<'a> {
                     }
                     return Some(TokenAndSpan {
                         tok: token::Shebang(self.name_from(start)),
-                        sp: codemap::mk_sp(start, self.last_pos),
+                        sp: syntax_pos::mk_sp(start, self.last_pos),
                     });
                 }
             }
@@ -591,7 +591,7 @@ impl<'a> StringReader<'a> {
                 }
                 let c = Some(TokenAndSpan {
                     tok: token::Whitespace,
-                    sp: codemap::mk_sp(start_bpos, self.last_pos),
+                    sp: syntax_pos::mk_sp(start_bpos, self.last_pos),
                 });
                 debug!("scanning whitespace: {:?}", c);
                 c
@@ -653,7 +653,7 @@ impl<'a> StringReader<'a> {
 
             Some(TokenAndSpan {
                 tok: tok,
-                sp: codemap::mk_sp(start_bpos, self.last_pos),
+                sp: syntax_pos::mk_sp(start_bpos, self.last_pos),
             })
         })
     }
@@ -850,7 +850,7 @@ impl<'a> StringReader<'a> {
                                 let valid = if self.curr_is('{') {
                                     self.scan_unicode_escape(delim) && !ascii_only
                                 } else {
-                                    let span = codemap::mk_sp(start, self.last_pos);
+                                    let span = syntax_pos::mk_sp(start, self.last_pos);
                                     self.span_diagnostic
                                         .struct_span_err(span, "incorrect unicode escape sequence")
                                         .span_help(span,
@@ -888,13 +888,13 @@ impl<'a> StringReader<'a> {
                                                                         },
                                                                         c);
                                 if e == '\r' {
-                                    err.span_help(codemap::mk_sp(escaped_pos, last_pos),
+                                    err.span_help(syntax_pos::mk_sp(escaped_pos, last_pos),
                                                   "this is an isolated carriage return; consider \
                                                    checking your editor and version control \
                                                    settings");
                                 }
                                 if (e == '{' || e == '}') && !ascii_only {
-                                    err.span_help(codemap::mk_sp(escaped_pos, last_pos),
+                                    err.span_help(syntax_pos::mk_sp(escaped_pos, last_pos),
                                                   "if used in a formatting string, curly braces \
                                                    are escaped with `{{` and `}}`");
                                 }
@@ -1677,7 +1677,8 @@ fn ident_continue(c: Option<char>) -> bool {
 mod tests {
     use super::*;
 
-    use codemap::{BytePos, CodeMap, Span, NO_EXPANSION};
+    use syntax_pos::{BytePos, Span, NO_EXPANSION};
+    use codemap::CodeMap;
     use errors;
     use parse::token;
     use parse::token::str_to_ident;
@@ -1686,7 +1687,10 @@ mod tests {
 
     fn mk_sh(cm: Rc<CodeMap>) -> errors::Handler {
         // FIXME (#22405): Replace `Box::new` with `box` here when/if possible.
-        let emitter = errors::emitter::EmitterWriter::new(Box::new(io::sink()), None, cm);
+        let emitter = errors::emitter::EmitterWriter::new(Box::new(io::sink()),
+                                                None,
+                                                cm,
+                                                errors::snippet::FormatMode::EnvironmentSelected);
         errors::Handler::with_emitter(true, false, Box::new(emitter))
     }
 
@@ -1889,7 +1893,7 @@ mod tests {
         let mut lexer = setup(&cm, &sh, "// test\r\n/// test\r\n".to_string());
         let comment = lexer.next_token();
         assert_eq!(comment.tok, token::Comment);
-        assert_eq!(comment.sp, ::codemap::mk_sp(BytePos(0), BytePos(7)));
+        assert_eq!(comment.sp, ::syntax_pos::mk_sp(BytePos(0), BytePos(7)));
         assert_eq!(lexer.next_token().tok, token::Whitespace);
         assert_eq!(lexer.next_token().tok,
                    token::DocComment(token::intern("/// test")));
diff --git a/src/libsyntax/parse/lexer/unicode_chars.rs b/src/libsyntax/parse/lexer/unicode_chars.rs
index d337c78bee8..dab97d1d5a6 100644
--- a/src/libsyntax/parse/lexer/unicode_chars.rs
+++ b/src/libsyntax/parse/lexer/unicode_chars.rs
@@ -11,7 +11,7 @@
 // Characters and their corresponding confusables were collected from
 // http://www.unicode.org/Public/security/revision-06/confusables.txt
 
-use codemap::mk_sp as make_span;
+use syntax_pos::mk_sp as make_span;
 use errors::DiagnosticBuilder;
 use super::StringReader;
 
diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs
index 2e4d46bc983..bbcc044d43c 100644
--- a/src/libsyntax/parse/mod.rs
+++ b/src/libsyntax/parse/mod.rs
@@ -11,12 +11,14 @@
 //! The main parser interface
 
 use ast;
-use codemap::{self, Span, CodeMap, FileMap};
+use codemap::CodeMap;
+use syntax_pos::{self, Span, FileMap};
 use errors::{Handler, ColorConfig, DiagnosticBuilder};
 use parse::parser::Parser;
 use parse::token::InternedString;
 use ptr::P;
 use str::char_at;
+use tokenstream;
 
 use std::cell::RefCell;
 use std::iter;
@@ -160,7 +162,7 @@ pub fn parse_tts_from_source_str<'a>(name: String,
                                      source: String,
                                      cfg: ast::CrateConfig,
                                      sess: &'a ParseSess)
-                                     -> PResult<'a, Vec<ast::TokenTree>> {
+                                     -> PResult<'a, Vec<tokenstream::TokenTree>> {
     let mut p = new_parser_from_source_str(
         sess,
         cfg,
@@ -211,8 +213,8 @@ pub fn filemap_to_parser<'a>(sess: &'a ParseSess,
     let end_pos = filemap.end_pos;
     let mut parser = tts_to_parser(sess, filemap_to_tts(sess, filemap), cfg);
 
-    if parser.token == token::Eof && parser.span == codemap::DUMMY_SP {
-        parser.span = codemap::mk_sp(end_pos, end_pos);
+    if parser.token == token::Eof && parser.span == syntax_pos::DUMMY_SP {
+        parser.span = syntax_pos::mk_sp(end_pos, end_pos);
     }
 
     parser
@@ -222,7 +224,7 @@ pub fn filemap_to_parser<'a>(sess: &'a ParseSess,
 // compiler expands into it
 pub fn new_parser_from_tts<'a>(sess: &'a ParseSess,
                                cfg: ast::CrateConfig,
-                               tts: Vec<ast::TokenTree>) -> Parser<'a> {
+                               tts: Vec<tokenstream::TokenTree>) -> Parser<'a> {
     tts_to_parser(sess, tts, cfg)
 }
 
@@ -247,7 +249,7 @@ fn file_to_filemap(sess: &ParseSess, path: &Path, spanopt: Option<Span>)
 
 /// Given a filemap, produce a sequence of token-trees
 pub fn filemap_to_tts(sess: &ParseSess, filemap: Rc<FileMap>)
-    -> Vec<ast::TokenTree> {
+    -> Vec<tokenstream::TokenTree> {
     // it appears to me that the cfg doesn't matter here... indeed,
     // parsing tt's probably shouldn't require a parser at all.
     let cfg = Vec::new();
@@ -258,7 +260,7 @@ pub fn filemap_to_tts(sess: &ParseSess, filemap: Rc<FileMap>)
 
 /// Given tts and cfg, produce a parser
 pub fn tts_to_parser<'a>(sess: &'a ParseSess,
-                         tts: Vec<ast::TokenTree>,
+                         tts: Vec<tokenstream::TokenTree>,
                          cfg: ast::CrateConfig) -> Parser<'a> {
     let trdr = lexer::new_tt_reader(&sess.span_diagnostic, None, None, tts);
     let mut p = Parser::new(sess, cfg, Box::new(trdr));
@@ -660,9 +662,9 @@ pub fn integer_lit(s: &str,
 #[cfg(test)]
 mod tests {
     use super::*;
-    use std::rc::Rc;
-    use codemap::{Span, BytePos, Pos, Spanned, NO_EXPANSION};
-    use ast::{self, TokenTree, PatKind};
+    use syntax_pos::{Span, BytePos, Pos, NO_EXPANSION};
+    use codemap::Spanned;
+    use ast::{self, PatKind};
     use abi::Abi;
     use attr::{first_attr_value_str_by_name, AttrMetaMethods};
     use parse;
@@ -670,10 +672,12 @@ mod tests {
     use parse::token::{str_to_ident};
     use print::pprust::item_to_string;
     use ptr::P;
+    use tokenstream::{self, TokenTree};
     use util::parser_testing::{string_to_tts, string_to_parser};
     use util::parser_testing::{string_to_expr, string_to_item, string_to_stmt};
+    use util::ThinVec;
 
-    // produce a codemap::span
+    // produce a syntax_pos::span
     fn sp(a: u32, b: u32) -> Span {
         Span {lo: BytePos(a), hi: BytePos(b), expn_id: NO_EXPANSION}
     }
@@ -693,7 +697,7 @@ mod tests {
                         ),
                     }),
                     span: sp(0, 1),
-                    attrs: None,
+                    attrs: ThinVec::new(),
                    }))
     }
 
@@ -716,7 +720,7 @@ mod tests {
                             )
                         }),
                     span: sp(0, 6),
-                    attrs: None,
+                    attrs: ThinVec::new(),
                    }))
     }
 
@@ -729,7 +733,7 @@ mod tests {
     #[test]
     fn string_to_tts_macro () {
         let tts = string_to_tts("macro_rules! zip (($a)=>($a))".to_string());
-        let tts: &[ast::TokenTree] = &tts[..];
+        let tts: &[tokenstream::TokenTree] = &tts[..];
 
         match (tts.len(), tts.get(0), tts.get(1), tts.get(2), tts.get(3)) {
             (
@@ -759,7 +763,7 @@ mod tests {
                             )
                             if first_delimed.delim == token::Paren
                             && ident.name.as_str() == "a" => {},
-                            _ => panic!("value 3: {:?}", **first_delimed),
+                            _ => panic!("value 3: {:?}", *first_delimed),
                         }
                         let tts = &second_delimed.tts[..];
                         match (tts.len(), tts.get(0), tts.get(1)) {
@@ -770,10 +774,10 @@ mod tests {
                             )
                             if second_delimed.delim == token::Paren
                             && ident.name.as_str() == "a" => {},
-                            _ => panic!("value 4: {:?}", **second_delimed),
+                            _ => panic!("value 4: {:?}", *second_delimed),
                         }
                     },
-                    _ => panic!("value 2: {:?}", **macro_delimed),
+                    _ => panic!("value 2: {:?}", *macro_delimed),
                 }
             },
             _ => panic!("value: {:?}",tts),
@@ -789,7 +793,7 @@ mod tests {
             TokenTree::Token(sp(3, 4), token::Ident(str_to_ident("a"))),
             TokenTree::Delimited(
                 sp(5, 14),
-                Rc::new(ast::Delimited {
+                tokenstream::Delimited {
                     delim: token::DelimToken::Paren,
                     open_span: sp(5, 6),
                     tts: vec![
@@ -798,10 +802,10 @@ mod tests {
                         TokenTree::Token(sp(10, 13), token::Ident(str_to_ident("i32"))),
                     ],
                     close_span: sp(13, 14),
-                })),
+                }),
             TokenTree::Delimited(
                 sp(15, 21),
-                Rc::new(ast::Delimited {
+                tokenstream::Delimited {
                     delim: token::DelimToken::Brace,
                     open_span: sp(15, 16),
                     tts: vec![
@@ -809,7 +813,7 @@ mod tests {
                         TokenTree::Token(sp(18, 19), token::Semi),
                     ],
                     close_span: sp(20, 21),
-                }))
+                })
         ];
 
         assert_eq!(tts, expected);
@@ -832,16 +836,16 @@ mod tests {
                             ),
                         }),
                         span:sp(7,8),
-                        attrs: None,
+                        attrs: ThinVec::new(),
                     }))),
                     span:sp(0,8),
-                    attrs: None,
+                    attrs: ThinVec::new(),
                    }))
     }
 
     #[test] fn parse_stmt_1 () {
         assert!(string_to_stmt("b;".to_string()) ==
-                   Some(Spanned{
+                   Some(ast::Stmt {
                        node: ast::StmtKind::Expr(P(ast::Expr {
                            id: ast::DUMMY_NODE_ID,
                            node: ast::ExprKind::Path(None, ast::Path {
@@ -855,8 +859,8 @@ mod tests {
                                ),
                             }),
                            span: sp(0,1),
-                           attrs: None}),
-                                           ast::DUMMY_NODE_ID),
+                           attrs: ThinVec::new()})),
+                       id: ast::DUMMY_NODE_ID,
                        span: sp(0,1)}))
 
     }
@@ -932,7 +936,7 @@ mod tests {
                                         }
                                     },
                                     P(ast::Block {
-                                        stmts: vec!(Spanned{
+                                        stmts: vec!(ast::Stmt {
                                             node: ast::StmtKind::Semi(P(ast::Expr{
                                                 id: ast::DUMMY_NODE_ID,
                                                 node: ast::ExprKind::Path(None,
@@ -950,10 +954,9 @@ mod tests {
                                                         ),
                                                       }),
                                                 span: sp(17,18),
-                                                attrs: None,}),
-                                                ast::DUMMY_NODE_ID),
+                                                attrs: ThinVec::new()})),
+                                            id: ast::DUMMY_NODE_ID,
                                             span: sp(17,19)}),
-                                        expr: None,
                                         id: ast::DUMMY_NODE_ID,
                                         rules: ast::BlockCheckMode::Default, // no idea
                                         span: sp(15,21),
@@ -992,8 +995,8 @@ mod tests {
         struct PatIdentVisitor {
             spans: Vec<Span>
         }
-        impl<'v> ::visit::Visitor<'v> for PatIdentVisitor {
-            fn visit_pat(&mut self, p: &'v ast::Pat) {
+        impl ::visit::Visitor for PatIdentVisitor {
+            fn visit_pat(&mut self, p: &ast::Pat) {
                 match p.node {
                     PatKind::Ident(_ , ref spannedident, _) => {
                         self.spans.push(spannedident.span.clone());
diff --git a/src/libsyntax/parse/obsolete.rs b/src/libsyntax/parse/obsolete.rs
index 75f1ac49c9a..a1d7ddcdf4b 100644
--- a/src/libsyntax/parse/obsolete.rs
+++ b/src/libsyntax/parse/obsolete.rs
@@ -13,7 +13,7 @@
 //!
 //! Obsolete syntax that becomes too hard to parse can be removed.
 
-use codemap::Span;
+use syntax_pos::Span;
 use parse::parser;
 
 /// The specific types of unsupported syntax
diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs
index 341b076e7cf..813d90103b8 100644
--- a/src/libsyntax/parse/parser.rs
+++ b/src/libsyntax/parse/parser.rs
@@ -16,8 +16,8 @@ use ast::{Mod, Arg, Arm, Attribute, BindingMode, TraitItemKind};
 use ast::Block;
 use ast::{BlockCheckMode, CaptureBy};
 use ast::{Constness, Crate, CrateConfig};
-use ast::{Decl, DeclKind, Defaultness};
-use ast::{EMPTY_CTXT, EnumDef};
+use ast::Defaultness;
+use ast::EnumDef;
 use ast::{Expr, ExprKind, RangeLimits};
 use ast::{Field, FnDecl};
 use ast::{ForeignItem, ForeignItemKind, FunctionRetTy};
@@ -33,14 +33,14 @@ use ast::{Stmt, StmtKind};
 use ast::{VariantData, StructField};
 use ast::StrStyle;
 use ast::SelfKind;
-use ast::{Delimited, SequenceRepetition, TokenTree, TraitItem, TraitRef};
+use ast::{TraitItem, TraitRef};
 use ast::{Ty, TyKind, TypeBinding, TyParam, TyParamBounds};
 use ast::{ViewPath, ViewPathGlob, ViewPathList, ViewPathSimple};
 use ast::{Visibility, WhereClause};
-use attr::{ThinAttributes, ThinAttributesExt, AttributesExt};
 use ast::{BinOpKind, UnOp};
 use ast;
-use codemap::{self, Span, BytePos, Spanned, spanned, mk_sp, CodeMap};
+use codemap::{self, CodeMap, Spanned, spanned};
+use syntax_pos::{self, Span, BytePos, mk_sp};
 use errors::{self, DiagnosticBuilder};
 use ext::tt::macro_parser;
 use parse;
@@ -55,6 +55,8 @@ use util::parser::{AssocOp, Fixity};
 use print::pprust;
 use ptr::P;
 use parse::PResult;
+use tokenstream::{self, Delimited, SequenceRepetition, TokenTree};
+use util::ThinVec;
 
 use std::collections::HashSet;
 use std::mem;
@@ -120,7 +122,7 @@ macro_rules! maybe_whole_expr {
                         _ => unreachable!()
                     };
                     let span = $p.span;
-                    Some($p.mk_expr(span.lo, span.hi, ExprKind::Path(None, pt), None))
+                    Some($p.mk_expr(span.lo, span.hi, ExprKind::Path(None, pt), ThinVec::new()))
                 }
                 token::Interpolated(token::NtBlock(_)) => {
                     // FIXME: The following avoids an issue with lexical borrowck scopes,
@@ -130,7 +132,7 @@ macro_rules! maybe_whole_expr {
                         _ => unreachable!()
                     };
                     let span = $p.span;
-                    Some($p.mk_expr(span.lo, span.hi, ExprKind::Block(b), None))
+                    Some($p.mk_expr(span.lo, span.hi, ExprKind::Block(b), ThinVec::new()))
                 }
                 _ => None
             };
@@ -316,12 +318,12 @@ pub struct ModulePathError {
 
 pub enum LhsExpr {
     NotYetParsed,
-    AttributesParsed(ThinAttributes),
+    AttributesParsed(ThinVec<Attribute>),
     AlreadyParsed(P<Expr>),
 }
 
-impl From<Option<ThinAttributes>> for LhsExpr {
-    fn from(o: Option<ThinAttributes>) -> Self {
+impl From<Option<ThinVec<Attribute>>> for LhsExpr {
+    fn from(o: Option<ThinVec<Attribute>>) -> Self {
         if let Some(attrs) = o {
             LhsExpr::AttributesParsed(attrs)
         } else {
@@ -344,7 +346,7 @@ impl<'a> Parser<'a> {
     {
         let tok0 = rdr.real_token();
         let span = tok0.sp;
-        let filename = if span != codemap::DUMMY_SP {
+        let filename = if span != syntax_pos::DUMMY_SP {
             Some(sess.codemap().span_to_filename(span))
         } else { None };
         let placeholder = TokenAndSpan {
@@ -1232,55 +1234,70 @@ impl<'a> Parser<'a> {
     }
 
     /// Parse the items in a trait declaration
-    pub fn parse_trait_items(&mut self) -> PResult<'a,  Vec<TraitItem>> {
-        self.parse_unspanned_seq(
-            &token::OpenDelim(token::Brace),
-            &token::CloseDelim(token::Brace),
-            SeqSep::none(),
-            |p| -> PResult<'a, TraitItem> {
-            maybe_whole!(no_clone_from_p p, NtTraitItem);
-            let mut attrs = p.parse_outer_attributes()?;
-            let lo = p.span.lo;
-
-            let (name, node) = if p.eat_keyword(keywords::Type) {
-                let TyParam {ident, bounds, default, ..} = p.parse_ty_param()?;
-                p.expect(&token::Semi)?;
-                (ident, TraitItemKind::Type(bounds, default))
-            } else if p.is_const_item() {
-                p.expect_keyword(keywords::Const)?;
-                let ident = p.parse_ident()?;
-                p.expect(&token::Colon)?;
-                let ty = p.parse_ty_sum()?;
-                let default = if p.check(&token::Eq) {
-                    p.bump();
-                    let expr = p.parse_expr()?;
-                    p.commit_expr_expecting(&expr, token::Semi)?;
-                    Some(expr)
-                } else {
-                    p.expect(&token::Semi)?;
-                    None
-                };
-                (ident, TraitItemKind::Const(ty, default))
+    pub fn parse_trait_item(&mut self) -> PResult<'a, TraitItem> {
+        maybe_whole!(no_clone_from_p self, NtTraitItem);
+        let mut attrs = self.parse_outer_attributes()?;
+        let lo = self.span.lo;
+
+        let (name, node) = if self.eat_keyword(keywords::Type) {
+            let TyParam {ident, bounds, default, ..} = self.parse_ty_param()?;
+            self.expect(&token::Semi)?;
+            (ident, TraitItemKind::Type(bounds, default))
+        } else if self.is_const_item() {
+                self.expect_keyword(keywords::Const)?;
+            let ident = self.parse_ident()?;
+            self.expect(&token::Colon)?;
+            let ty = self.parse_ty_sum()?;
+            let default = if self.check(&token::Eq) {
+                self.bump();
+                let expr = self.parse_expr()?;
+                self.commit_expr_expecting(&expr, token::Semi)?;
+                Some(expr)
+            } else {
+                self.expect(&token::Semi)?;
+                None
+            };
+            (ident, TraitItemKind::Const(ty, default))
+        } else if !self.token.is_any_keyword()
+            && self.look_ahead(1, |t| *t == token::Not)
+            && (self.look_ahead(2, |t| *t == token::OpenDelim(token::Paren))
+                || self.look_ahead(2, |t| *t == token::OpenDelim(token::Brace))) {
+                // trait item macro.
+                // code copied from parse_macro_use_or_failure... abstraction!
+                let lo = self.span.lo;
+                let pth = self.parse_ident_into_path()?;
+                self.expect(&token::Not)?;
+
+                // eat a matched-delimiter token tree:
+                let delim = self.expect_open_delim()?;
+                let tts = self.parse_seq_to_end(&token::CloseDelim(delim),
+                                             SeqSep::none(),
+                                             |pp| pp.parse_token_tree())?;
+                let m_ = Mac_ { path: pth, tts: tts };
+                let m: ast::Mac = codemap::Spanned { node: m_,
+                                                     span: mk_sp(lo,
+                                                                 self.last_span.hi) };
+                if delim != token::Brace {
+                    self.expect(&token::Semi)?
+                }
+                (keywords::Invalid.ident(), ast::TraitItemKind::Macro(m))
             } else {
-                let (constness, unsafety, abi) = match p.parse_fn_front_matter() {
+                let (constness, unsafety, abi) = match self.parse_fn_front_matter() {
                     Ok(cua) => cua,
                     Err(e) => {
                         loop {
-                            match p.token {
+                            match self.token {
                                 token::Eof => break,
-
                                 token::CloseDelim(token::Brace) |
                                 token::Semi => {
-                                    p.bump();
+                                    self.bump();
                                     break;
                                 }
-
                                 token::OpenDelim(token::Brace) => {
-                                    p.parse_token_tree()?;
+                                    self.parse_token_tree()?;
                                     break;
                                 }
-
-                                _ => p.bump()
+                                _ => self.bump()
                             }
                         }
 
@@ -1288,17 +1305,17 @@ impl<'a> Parser<'a> {
                     }
                 };
 
-                let ident = p.parse_ident()?;
-                let mut generics = p.parse_generics()?;
+                let ident = self.parse_ident()?;
+                let mut generics = self.parse_generics()?;
 
-                let d = p.parse_fn_decl_with_self(|p: &mut Parser<'a>|{
+                let d = self.parse_fn_decl_with_self(|p: &mut Parser<'a>|{
                     // This is somewhat dubious; We don't want to allow
                     // argument names to be left off if there is a
                     // definition...
                     p.parse_arg_general(false)
                 })?;
 
-                generics.where_clause = p.parse_where_clause()?;
+                generics.where_clause = self.parse_where_clause()?;
                 let sig = ast::MethodSig {
                     unsafety: unsafety,
                     constness: constness,
@@ -1307,37 +1324,47 @@ impl<'a> Parser<'a> {
                     abi: abi,
                 };
 
-                let body = match p.token {
-                  token::Semi => {
-                    p.bump();
-                    debug!("parse_trait_methods(): parsing required method");
-                    None
-                  }
-                  token::OpenDelim(token::Brace) => {
-                    debug!("parse_trait_methods(): parsing provided method");
-                    let (inner_attrs, body) =
-                        p.parse_inner_attrs_and_block()?;
-                    attrs.extend(inner_attrs.iter().cloned());
-                    Some(body)
-                  }
+                let body = match self.token {
+                    token::Semi => {
+                        self.bump();
+                        debug!("parse_trait_methods(): parsing required method");
+                        None
+                    }
+                    token::OpenDelim(token::Brace) => {
+                        debug!("parse_trait_methods(): parsing provided method");
+                        let (inner_attrs, body) =
+                            self.parse_inner_attrs_and_block()?;
+                        attrs.extend(inner_attrs.iter().cloned());
+                        Some(body)
+                    }
 
-                  _ => {
-                      let token_str = p.this_token_to_string();
-                      return Err(p.fatal(&format!("expected `;` or `{{`, found `{}`",
-                                       token_str)[..]))
-                  }
+                    _ => {
+                        let token_str = self.this_token_to_string();
+                        return Err(self.fatal(&format!("expected `;` or `{{`, found `{}`",
+                                                    token_str)[..]))
+                    }
                 };
                 (ident, ast::TraitItemKind::Method(sig, body))
             };
+        Ok(TraitItem {
+            id: ast::DUMMY_NODE_ID,
+            ident: name,
+            attrs: attrs,
+            node: node,
+            span: mk_sp(lo, self.last_span.hi),
+        })
+    }
 
-            Ok(TraitItem {
-                id: ast::DUMMY_NODE_ID,
-                ident: name,
-                attrs: attrs,
-                node: node,
-                span: mk_sp(lo, p.last_span.hi),
+
+    /// Parse the items in a trait declaration
+    pub fn parse_trait_items(&mut self) -> PResult<'a,  Vec<TraitItem>> {
+        self.parse_unspanned_seq(
+            &token::OpenDelim(token::Brace),
+            &token::CloseDelim(token::Brace),
+            SeqSep::none(),
+            |p| -> PResult<'a, TraitItem> {
+                p.parse_trait_item()
             })
-        })
     }
 
     /// Parse a possibly mutable type
@@ -1467,7 +1494,7 @@ impl<'a> Parser<'a> {
                                                 SeqSep::none(),
                                                 |p| p.parse_token_tree())?;
                 let hi = self.span.hi;
-                TyKind::Mac(spanned(lo, hi, Mac_ { path: path, tts: tts, ctxt: EMPTY_CTXT }))
+                TyKind::Mac(spanned(lo, hi, Mac_ { path: path, tts: tts }))
             } else {
                 // NAMED TYPE
                 TyKind::Path(None, path)
@@ -1676,12 +1703,12 @@ impl<'a> Parser<'a> {
         let lo = self.span.lo;
         let literal = P(self.parse_lit()?);
         let hi = self.last_span.hi;
-        let expr = self.mk_expr(lo, hi, ExprKind::Lit(literal), None);
+        let expr = self.mk_expr(lo, hi, ExprKind::Lit(literal), ThinVec::new());
 
         if minus_present {
             let minus_hi = self.last_span.hi;
             let unary = self.mk_unary(UnOp::Neg, expr);
-            Ok(self.mk_expr(minus_lo, minus_hi, unary, None))
+            Ok(self.mk_expr(minus_lo, minus_hi, unary, ThinVec::new()))
         } else {
             Ok(expr)
         }
@@ -2039,13 +2066,13 @@ impl<'a> Parser<'a> {
         })
     }
 
-    pub fn mk_expr(&mut self, lo: BytePos, hi: BytePos,
-                   node: ExprKind, attrs: ThinAttributes) -> P<Expr> {
+    pub fn mk_expr(&mut self, lo: BytePos, hi: BytePos, node: ExprKind, attrs: ThinVec<Attribute>)
+                   -> P<Expr> {
         P(Expr {
             id: ast::DUMMY_NODE_ID,
             node: node,
             span: mk_sp(lo, hi),
-            attrs: attrs,
+            attrs: attrs.into(),
         })
     }
 
@@ -2102,7 +2129,7 @@ impl<'a> Parser<'a> {
     }
 
     pub fn mk_mac_expr(&mut self, lo: BytePos, hi: BytePos,
-                       m: Mac_, attrs: ThinAttributes) -> P<Expr> {
+                       m: Mac_, attrs: ThinVec<Attribute>) -> P<Expr> {
         P(Expr {
             id: ast::DUMMY_NODE_ID,
             node: ExprKind::Mac(codemap::Spanned {node: m, span: mk_sp(lo, hi)}),
@@ -2111,7 +2138,7 @@ impl<'a> Parser<'a> {
         })
     }
 
-    pub fn mk_lit_u32(&mut self, i: u32, attrs: ThinAttributes) -> P<Expr> {
+    pub fn mk_lit_u32(&mut self, i: u32, attrs: ThinVec<Attribute>) -> P<Expr> {
         let span = &self.span;
         let lv_lit = P(codemap::Spanned {
             node: LitKind::Int(i as u64, ast::LitIntType::Unsigned(UintTy::U32)),
@@ -2152,7 +2179,7 @@ impl<'a> Parser<'a> {
         //
         // Therefore, prevent sub-parser from parsing
         // attributes by giving them a empty "already parsed" list.
-        let mut attrs = None;
+        let mut attrs = ThinVec::new();
 
         let lo = self.span.lo;
         let mut hi = self.span.hi;
@@ -2164,9 +2191,7 @@ impl<'a> Parser<'a> {
             token::OpenDelim(token::Paren) => {
                 self.bump();
 
-                let attrs = self.parse_inner_attributes()?
-                    .into_thin_attrs()
-                    .prepend(attrs);
+                attrs.extend(self.parse_inner_attributes()?);
 
                 // (e) is parenthesized e
                 // (e,) is a tuple with only one field, e
@@ -2204,9 +2229,7 @@ impl<'a> Parser<'a> {
             token::OpenDelim(token::Bracket) => {
                 self.bump();
 
-                let inner_attrs = self.parse_inner_attributes()?
-                    .into_thin_attrs();
-                attrs.update(|attrs| attrs.append(inner_attrs));
+                attrs.extend(self.parse_inner_attributes()?);
 
                 if self.check(&token::CloseDelim(token::Bracket)) {
                     // Empty vector.
@@ -2285,14 +2308,14 @@ impl<'a> Parser<'a> {
                 }
                 if self.eat_keyword(keywords::Continue) {
                     let ex = if self.token.is_lifetime() {
-                        let ex = ExprKind::Again(Some(Spanned{
+                        let ex = ExprKind::Continue(Some(Spanned{
                             node: self.get_lifetime(),
                             span: self.span
                         }));
                         self.bump();
                         ex
                     } else {
-                        ExprKind::Again(None)
+                        ExprKind::Continue(None)
                     };
                     let hi = self.last_span.hi;
                     return Ok(self.mk_expr(lo, hi, ex, attrs));
@@ -2348,7 +2371,7 @@ impl<'a> Parser<'a> {
 
                         return Ok(self.mk_mac_expr(lo,
                                                    hi,
-                                                   Mac_ { path: pth, tts: tts, ctxt: EMPTY_CTXT },
+                                                   Mac_ { path: pth, tts: tts },
                                                    attrs));
                     }
                     if self.check(&token::OpenDelim(token::Brace)) {
@@ -2363,9 +2386,7 @@ impl<'a> Parser<'a> {
                             let mut fields = Vec::new();
                             let mut base = None;
 
-                            let attrs = attrs.append(
-                                self.parse_inner_attributes()?
-                                    .into_thin_attrs());
+                            attrs.extend(self.parse_inner_attributes()?);
 
                             while self.token != token::CloseDelim(token::Brace) {
                                 if self.eat(&token::DotDot) {
@@ -2432,25 +2453,24 @@ impl<'a> Parser<'a> {
     }
 
     fn parse_or_use_outer_attributes(&mut self,
-                                     already_parsed_attrs: Option<ThinAttributes>)
-                                     -> PResult<'a, ThinAttributes> {
+                                     already_parsed_attrs: Option<ThinVec<Attribute>>)
+                                     -> PResult<'a, ThinVec<Attribute>> {
         if let Some(attrs) = already_parsed_attrs {
             Ok(attrs)
         } else {
-            self.parse_outer_attributes().map(|a| a.into_thin_attrs())
+            self.parse_outer_attributes().map(|a| a.into())
         }
     }
 
     /// Parse a block or unsafe block
     pub fn parse_block_expr(&mut self, lo: BytePos, blk_mode: BlockCheckMode,
-                            attrs: ThinAttributes)
+                            outer_attrs: ThinVec<Attribute>)
                             -> PResult<'a, P<Expr>> {
 
-        let outer_attrs = attrs;
         self.expect(&token::OpenDelim(token::Brace))?;
 
-        let inner_attrs = self.parse_inner_attributes()?.into_thin_attrs();
-        let attrs = outer_attrs.append(inner_attrs);
+        let mut attrs = outer_attrs;
+        attrs.extend(self.parse_inner_attributes()?);
 
         let blk = self.parse_block_tail(lo, blk_mode)?;
         return Ok(self.mk_expr(blk.span.lo, blk.span.hi, ExprKind::Block(blk), attrs));
@@ -2458,7 +2478,7 @@ impl<'a> Parser<'a> {
 
     /// parse a.b or a(13) or a[4] or just a
     pub fn parse_dot_or_call_expr(&mut self,
-                                  already_parsed_attrs: Option<ThinAttributes>)
+                                  already_parsed_attrs: Option<ThinVec<Attribute>>)
                                   -> PResult<'a, P<Expr>> {
         let attrs = self.parse_or_use_outer_attributes(already_parsed_attrs)?;
 
@@ -2470,7 +2490,7 @@ impl<'a> Parser<'a> {
     pub fn parse_dot_or_call_expr_with(&mut self,
                                        e0: P<Expr>,
                                        lo: BytePos,
-                                       attrs: ThinAttributes)
+                                       mut attrs: ThinVec<Attribute>)
                                        -> PResult<'a, P<Expr>> {
         // Stitch the list of outer attributes onto the return value.
         // A little bit ugly, but the best way given the current code
@@ -2478,12 +2498,13 @@ impl<'a> Parser<'a> {
         self.parse_dot_or_call_expr_with_(e0, lo)
         .map(|expr|
             expr.map(|mut expr| {
-                expr.attrs.update(|a| a.prepend(attrs));
+                attrs.extend::<Vec<_>>(expr.attrs.into());
+                expr.attrs = attrs;
                 match expr.node {
                     ExprKind::If(..) | ExprKind::IfLet(..) => {
-                        if !expr.attrs.as_attr_slice().is_empty() {
+                        if !expr.attrs.is_empty() {
                             // Just point to the first attribute in there...
-                            let span = expr.attrs.as_attr_slice()[0].span;
+                            let span = expr.attrs[0].span;
 
                             self.span_err(span,
                                 "attributes are not yet allowed on `if` \
@@ -2531,7 +2552,7 @@ impl<'a> Parser<'a> {
                 es.insert(0, self_value);
                 let id = spanned(ident_span.lo, ident_span.hi, ident);
                 let nd = self.mk_method_call(id, tys, es);
-                self.mk_expr(lo, hi, nd, None)
+                self.mk_expr(lo, hi, nd, ThinVec::new())
             }
             // Field access.
             _ => {
@@ -2544,7 +2565,7 @@ impl<'a> Parser<'a> {
 
                 let id = spanned(ident_span.lo, ident_span.hi, ident);
                 let field = self.mk_field(self_value, id);
-                self.mk_expr(lo, ident_span.hi, field, None)
+                self.mk_expr(lo, ident_span.hi, field, ThinVec::new())
             }
         })
     }
@@ -2556,7 +2577,7 @@ impl<'a> Parser<'a> {
             // expr?
             while self.eat(&token::Question) {
                 let hi = self.last_span.hi;
-                e = self.mk_expr(lo, hi, ExprKind::Try(e), None);
+                e = self.mk_expr(lo, hi, ExprKind::Try(e), ThinVec::new());
             }
 
             // expr.f
@@ -2584,7 +2605,7 @@ impl<'a> Parser<'a> {
                         Some(n) => {
                             let id = spanned(dot, hi, n);
                             let field = self.mk_tup_field(e, id);
-                            e = self.mk_expr(lo, hi, field, None);
+                            e = self.mk_expr(lo, hi, field, ThinVec::new());
                         }
                         None => {
                             let last_span = self.last_span;
@@ -2636,7 +2657,7 @@ impl<'a> Parser<'a> {
                 hi = self.last_span.hi;
 
                 let nd = self.mk_call(e, es);
-                e = self.mk_expr(lo, hi, nd, None);
+                e = self.mk_expr(lo, hi, nd, ThinVec::new());
               }
 
               // expr[...]
@@ -2647,7 +2668,7 @@ impl<'a> Parser<'a> {
                 hi = self.span.hi;
                 self.commit_expr_expecting(&ix, token::CloseDelim(token::Bracket))?;
                 let index = self.mk_index(e, ix);
-                e = self.mk_expr(lo, hi, index, None)
+                e = self.mk_expr(lo, hi, index, ThinVec::new())
               }
               _ => return Ok(e)
             }
@@ -2671,13 +2692,12 @@ impl<'a> Parser<'a> {
                     )?;
                     let (sep, repeat) = self.parse_sep_and_kleene_op()?;
                     let name_num = macro_parser::count_names(&seq);
-                    return Ok(TokenTree::Sequence(mk_sp(sp.lo, seq_span.hi),
-                                      Rc::new(SequenceRepetition {
-                                          tts: seq,
-                                          separator: sep,
-                                          op: repeat,
-                                          num_captures: name_num
-                                      })));
+                    return Ok(TokenTree::Sequence(mk_sp(sp.lo, seq_span.hi), SequenceRepetition {
+                        tts: seq,
+                        separator: sep,
+                        op: repeat,
+                        num_captures: name_num
+                    }));
                 } else if self.token.is_keyword(keywords::Crate) {
                     self.bump();
                     return Ok(TokenTree::Token(sp, SpecialVarNt(SpecialMacroVar::CrateMacroVar)));
@@ -2720,16 +2740,17 @@ impl<'a> Parser<'a> {
     /// Parse an optional separator followed by a Kleene-style
     /// repetition token (+ or *).
     pub fn parse_sep_and_kleene_op(&mut self)
-                                   -> PResult<'a, (Option<token::Token>, ast::KleeneOp)> {
-        fn parse_kleene_op<'a>(parser: &mut Parser<'a>) -> PResult<'a,  Option<ast::KleeneOp>> {
+                                   -> PResult<'a, (Option<token::Token>, tokenstream::KleeneOp)> {
+        fn parse_kleene_op<'a>(parser: &mut Parser<'a>) ->
+          PResult<'a,  Option<tokenstream::KleeneOp>> {
             match parser.token {
                 token::BinOp(token::Star) => {
                     parser.bump();
-                    Ok(Some(ast::KleeneOp::ZeroOrMore))
+                    Ok(Some(tokenstream::KleeneOp::ZeroOrMore))
                 },
                 token::BinOp(token::Plus) => {
                     parser.bump();
-                    Ok(Some(ast::KleeneOp::OneOrMore))
+                    Ok(Some(tokenstream::KleeneOp::OneOrMore))
                 },
                 _ => Ok(None)
             }
@@ -2832,12 +2853,12 @@ impl<'a> Parser<'a> {
                     _ => {}
                 }
 
-                Ok(TokenTree::Delimited(span, Rc::new(Delimited {
+                Ok(TokenTree::Delimited(span, Delimited {
                     delim: delim,
                     open_span: open_span,
                     tts: tts,
                     close_span: close_span,
-                })))
+                }))
             },
             _ => {
                 // invariants: the current token is not a left-delimiter,
@@ -2878,7 +2899,7 @@ impl<'a> Parser<'a> {
 
     /// Parse a prefix-unary-operator expr
     pub fn parse_prefix_expr(&mut self,
-                             already_parsed_attrs: Option<ThinAttributes>)
+                             already_parsed_attrs: Option<ThinVec<Attribute>>)
                              -> PResult<'a, P<Expr>> {
         let attrs = self.parse_or_use_outer_attributes(already_parsed_attrs)?;
         let lo = self.span.lo;
@@ -2923,8 +2944,7 @@ impl<'a> Parser<'a> {
                 let blk = self.parse_block()?;
                 let span = blk.span;
                 hi = span.hi;
-                let blk_expr = self.mk_expr(span.lo, span.hi, ExprKind::Block(blk),
-                                            None);
+                let blk_expr = self.mk_expr(span.lo, hi, ExprKind::Block(blk), ThinVec::new());
                 ExprKind::InPlace(place, blk_expr)
             }
             token::Ident(..) if self.token.is_keyword(keywords::Box) => {
@@ -2944,7 +2964,7 @@ impl<'a> Parser<'a> {
     /// This parses an expression accounting for associativity and precedence of the operators in
     /// the expression.
     pub fn parse_assoc_expr(&mut self,
-                            already_parsed_attrs: Option<ThinAttributes>)
+                            already_parsed_attrs: Option<ThinVec<Attribute>>)
                             -> PResult<'a, P<Expr>> {
         self.parse_assoc_expr_with(0, already_parsed_attrs.into())
     }
@@ -2997,13 +3017,13 @@ impl<'a> Parser<'a> {
             // Special cases:
             if op == AssocOp::As {
                 let rhs = self.parse_ty()?;
-                lhs = self.mk_expr(lhs_span.lo, rhs.span.hi,
-                                   ExprKind::Cast(lhs, rhs), None);
+                let (lo, hi) = (lhs_span.lo, rhs.span.hi);
+                lhs = self.mk_expr(lo, hi, ExprKind::Cast(lhs, rhs), ThinVec::new());
                 continue
             } else if op == AssocOp::Colon {
                 let rhs = self.parse_ty()?;
-                lhs = self.mk_expr(lhs_span.lo, rhs.span.hi,
-                                   ExprKind::Type(lhs, rhs), None);
+                let (lo, hi) = (lhs_span.lo, rhs.span.hi);
+                lhs = self.mk_expr(lo, hi, ExprKind::Type(lhs, rhs), ThinVec::new());
                 continue
             } else if op == AssocOp::DotDot || op == AssocOp::DotDotDot {
                 // If we didn’t have to handle `x..`/`x...`, it would be pretty easy to
@@ -3029,7 +3049,7 @@ impl<'a> Parser<'a> {
                 };
 
                 let r = try!(self.mk_range(Some(lhs), rhs, limits));
-                lhs = self.mk_expr(lhs_span.lo, rhs_span.hi, r, None);
+                lhs = self.mk_expr(lhs_span.lo, rhs_span.hi, r, ThinVec::new());
                 break
             }
 
@@ -3056,6 +3076,7 @@ impl<'a> Parser<'a> {
                 }),
             }?;
 
+            let (lo, hi) = (lhs_span.lo, rhs.span.hi);
             lhs = match op {
                 AssocOp::Add | AssocOp::Subtract | AssocOp::Multiply | AssocOp::Divide |
                 AssocOp::Modulus | AssocOp::LAnd | AssocOp::LOr | AssocOp::BitXor |
@@ -3063,14 +3084,13 @@ impl<'a> Parser<'a> {
                 AssocOp::Equal | AssocOp::Less | AssocOp::LessEqual | AssocOp::NotEqual |
                 AssocOp::Greater | AssocOp::GreaterEqual => {
                     let ast_op = op.to_ast_binop().unwrap();
-                    let (lhs_span, rhs_span) = (lhs_span, rhs.span);
                     let binary = self.mk_binary(codemap::respan(cur_op_span, ast_op), lhs, rhs);
-                    self.mk_expr(lhs_span.lo, rhs_span.hi, binary, None)
+                    self.mk_expr(lo, hi, binary, ThinVec::new())
                 }
                 AssocOp::Assign =>
-                    self.mk_expr(lhs_span.lo, rhs.span.hi, ExprKind::Assign(lhs, rhs), None),
+                    self.mk_expr(lo, hi, ExprKind::Assign(lhs, rhs), ThinVec::new()),
                 AssocOp::Inplace =>
-                    self.mk_expr(lhs_span.lo, rhs.span.hi, ExprKind::InPlace(lhs, rhs), None),
+                    self.mk_expr(lo, hi, ExprKind::InPlace(lhs, rhs), ThinVec::new()),
                 AssocOp::AssignOp(k) => {
                     let aop = match k {
                         token::Plus =>    BinOpKind::Add,
@@ -3084,9 +3104,8 @@ impl<'a> Parser<'a> {
                         token::Shl =>     BinOpKind::Shl,
                         token::Shr =>     BinOpKind::Shr,
                     };
-                    let (lhs_span, rhs_span) = (lhs_span, rhs.span);
                     let aopexpr = self.mk_assign_op(codemap::respan(cur_op_span, aop), lhs, rhs);
-                    self.mk_expr(lhs_span.lo, rhs_span.hi, aopexpr, None)
+                    self.mk_expr(lo, hi, aopexpr, ThinVec::new())
                 }
                 AssocOp::As | AssocOp::Colon | AssocOp::DotDot | AssocOp::DotDotDot => {
                     self.bug("As, Colon, DotDot or DotDotDot branch reached")
@@ -3121,7 +3140,7 @@ impl<'a> Parser<'a> {
 
     /// Parse prefix-forms of range notation: `..expr`, `..`, `...expr`
     fn parse_prefix_range_expr(&mut self,
-                               already_parsed_attrs: Option<ThinAttributes>)
+                               already_parsed_attrs: Option<ThinVec<Attribute>>)
                                -> PResult<'a, P<Expr>> {
         debug_assert!(self.token == token::DotDot || self.token == token::DotDotDot);
         let tok = self.token.clone();
@@ -3166,7 +3185,7 @@ impl<'a> Parser<'a> {
     }
 
     /// Parse an 'if' or 'if let' expression ('if' token already eaten)
-    pub fn parse_if_expr(&mut self, attrs: ThinAttributes) -> PResult<'a, P<Expr>> {
+    pub fn parse_if_expr(&mut self, attrs: ThinVec<Attribute>) -> PResult<'a, P<Expr>> {
         if self.check_keyword(keywords::Let) {
             return self.parse_if_let_expr(attrs);
         }
@@ -3184,7 +3203,7 @@ impl<'a> Parser<'a> {
     }
 
     /// Parse an 'if let' expression ('if' token already eaten)
-    pub fn parse_if_let_expr(&mut self, attrs: ThinAttributes)
+    pub fn parse_if_let_expr(&mut self, attrs: ThinVec<Attribute>)
                              -> PResult<'a, P<Expr>> {
         let lo = self.last_span.lo;
         self.expect_keyword(keywords::Let)?;
@@ -3205,7 +3224,7 @@ impl<'a> Parser<'a> {
     pub fn parse_lambda_expr(&mut self,
                              lo: BytePos,
                              capture_clause: CaptureBy,
-                             attrs: ThinAttributes)
+                             attrs: ThinVec<Attribute>)
                              -> PResult<'a, P<Expr>>
     {
         let decl = self.parse_fn_block_decl()?;
@@ -3217,9 +3236,12 @@ impl<'a> Parser<'a> {
                 let body_expr = self.parse_expr()?;
                 P(ast::Block {
                     id: ast::DUMMY_NODE_ID,
-                    stmts: vec![],
                     span: body_expr.span,
-                    expr: Some(body_expr),
+                    stmts: vec![Stmt {
+                        span: body_expr.span,
+                        node: StmtKind::Expr(body_expr),
+                        id: ast::DUMMY_NODE_ID,
+                    }],
                     rules: BlockCheckMode::Default,
                 })
             }
@@ -3240,24 +3262,24 @@ impl<'a> Parser<'a> {
     // `else` token already eaten
     pub fn parse_else_expr(&mut self) -> PResult<'a, P<Expr>> {
         if self.eat_keyword(keywords::If) {
-            return self.parse_if_expr(None);
+            return self.parse_if_expr(ThinVec::new());
         } else {
             let blk = self.parse_block()?;
-            return Ok(self.mk_expr(blk.span.lo, blk.span.hi, ExprKind::Block(blk), None));
+            return Ok(self.mk_expr(blk.span.lo, blk.span.hi, ExprKind::Block(blk), ThinVec::new()));
         }
     }
 
     /// Parse a 'for' .. 'in' expression ('for' token already eaten)
     pub fn parse_for_expr(&mut self, opt_ident: Option<ast::SpannedIdent>,
                           span_lo: BytePos,
-                          attrs: ThinAttributes) -> PResult<'a, P<Expr>> {
+                          mut attrs: ThinVec<Attribute>) -> PResult<'a, P<Expr>> {
         // Parse: `for <src_pat> in <src_expr> <src_loop_block>`
 
         let pat = self.parse_pat()?;
         self.expect_keyword(keywords::In)?;
         let expr = self.parse_expr_res(Restrictions::RESTRICTION_NO_STRUCT_LITERAL, None)?;
         let (iattrs, loop_block) = self.parse_inner_attrs_and_block()?;
-        let attrs = attrs.append(iattrs.into_thin_attrs());
+        attrs.extend(iattrs);
 
         let hi = self.last_span.hi;
 
@@ -3269,13 +3291,13 @@ impl<'a> Parser<'a> {
     /// Parse a 'while' or 'while let' expression ('while' token already eaten)
     pub fn parse_while_expr(&mut self, opt_ident: Option<ast::SpannedIdent>,
                             span_lo: BytePos,
-                            attrs: ThinAttributes) -> PResult<'a, P<Expr>> {
+                            mut attrs: ThinVec<Attribute>) -> PResult<'a, P<Expr>> {
         if self.token.is_keyword(keywords::Let) {
             return self.parse_while_let_expr(opt_ident, span_lo, attrs);
         }
         let cond = self.parse_expr_res(Restrictions::RESTRICTION_NO_STRUCT_LITERAL, None)?;
         let (iattrs, body) = self.parse_inner_attrs_and_block()?;
-        let attrs = attrs.append(iattrs.into_thin_attrs());
+        attrs.extend(iattrs);
         let hi = body.span.hi;
         return Ok(self.mk_expr(span_lo, hi, ExprKind::While(cond, body, opt_ident),
                                attrs));
@@ -3284,13 +3306,13 @@ impl<'a> Parser<'a> {
     /// Parse a 'while let' expression ('while' token already eaten)
     pub fn parse_while_let_expr(&mut self, opt_ident: Option<ast::SpannedIdent>,
                                 span_lo: BytePos,
-                                attrs: ThinAttributes) -> PResult<'a, P<Expr>> {
+                                mut attrs: ThinVec<Attribute>) -> PResult<'a, P<Expr>> {
         self.expect_keyword(keywords::Let)?;
         let pat = self.parse_pat()?;
         self.expect(&token::Eq)?;
         let expr = self.parse_expr_res(Restrictions::RESTRICTION_NO_STRUCT_LITERAL, None)?;
         let (iattrs, body) = self.parse_inner_attrs_and_block()?;
-        let attrs = attrs.append(iattrs.into_thin_attrs());
+        attrs.extend(iattrs);
         let hi = body.span.hi;
         return Ok(self.mk_expr(span_lo, hi, ExprKind::WhileLet(pat, expr, body, opt_ident), attrs));
     }
@@ -3298,15 +3320,15 @@ impl<'a> Parser<'a> {
     // parse `loop {...}`, `loop` token already eaten
     pub fn parse_loop_expr(&mut self, opt_ident: Option<ast::SpannedIdent>,
                            span_lo: BytePos,
-                           attrs: ThinAttributes) -> PResult<'a, P<Expr>> {
+                           mut attrs: ThinVec<Attribute>) -> PResult<'a, P<Expr>> {
         let (iattrs, body) = self.parse_inner_attrs_and_block()?;
-        let attrs = attrs.append(iattrs.into_thin_attrs());
+        attrs.extend(iattrs);
         let hi = body.span.hi;
         Ok(self.mk_expr(span_lo, hi, ExprKind::Loop(body, opt_ident), attrs))
     }
 
     // `match` token already eaten
-    fn parse_match_expr(&mut self, attrs: ThinAttributes) -> PResult<'a, P<Expr>> {
+    fn parse_match_expr(&mut self, mut attrs: ThinVec<Attribute>) -> PResult<'a, P<Expr>> {
         let match_span = self.last_span;
         let lo = self.last_span.lo;
         let discriminant = self.parse_expr_res(Restrictions::RESTRICTION_NO_STRUCT_LITERAL,
@@ -3318,8 +3340,8 @@ impl<'a> Parser<'a> {
             }
             return Err(e)
         }
-        let attrs = attrs.append(
-            self.parse_inner_attributes()?.into_thin_attrs());
+        attrs.extend(self.parse_inner_attributes()?);
+
         let mut arms: Vec<Arm> = Vec::new();
         while self.token != token::CloseDelim(token::Brace) {
             match self.parse_arm() {
@@ -3392,7 +3414,7 @@ impl<'a> Parser<'a> {
 
     /// Parse an expression, subject to the given restrictions
     pub fn parse_expr_res(&mut self, r: Restrictions,
-                          already_parsed_attrs: Option<ThinAttributes>)
+                          already_parsed_attrs: Option<ThinVec<Attribute>>)
                           -> PResult<'a, P<Expr>> {
         self.with_res(r, |this| this.parse_assoc_expr(already_parsed_attrs))
     }
@@ -3500,7 +3522,7 @@ impl<'a> Parser<'a> {
     }
 
     /// Parse the fields of a struct-like pattern
-    fn parse_pat_fields(&mut self) -> PResult<'a, (Vec<codemap::Spanned<ast::FieldPat>> , bool)> {
+    fn parse_pat_fields(&mut self) -> PResult<'a, (Vec<codemap::Spanned<ast::FieldPat>>, bool)> {
         let mut fields = Vec::new();
         let mut etc = false;
         let mut first = true;
@@ -3570,9 +3592,9 @@ impl<'a> Parser<'a> {
             };
 
             fields.push(codemap::Spanned { span: mk_sp(lo, hi),
-                                           node: ast::FieldPat { ident: fieldname,
-                                                                 pat: subpat,
-                                                                 is_shorthand: is_shorthand }});
+                                              node: ast::FieldPat { ident: fieldname,
+                                                                    pat: subpat,
+                                                                    is_shorthand: is_shorthand }});
         }
         return Ok((fields, etc));
     }
@@ -3590,7 +3612,7 @@ impl<'a> Parser<'a> {
                 (None, self.parse_path(PathStyle::Expr)?)
             };
             let hi = self.last_span.hi;
-            Ok(self.mk_expr(lo, hi, ExprKind::Path(qself, path), None))
+            Ok(self.mk_expr(lo, hi, ExprKind::Path(qself, path), ThinVec::new()))
         } else {
             self.parse_pat_literal_maybe_minus()
         }
@@ -3661,9 +3683,9 @@ impl<'a> Parser<'a> {
                         let tts = self.parse_seq_to_end(
                             &token::CloseDelim(delim),
                             SeqSep::none(), |p| p.parse_token_tree())?;
-                        let mac = Mac_ { path: path, tts: tts, ctxt: EMPTY_CTXT };
+                        let mac = Mac_ { path: path, tts: tts };
                         pat = PatKind::Mac(codemap::Spanned {node: mac,
-                                                       span: mk_sp(lo, self.last_span.hi)});
+                                                               span: mk_sp(lo, self.last_span.hi)});
                     } else {
                         // Parse ident @ pat
                         // This can give false positives and parse nullary enums,
@@ -3685,7 +3707,8 @@ impl<'a> Parser<'a> {
                       token::DotDotDot => {
                         // Parse range
                         let hi = self.last_span.hi;
-                        let begin = self.mk_expr(lo, hi, ExprKind::Path(qself, path), None);
+                        let begin =
+                              self.mk_expr(lo, hi, ExprKind::Path(qself, path), ThinVec::new());
                         self.bump();
                         let end = self.parse_pat_range_end()?;
                         pat = PatKind::Range(begin, end);
@@ -3715,12 +3738,7 @@ impl<'a> Parser<'a> {
                         pat = PatKind::TupleStruct(path, fields, ddpos)
                       }
                       _ => {
-                        pat = match qself {
-                            // Parse qualified path
-                            Some(qself) => PatKind::QPath(qself, path),
-                            // Parse nullary enum
-                            None => PatKind::Path(path)
-                        };
+                        pat = PatKind::Path(qself, path);
                       }
                     }
                 }
@@ -3785,7 +3803,7 @@ impl<'a> Parser<'a> {
     }
 
     /// Parse a local variable declaration
-    fn parse_local(&mut self, attrs: ThinAttributes) -> PResult<'a, P<Local>> {
+    fn parse_local(&mut self, attrs: ThinVec<Attribute>) -> PResult<'a, P<Local>> {
         let lo = self.span.lo;
         let pat = self.parse_pat()?;
 
@@ -3804,13 +3822,6 @@ impl<'a> Parser<'a> {
         }))
     }
 
-    /// Parse a "let" stmt
-    fn parse_let(&mut self, attrs: ThinAttributes) -> PResult<'a, P<Decl>> {
-        let lo = self.span.lo;
-        let local = self.parse_local(attrs)?;
-        Ok(P(spanned(lo, self.last_span.hi, DeclKind::Local(local))))
-    }
-
     /// Parse a structure field
     fn parse_name_and_ty(&mut self, pr: Visibility,
                          attrs: Vec<Attribute> ) -> PResult<'a, StructField> {
@@ -3923,12 +3934,12 @@ impl<'a> Parser<'a> {
         let attrs = self.parse_outer_attributes()?;
         let lo = self.span.lo;
 
-        Ok(Some(if self.check_keyword(keywords::Let) {
-            self.expect_keyword(keywords::Let)?;
-            let decl = self.parse_let(attrs.into_thin_attrs())?;
-            let hi = decl.span.hi;
-            let stmt = StmtKind::Decl(decl, ast::DUMMY_NODE_ID);
-            spanned(lo, hi, stmt)
+        Ok(Some(if self.eat_keyword(keywords::Let) {
+            Stmt {
+                id: ast::DUMMY_NODE_ID,
+                node: StmtKind::Local(self.parse_local(attrs.into())?),
+                span: mk_sp(lo, self.last_span.hi),
+            }
         } else if self.token.is_ident()
             && !self.token.is_any_keyword()
             && self.look_ahead(1, |t| *t == token::Not) {
@@ -3979,9 +3990,12 @@ impl<'a> Parser<'a> {
             };
 
             if id.name == keywords::Invalid.name() {
-                let mac = P(spanned(lo, hi, Mac_ { path: pth, tts: tts, ctxt: EMPTY_CTXT }));
-                let stmt = StmtKind::Mac(mac, style, attrs.into_thin_attrs());
-                spanned(lo, hi, stmt)
+                let mac = spanned(lo, hi, Mac_ { path: pth, tts: tts });
+                Stmt {
+                    id: ast::DUMMY_NODE_ID,
+                    node: StmtKind::Mac(P((mac, style, attrs.into()))),
+                    span: mk_sp(lo, hi),
+                }
             } else {
                 // if it has a special ident, it's definitely an item
                 //
@@ -3995,25 +4009,28 @@ impl<'a> Parser<'a> {
                                        followed by a semicolon");
                     }
                 }
-                spanned(lo, hi, StmtKind::Decl(
-                    P(spanned(lo, hi, DeclKind::Item(
+                Stmt {
+                    id: ast::DUMMY_NODE_ID,
+                    span: mk_sp(lo, hi),
+                    node: StmtKind::Item({
                         self.mk_item(
                             lo, hi, id /*id is good here*/,
-                            ItemKind::Mac(spanned(lo, hi,
-                                            Mac_ { path: pth, tts: tts, ctxt: EMPTY_CTXT })),
-                            Visibility::Inherited, attrs)))),
-                    ast::DUMMY_NODE_ID))
+                            ItemKind::Mac(spanned(lo, hi, Mac_ { path: pth, tts: tts })),
+                            Visibility::Inherited,
+                            attrs)
+                    }),
+                }
             }
         } else {
             // FIXME: Bad copy of attrs
             let restrictions = self.restrictions | Restrictions::NO_NONINLINE_MOD;
             match self.with_res(restrictions,
                                 |this| this.parse_item_(attrs.clone(), false, true))? {
-                Some(i) => {
-                    let hi = i.span.hi;
-                    let decl = P(spanned(lo, hi, DeclKind::Item(i)));
-                    spanned(lo, hi, StmtKind::Decl(decl, ast::DUMMY_NODE_ID))
-                }
+                Some(i) => Stmt {
+                    id: ast::DUMMY_NODE_ID,
+                    span: mk_sp(lo, i.span.hi),
+                    node: StmtKind::Item(i),
+                },
                 None => {
                     let unused_attrs = |attrs: &[_], s: &mut Self| {
                         if attrs.len() > 0 {
@@ -4036,10 +4053,12 @@ impl<'a> Parser<'a> {
 
                     // Remainder are line-expr stmts.
                     let e = self.parse_expr_res(
-                        Restrictions::RESTRICTION_STMT_EXPR, Some(attrs.into_thin_attrs()))?;
-                    let hi = e.span.hi;
-                    let stmt = StmtKind::Expr(e, ast::DUMMY_NODE_ID);
-                    spanned(lo, hi, stmt)
+                        Restrictions::RESTRICTION_STMT_EXPR, Some(attrs.into()))?;
+                    Stmt {
+                        id: ast::DUMMY_NODE_ID,
+                        span: mk_sp(lo, e.span.hi),
+                        node: StmtKind::Expr(e),
+                    }
                 }
             }
         }))
@@ -4082,10 +4101,9 @@ impl<'a> Parser<'a> {
     /// Precondition: already parsed the '{'.
     fn parse_block_tail(&mut self, lo: BytePos, s: BlockCheckMode) -> PResult<'a, P<Block>> {
         let mut stmts = vec![];
-        let mut expr = None;
 
         while !self.eat(&token::CloseDelim(token::Brace)) {
-            let Spanned {node, span} = if let Some(s) = self.parse_stmt_() {
+            let Stmt {node, span, ..} = if let Some(s) = self.parse_stmt_() {
                 s
             } else if self.token == token::Eof {
                 break;
@@ -4093,60 +4111,13 @@ impl<'a> Parser<'a> {
                 // Found only `;` or `}`.
                 continue;
             };
+
             match node {
-                StmtKind::Expr(e, _) => {
-                    self.handle_expression_like_statement(e, span, &mut stmts, &mut expr)?;
+                StmtKind::Expr(e) => {
+                    self.handle_expression_like_statement(e, span, &mut stmts)?;
                 }
-                StmtKind::Mac(mac, MacStmtStyle::NoBraces, attrs) => {
-                    // statement macro without braces; might be an
-                    // expr depending on whether a semicolon follows
-                    match self.token {
-                        token::Semi => {
-                            stmts.push(Spanned {
-                                node: StmtKind::Mac(mac, MacStmtStyle::Semicolon, attrs),
-                                span: mk_sp(span.lo, self.span.hi),
-                            });
-                            self.bump();
-                        }
-                        _ => {
-                            let e = self.mk_mac_expr(span.lo, span.hi,
-                                                     mac.and_then(|m| m.node),
-                                                     None);
-                            let lo = e.span.lo;
-                            let e = self.parse_dot_or_call_expr_with(e, lo, attrs)?;
-                            let e = self.parse_assoc_expr_with(0, LhsExpr::AlreadyParsed(e))?;
-                            self.handle_expression_like_statement(
-                                e,
-                                span,
-                                &mut stmts,
-                                &mut expr)?;
-                        }
-                    }
-                }
-                StmtKind::Mac(m, style, attrs) => {
-                    // statement macro; might be an expr
-                    match self.token {
-                        token::Semi => {
-                            stmts.push(Spanned {
-                                node: StmtKind::Mac(m, MacStmtStyle::Semicolon, attrs),
-                                span: mk_sp(span.lo, self.span.hi),
-                            });
-                            self.bump();
-                        }
-                        token::CloseDelim(token::Brace) => {
-                            // if a block ends in `m!(arg)` without
-                            // a `;`, it must be an expr
-                            expr = Some(self.mk_mac_expr(span.lo, span.hi,
-                                                         m.and_then(|x| x.node),
-                                                         attrs));
-                        }
-                        _ => {
-                            stmts.push(Spanned {
-                                node: StmtKind::Mac(m, style, attrs),
-                                span: span
-                            });
-                        }
-                    }
+                StmtKind::Mac(mac) => {
+                    self.handle_macro_in_block(mac.unwrap(), span, &mut stmts)?;
                 }
                 _ => { // all other kinds of statements:
                     let mut hi = span.hi;
@@ -4155,7 +4126,8 @@ impl<'a> Parser<'a> {
                         hi = self.last_span.hi;
                     }
 
-                    stmts.push(Spanned {
+                    stmts.push(Stmt {
+                        id: ast::DUMMY_NODE_ID,
                         node: node,
                         span: mk_sp(span.lo, hi)
                     });
@@ -4165,18 +4137,64 @@ impl<'a> Parser<'a> {
 
         Ok(P(ast::Block {
             stmts: stmts,
-            expr: expr,
             id: ast::DUMMY_NODE_ID,
             rules: s,
             span: mk_sp(lo, self.last_span.hi),
         }))
     }
 
+    fn handle_macro_in_block(&mut self,
+                             (mac, style, attrs): (ast::Mac, MacStmtStyle, ThinVec<Attribute>),
+                             span: Span,
+                             stmts: &mut Vec<Stmt>)
+                             -> PResult<'a, ()> {
+        if style == MacStmtStyle::NoBraces {
+            // statement macro without braces; might be an
+            // expr depending on whether a semicolon follows
+            match self.token {
+                token::Semi => {
+                    stmts.push(Stmt {
+                        id: ast::DUMMY_NODE_ID,
+                        node: StmtKind::Mac(P((mac, MacStmtStyle::Semicolon, attrs))),
+                        span: mk_sp(span.lo, self.span.hi),
+                    });
+                    self.bump();
+                }
+                _ => {
+                    let e = self.mk_mac_expr(span.lo, span.hi, mac.node, ThinVec::new());
+                    let lo = e.span.lo;
+                    let e = self.parse_dot_or_call_expr_with(e, lo, attrs)?;
+                    let e = self.parse_assoc_expr_with(0, LhsExpr::AlreadyParsed(e))?;
+                    self.handle_expression_like_statement(e, span, stmts)?;
+                }
+            }
+        } else {
+            // statement macro; might be an expr
+            match self.token {
+                token::Semi => {
+                    stmts.push(Stmt {
+                        id: ast::DUMMY_NODE_ID,
+                        node: StmtKind::Mac(P((mac, MacStmtStyle::Semicolon, attrs))),
+                        span: mk_sp(span.lo, self.span.hi),
+                    });
+                    self.bump();
+                }
+                _ => {
+                    stmts.push(Stmt {
+                        id: ast::DUMMY_NODE_ID,
+                        node: StmtKind::Mac(P((mac, style, attrs))),
+                        span: span
+                    });
+                }
+            }
+        }
+        Ok(())
+    }
+
     fn handle_expression_like_statement(&mut self,
                                         e: P<Expr>,
                                         span: Span,
-                                        stmts: &mut Vec<Stmt>,
-                                        last_block_expr: &mut Option<P<Expr>>)
+                                        stmts: &mut Vec<Stmt>)
                                         -> PResult<'a, ()> {
         // expression without semicolon
         if classify::expr_requires_semi_to_be_stmt(&e) {
@@ -4197,15 +4215,16 @@ impl<'a> Parser<'a> {
                     hi: self.last_span.hi,
                     expn_id: span.expn_id,
                 };
-                stmts.push(Spanned {
-                    node: StmtKind::Semi(e, ast::DUMMY_NODE_ID),
+                stmts.push(Stmt {
+                    id: ast::DUMMY_NODE_ID,
+                    node: StmtKind::Semi(e),
                     span: span_with_semi,
                 });
             }
-            token::CloseDelim(token::Brace) => *last_block_expr = Some(e),
             _ => {
-                stmts.push(Spanned {
-                    node: StmtKind::Expr(e, ast::DUMMY_NODE_ID),
+                stmts.push(Stmt {
+                    id: ast::DUMMY_NODE_ID,
+                    node: StmtKind::Expr(e),
                     span: span
                 });
             }
@@ -4913,10 +4932,10 @@ impl<'a> Parser<'a> {
             let tts = self.parse_seq_to_end(&token::CloseDelim(delim),
                                             SeqSep::none(),
                                             |p| p.parse_token_tree())?;
-            let m_ = Mac_ { path: pth, tts: tts, ctxt: EMPTY_CTXT };
+            let m_ = Mac_ { path: pth, tts: tts };
             let m: ast::Mac = codemap::Spanned { node: m_,
-                                                span: mk_sp(lo,
-                                                            self.last_span.hi) };
+                                                    span: mk_sp(lo,
+                                                                self.last_span.hi) };
             if delim != token::Brace {
                 self.expect(&token::Semi)?
             }
@@ -4940,7 +4959,6 @@ impl<'a> Parser<'a> {
 
     /// Parse trait Foo { ... }
     fn parse_item_trait(&mut self, unsafety: Unsafety) -> PResult<'a, ItemInfo> {
-
         let ident = self.parse_ident()?;
         let mut tps = self.parse_generics()?;
 
@@ -5255,7 +5273,7 @@ impl<'a> Parser<'a> {
             return Err(self.fatal(&format!("expected item, found `{}`", token_str)));
         }
 
-        let hi = if self.span == codemap::DUMMY_SP {
+        let hi = if self.span == syntax_pos::DUMMY_SP {
             inner_lo
         } else {
             self.last_span.hi
@@ -5999,10 +6017,10 @@ impl<'a> Parser<'a> {
                                             SeqSep::none(),
                                             |p| p.parse_token_tree())?;
             // single-variant-enum... :
-            let m = Mac_ { path: pth, tts: tts, ctxt: EMPTY_CTXT };
+            let m = Mac_ { path: pth, tts: tts };
             let m: ast::Mac = codemap::Spanned { node: m,
-                                             span: mk_sp(mac_lo,
-                                                         self.last_span.hi) };
+                                                 span: mk_sp(mac_lo,
+                                                             self.last_span.hi) };
 
             if delim != token::Brace {
                 if !self.eat(&token::Semi) {
diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs
index 47de32ed7d0..8376d28164d 100644
--- a/src/libsyntax/parse/token.rs
+++ b/src/libsyntax/parse/token.rs
@@ -19,6 +19,7 @@ use ext::mtwt;
 use ptr::P;
 use util::interner::{RcStr, StrInterner};
 use util::interner;
+use tokenstream;
 
 use serialize::{Decodable, Decoder, Encodable, Encoder};
 use std::fmt;
@@ -338,7 +339,7 @@ pub enum Nonterminal {
     /// Stuff inside brackets for attributes
     NtMeta(P<ast::MetaItem>),
     NtPath(Box<ast::Path>),
-    NtTT(P<ast::TokenTree>), // needs P'ed to break a circularity
+    NtTT(P<tokenstream::TokenTree>), // needs P'ed to break a circularity
     // These are not exposed to macros, but are used by quasiquote.
     NtArm(ast::Arm),
     NtImplItem(P<ast::ImplItem>),