Auto merge of #34424 - jseyfried:breaking_batch, r=Manishearth

Batch up libsyntax breaking changes

Batch of the following syntax-[breaking-change] changes:
 - #34213: Add a variant `Macro` to `TraitItemKind`
 - #34368: Merge the variant `QPath` of `PatKind` into the variant `PatKind::Path`
 - #34385: Move `syntax::ast::TokenTree` into a new module `syntax::tokenstream`
 - #33943:
  - Remove the type parameter from `visit::Visitor`
  - Remove `attr::WithAttrs` -- use `attr::HasAttrs` instead.
  - Change `fold_tt`/`fold_tts` to take token trees by value and avoid wrapping token trees in `Rc`.
  - Remove the field `ctxt` of `ast::Mac_`
  - Remove inherent method `attrs()` of types -- use the method `attrs` of `HasAttrs` instead.
 - #34316:
  - Remove `ast::Decl`/`ast::DeclKind` and add variants `Local` and `Item` to `StmtKind`.
  - Move the node id for statements from the `StmtKind` variants to a field of `Stmt` (making `Stmt` a struct instead of an alias for `Spanned<StmtKind>`)
  - Rename `ast::ExprKind::Again` to `Continue`.
 - #34339: Generalize and abstract `ThinAttributes` to `ThinVec<Attribute>`
  - Use `.into()` in convert between `Vec<Attribute>` and `ThinVec<Attribute>`
  - Use autoderef instead of `.as_attr_slice()`
 - #34436: Remove the optional expression from `ast::Block` and instead use a `StmtKind::Expr` at the end of the statement list.
 - #34403: Move errors into a separate crate (unlikely to cause breakage)

8 files changed, 324 insertions, 301 deletions

diff --git a/src/libsyntax/ext/base.rs b/src/libsyntax/ext/base.rs
index f2a005573d5..757b039fcac 100644
--- a/src/libsyntax/ext/base.rs
+++ b/src/libsyntax/ext/base.rs
@@ -13,8 +13,8 @@ pub use self::SyntaxExtension::*;
 use ast;
 use ast::{Name, PatKind};
 use attr::HasAttrs;
-use codemap;
-use codemap::{CodeMap, Span, ExpnId, ExpnInfo, NO_EXPANSION};
+use codemap::{self, CodeMap, ExpnInfo};
+use syntax_pos::{Span, ExpnId, NO_EXPANSION};
 use errors::DiagnosticBuilder;
 use ext;
 use ext::expand;
@@ -32,6 +32,7 @@ use fold::Folder;
 use std::collections::{HashMap, HashSet};
 use std::rc::Rc;
 use std::default::Default;
+use tokenstream;
 
 
 #[derive(Debug,Clone)]
@@ -168,20 +169,22 @@ pub trait TTMacroExpander {
     fn expand<'cx>(&self,
                    ecx: &'cx mut ExtCtxt,
                    span: Span,
-                   token_tree: &[ast::TokenTree])
+                   token_tree: &[tokenstream::TokenTree])
                    -> Box<MacResult+'cx>;
 }
 
 pub type MacroExpanderFn =
-    for<'cx> fn(&'cx mut ExtCtxt, Span, &[ast::TokenTree]) -> Box<MacResult+'cx>;
+    for<'cx> fn(&'cx mut ExtCtxt, Span, &[tokenstream::TokenTree])
+                -> Box<MacResult+'cx>;
 
 impl<F> TTMacroExpander for F
-    where F : for<'cx> Fn(&'cx mut ExtCtxt, Span, &[ast::TokenTree]) -> Box<MacResult+'cx>
+    where F : for<'cx> Fn(&'cx mut ExtCtxt, Span, &[tokenstream::TokenTree])
+                          -> Box<MacResult+'cx>
 {
     fn expand<'cx>(&self,
                    ecx: &'cx mut ExtCtxt,
                    span: Span,
-                   token_tree: &[ast::TokenTree])
+                   token_tree: &[tokenstream::TokenTree])
                    -> Box<MacResult+'cx> {
         (*self)(ecx, span, token_tree)
     }
@@ -192,22 +195,23 @@ pub trait IdentMacroExpander {
                    cx: &'cx mut ExtCtxt,
                    sp: Span,
                    ident: ast::Ident,
-                   token_tree: Vec<ast::TokenTree> )
+                   token_tree: Vec<tokenstream::TokenTree> )
                    -> Box<MacResult+'cx>;
 }
 
 pub type IdentMacroExpanderFn =
-    for<'cx> fn(&'cx mut ExtCtxt, Span, ast::Ident, Vec<ast::TokenTree>) -> Box<MacResult+'cx>;
+    for<'cx> fn(&'cx mut ExtCtxt, Span, ast::Ident, Vec<tokenstream::TokenTree>)
+                -> Box<MacResult+'cx>;
 
 impl<F> IdentMacroExpander for F
     where F : for<'cx> Fn(&'cx mut ExtCtxt, Span, ast::Ident,
-                          Vec<ast::TokenTree>) -> Box<MacResult+'cx>
+                          Vec<tokenstream::TokenTree>) -> Box<MacResult+'cx>
 {
     fn expand<'cx>(&self,
                    cx: &'cx mut ExtCtxt,
                    sp: Span,
                    ident: ast::Ident,
-                   token_tree: Vec<ast::TokenTree> )
+                   token_tree: Vec<tokenstream::TokenTree> )
                    -> Box<MacResult+'cx>
     {
         (*self)(cx, sp, ident, token_tree)
@@ -217,10 +221,11 @@ impl<F> IdentMacroExpander for F
 // Use a macro because forwarding to a simple function has type system issues
 macro_rules! make_stmts_default {
     ($me:expr) => {
-        $me.make_expr().map(|e| {
-            SmallVector::one(codemap::respan(
-                e.span, ast::StmtKind::Expr(e, ast::DUMMY_NODE_ID)))
-        })
+        $me.make_expr().map(|e| SmallVector::one(ast::Stmt {
+            id: ast::DUMMY_NODE_ID,
+            span: e.span,
+            node: ast::StmtKind::Expr(e),
+        }))
     }
 }
 
@@ -241,6 +246,11 @@ pub trait MacResult {
         None
     }
 
+    /// Create zero or more trait items.
+    fn make_trait_items(self: Box<Self>) -> Option<SmallVector<ast::TraitItem>> {
+        None
+    }
+
     /// Create a pattern.
     fn make_pat(self: Box<Self>) -> Option<P<ast::Pat>> {
         None
@@ -288,6 +298,7 @@ make_MacEager! {
     pat: P<ast::Pat>,
     items: SmallVector<P<ast::Item>>,
     impl_items: SmallVector<ast::ImplItem>,
+    trait_items: SmallVector<ast::TraitItem>,
     stmts: SmallVector<ast::Stmt>,
     ty: P<ast::Ty>,
 }
@@ -305,6 +316,10 @@ impl MacResult for MacEager {
         self.impl_items
     }
 
+    fn make_trait_items(self: Box<Self>) -> Option<SmallVector<ast::TraitItem>> {
+        self.trait_items
+    }
+
     fn make_stmts(self: Box<Self>) -> Option<SmallVector<ast::Stmt>> {
         match self.stmts.as_ref().map_or(0, |s| s.len()) {
             0 => make_stmts_default!(self),
@@ -365,7 +380,7 @@ impl DummyResult {
             id: ast::DUMMY_NODE_ID,
             node: ast::ExprKind::Lit(P(codemap::respan(sp, ast::LitKind::Bool(false)))),
             span: sp,
-            attrs: None,
+            attrs: ast::ThinVec::new(),
         })
     }
 
@@ -413,11 +428,20 @@ impl MacResult for DummyResult {
         }
     }
 
+    fn make_trait_items(self: Box<DummyResult>) -> Option<SmallVector<ast::TraitItem>> {
+        if self.expr_only {
+            None
+        } else {
+            Some(SmallVector::zero())
+        }
+    }
+
     fn make_stmts(self: Box<DummyResult>) -> Option<SmallVector<ast::Stmt>> {
-        Some(SmallVector::one(
-            codemap::respan(self.span,
-                            ast::StmtKind::Expr(DummyResult::raw_expr(self.span),
-                                                ast::DUMMY_NODE_ID))))
+        Some(SmallVector::one(ast::Stmt {
+            id: ast::DUMMY_NODE_ID,
+            node: ast::StmtKind::Expr(DummyResult::raw_expr(self.span)),
+            span: self.span,
+        }))
     }
 }
 
@@ -612,7 +636,7 @@ impl<'a> ExtCtxt<'a> {
         expand::MacroExpander::new(self)
     }
 
-    pub fn new_parser_from_tts(&self, tts: &[ast::TokenTree])
+    pub fn new_parser_from_tts(&self, tts: &[tokenstream::TokenTree])
         -> parser::Parser<'a> {
         parse::tts_to_parser(self.parse_sess, tts.to_vec(), self.cfg())
     }
@@ -811,7 +835,7 @@ pub fn expr_to_string(cx: &mut ExtCtxt, expr: P<ast::Expr>, err_msg: &str)
 /// done as rarely as possible).
 pub fn check_zero_tts(cx: &ExtCtxt,
                       sp: Span,
-                      tts: &[ast::TokenTree],
+                      tts: &[tokenstream::TokenTree],
                       name: &str) {
     if !tts.is_empty() {
         cx.span_err(sp, &format!("{} takes no arguments", name));
@@ -822,7 +846,7 @@ pub fn check_zero_tts(cx: &ExtCtxt,
 /// is not a string literal, emit an error and return None.
 pub fn get_single_str_from_tts(cx: &mut ExtCtxt,
                                sp: Span,
-                               tts: &[ast::TokenTree],
+                               tts: &[tokenstream::TokenTree],
                                name: &str)
                                -> Option<String> {
     let mut p = cx.new_parser_from_tts(tts);
@@ -843,7 +867,7 @@ pub fn get_single_str_from_tts(cx: &mut ExtCtxt,
 /// parsing error, emit a non-fatal error and return None.
 pub fn get_exprs_from_tts(cx: &mut ExtCtxt,
                           sp: Span,
-                          tts: &[ast::TokenTree]) -> Option<Vec<P<ast::Expr>>> {
+                          tts: &[tokenstream::TokenTree]) -> Option<Vec<P<ast::Expr>>> {
     let mut p = cx.new_parser_from_tts(tts);
     let mut es = Vec::new();
     while p.token != token::Eof {
diff --git a/src/libsyntax/ext/build.rs b/src/libsyntax/ext/build.rs
index 3a1cdae9bfb..435241f426e 100644
--- a/src/libsyntax/ext/build.rs
+++ b/src/libsyntax/ext/build.rs
@@ -11,7 +11,8 @@
 use abi::Abi;
 use ast::{self, Ident, Generics, Expr, BlockCheckMode, UnOp, PatKind};
 use attr;
-use codemap::{Span, respan, Spanned, DUMMY_SP, Pos};
+use syntax_pos::{Span, DUMMY_SP, Pos};
+use codemap::{respan, Spanned};
 use ext::base::ExtCtxt;
 use parse::token::{self, keywords, InternedString};
 use ptr::P;
@@ -87,6 +88,7 @@ pub trait AstBuilder {
 
     // statements
     fn stmt_expr(&self, expr: P<ast::Expr>) -> ast::Stmt;
+    fn stmt_semi(&self, expr: P<ast::Expr>) -> ast::Stmt;
     fn stmt_let(&self, sp: Span, mutbl: bool, ident: ast::Ident, ex: P<ast::Expr>) -> ast::Stmt;
     fn stmt_let_typed(&self,
                       sp: Span,
@@ -98,12 +100,8 @@ pub trait AstBuilder {
     fn stmt_item(&self, sp: Span, item: P<ast::Item>) -> ast::Stmt;
 
     // blocks
-    fn block(&self, span: Span, stmts: Vec<ast::Stmt>,
-             expr: Option<P<ast::Expr>>) -> P<ast::Block>;
+    fn block(&self, span: Span, stmts: Vec<ast::Stmt>) -> P<ast::Block>;
     fn block_expr(&self, expr: P<ast::Expr>) -> P<ast::Block>;
-    fn block_all(&self, span: Span,
-                 stmts: Vec<ast::Stmt>,
-                 expr: Option<P<ast::Expr>>) -> P<ast::Block>;
 
     // expressions
     fn expr(&self, span: Span, node: ast::ExprKind) -> P<ast::Expr>;
@@ -508,7 +506,19 @@ impl<'a> AstBuilder for ExtCtxt<'a> {
     }
 
     fn stmt_expr(&self, expr: P<ast::Expr>) -> ast::Stmt {
-        respan(expr.span, ast::StmtKind::Semi(expr, ast::DUMMY_NODE_ID))
+        ast::Stmt {
+            id: ast::DUMMY_NODE_ID,
+            span: expr.span,
+            node: ast::StmtKind::Expr(expr),
+        }
+    }
+
+    fn stmt_semi(&self, expr: P<ast::Expr>) -> ast::Stmt {
+        ast::Stmt {
+            id: ast::DUMMY_NODE_ID,
+            span: expr.span,
+            node: ast::StmtKind::Semi(expr),
+        }
     }
 
     fn stmt_let(&self, sp: Span, mutbl: bool, ident: ast::Ident,
@@ -525,10 +535,13 @@ impl<'a> AstBuilder for ExtCtxt<'a> {
             init: Some(ex),
             id: ast::DUMMY_NODE_ID,
             span: sp,
-            attrs: None,
+            attrs: ast::ThinVec::new(),
         });
-        let decl = respan(sp, ast::DeclKind::Local(local));
-        respan(sp, ast::StmtKind::Decl(P(decl), ast::DUMMY_NODE_ID))
+        ast::Stmt {
+            id: ast::DUMMY_NODE_ID,
+            node: ast::StmtKind::Local(local),
+            span: sp,
+        }
     }
 
     fn stmt_let_typed(&self,
@@ -550,36 +563,37 @@ impl<'a> AstBuilder for ExtCtxt<'a> {
             init: Some(ex),
             id: ast::DUMMY_NODE_ID,
             span: sp,
-            attrs: None,
+            attrs: ast::ThinVec::new(),
         });
-        let decl = respan(sp, ast::DeclKind::Local(local));
-        P(respan(sp, ast::StmtKind::Decl(P(decl), ast::DUMMY_NODE_ID)))
-    }
-
-    fn block(&self, span: Span, stmts: Vec<ast::Stmt>,
-             expr: Option<P<Expr>>) -> P<ast::Block> {
-        self.block_all(span, stmts, expr)
+        P(ast::Stmt {
+            id: ast::DUMMY_NODE_ID,
+            node: ast::StmtKind::Local(local),
+            span: sp,
+        })
     }
 
     fn stmt_item(&self, sp: Span, item: P<ast::Item>) -> ast::Stmt {
-        let decl = respan(sp, ast::DeclKind::Item(item));
-        respan(sp, ast::StmtKind::Decl(P(decl), ast::DUMMY_NODE_ID))
+        ast::Stmt {
+            id: ast::DUMMY_NODE_ID,
+            node: ast::StmtKind::Item(item),
+            span: sp,
+        }
     }
 
     fn block_expr(&self, expr: P<ast::Expr>) -> P<ast::Block> {
-        self.block_all(expr.span, Vec::new(), Some(expr))
-    }
-    fn block_all(&self,
-                 span: Span,
-                 stmts: Vec<ast::Stmt>,
-                 expr: Option<P<ast::Expr>>) -> P<ast::Block> {
-            P(ast::Block {
-               stmts: stmts,
-               expr: expr,
-               id: ast::DUMMY_NODE_ID,
-               rules: BlockCheckMode::Default,
-               span: span,
-            })
+        self.block(expr.span, vec![ast::Stmt {
+            id: ast::DUMMY_NODE_ID,
+            span: expr.span,
+            node: ast::StmtKind::Expr(expr),
+        }])
+    }
+    fn block(&self, span: Span, stmts: Vec<ast::Stmt>) -> P<ast::Block> {
+        P(ast::Block {
+           stmts: stmts,
+           id: ast::DUMMY_NODE_ID,
+           rules: BlockCheckMode::Default,
+           span: span,
+        })
     }
 
     fn expr(&self, span: Span, node: ast::ExprKind) -> P<ast::Expr> {
@@ -587,7 +601,7 @@ impl<'a> AstBuilder for ExtCtxt<'a> {
             id: ast::DUMMY_NODE_ID,
             node: node,
             span: span,
-            attrs: None,
+            attrs: ast::ThinVec::new(),
         })
     }
 
@@ -830,7 +844,7 @@ impl<'a> AstBuilder for ExtCtxt<'a> {
     }
     fn pat_enum(&self, span: Span, path: ast::Path, subpats: Vec<P<ast::Pat>>) -> P<ast::Pat> {
         let pat = if subpats.is_empty() {
-            PatKind::Path(path)
+            PatKind::Path(None, path)
         } else {
             PatKind::TupleStruct(path, subpats, None)
         };
@@ -948,14 +962,14 @@ impl<'a> AstBuilder for ExtCtxt<'a> {
                     ids: Vec<ast::Ident>,
                     stmts: Vec<ast::Stmt>)
                     -> P<ast::Expr> {
-        self.lambda(span, ids, self.block(span, stmts, None))
+        self.lambda(span, ids, self.block(span, stmts))
     }
     fn lambda_stmts_0(&self, span: Span, stmts: Vec<ast::Stmt>) -> P<ast::Expr> {
-        self.lambda0(span, self.block(span, stmts, None))
+        self.lambda0(span, self.block(span, stmts))
     }
     fn lambda_stmts_1(&self, span: Span, stmts: Vec<ast::Stmt>,
                       ident: ast::Ident) -> P<ast::Expr> {
-        self.lambda1(span, self.block(span, stmts, None), ident)
+        self.lambda1(span, self.block(span, stmts), ident)
     }
 
     fn arg(&self, span: Span, ident: ast::Ident, ty: P<ast::Ty>) -> ast::Arg {
diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index 5beb4937207..3036a88430a 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -8,18 +8,16 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use ast::{Block, Crate, DeclKind, PatKind};
+use ast::{Block, Crate, PatKind};
 use ast::{Local, Ident, Mac_, Name, SpannedIdent};
 use ast::{MacStmtStyle, Mrk, Stmt, StmtKind, ItemKind};
-use ast::TokenTree;
 use ast;
 use attr::HasAttrs;
 use ext::mtwt;
-use ext::build::AstBuilder;
 use attr;
-use attr::{AttrMetaMethods, WithAttrs, ThinAttributesExt};
-use codemap;
-use codemap::{Span, Spanned, ExpnInfo, ExpnId, NameAndSpan, MacroBang, MacroAttribute};
+use attr::AttrMetaMethods;
+use codemap::{Spanned, ExpnInfo, NameAndSpan, MacroBang, MacroAttribute};
+use syntax_pos::{self, Span, ExpnId};
 use config::StripUnconfigured;
 use ext::base::*;
 use feature_gate::{self, Features};
@@ -28,6 +26,7 @@ use fold::*;
 use util::move_map::MoveMap;
 use parse::token::{fresh_mark, fresh_name, intern, keywords};
 use ptr::P;
+use tokenstream::TokenTree;
 use util::small_vector::SmallVector;
 use visit;
 use visit::Visitor;
@@ -42,7 +41,7 @@ trait MacroGenerable: Sized {
 
     // Fold this node or list of nodes using the given folder.
     fn fold_with<F: Folder>(self, folder: &mut F) -> Self;
-    fn visit_with<'v, V: Visitor<'v>>(&'v self, visitor: &mut V);
+    fn visit_with<V: Visitor>(&self, visitor: &mut V);
 
     // Return a placeholder expansion to allow compilation to continue after an erroring expansion.
     fn dummy(span: Span) -> Self;
@@ -63,7 +62,7 @@ macro_rules! impl_macro_generable {
                 $( folder.$fold(self) )*
                 $( self.into_iter().flat_map(|item| folder. $fold_elt (item)).collect() )*
             }
-            fn visit_with<'v, V: Visitor<'v>>(&'v self, visitor: &mut V) {
+            fn visit_with<V: Visitor>(&self, visitor: &mut V) {
                 $( visitor.$visit(self) )*
                 $( for item in self.as_slice() { visitor. $visit_elt (item) } )*
             }
@@ -81,8 +80,11 @@ impl_macro_generable! {
         "statement",  .make_stmts, lift .fold_stmt, lift .visit_stmt, |_span| SmallVector::zero();
     SmallVector<P<ast::Item>>:
         "item",       .make_items, lift .fold_item, lift .visit_item, |_span| SmallVector::zero();
+    SmallVector<ast::TraitItem>:
+        "trait item", .make_trait_items, lift .fold_trait_item, lift .visit_trait_item,
+        |_span| SmallVector::zero();
     SmallVector<ast::ImplItem>:
-        "impl item",  .make_impl_items, lift .fold_impl_item, lift .visit_impl_item,
+        "impl item",  .make_impl_items,  lift .fold_impl_item,  lift .visit_impl_item,
         |_span| SmallVector::zero();
 }
 
@@ -95,24 +97,23 @@ impl MacroGenerable for Option<P<ast::Expr>> {
     fn fold_with<F: Folder>(self, folder: &mut F) -> Self {
         self.and_then(|expr| folder.fold_opt_expr(expr))
     }
-    fn visit_with<'v, V: Visitor<'v>>(&'v self, visitor: &mut V) {
+    fn visit_with<V: Visitor>(&self, visitor: &mut V) {
         self.as_ref().map(|expr| visitor.visit_expr(expr));
     }
 }
 
-pub fn expand_expr(expr: ast::Expr, fld: &mut MacroExpander) -> P<ast::Expr> {
+pub fn expand_expr(mut expr: ast::Expr, fld: &mut MacroExpander) -> P<ast::Expr> {
     match expr.node {
         // expr_mac should really be expr_ext or something; it's the
         // entry-point for all syntax extensions.
         ast::ExprKind::Mac(mac) => {
-            expand_mac_invoc(mac, None, expr.attrs.into_attr_vec(), expr.span, fld)
+            return expand_mac_invoc(mac, None, expr.attrs.into(), expr.span, fld);
         }
 
         ast::ExprKind::While(cond, body, opt_ident) => {
             let cond = fld.fold_expr(cond);
             let (body, opt_ident) = expand_loop_block(body, opt_ident, fld);
-            fld.cx.expr(expr.span, ast::ExprKind::While(cond, body, opt_ident))
-                .with_attrs(fold_thin_attrs(expr.attrs, fld))
+            expr.node = ast::ExprKind::While(cond, body, opt_ident);
         }
 
         ast::ExprKind::WhileLet(pat, cond, body, opt_ident) => {
@@ -129,14 +130,12 @@ pub fn expand_expr(expr: ast::Expr, fld: &mut MacroExpander) -> P<ast::Expr> {
             });
             assert!(rewritten_pats.len() == 1);
 
-            let wl = ast::ExprKind::WhileLet(rewritten_pats.remove(0), cond, body, opt_ident);
-            fld.cx.expr(expr.span, wl).with_attrs(fold_thin_attrs(expr.attrs, fld))
+            expr.node = ast::ExprKind::WhileLet(rewritten_pats.remove(0), cond, body, opt_ident);
         }
 
         ast::ExprKind::Loop(loop_block, opt_ident) => {
             let (loop_block, opt_ident) = expand_loop_block(loop_block, opt_ident, fld);
-            fld.cx.expr(expr.span, ast::ExprKind::Loop(loop_block, opt_ident))
-                .with_attrs(fold_thin_attrs(expr.attrs, fld))
+            expr.node = ast::ExprKind::Loop(loop_block, opt_ident);
         }
 
         ast::ExprKind::ForLoop(pat, head, body, opt_ident) => {
@@ -153,8 +152,7 @@ pub fn expand_expr(expr: ast::Expr, fld: &mut MacroExpander) -> P<ast::Expr> {
             assert!(rewritten_pats.len() == 1);
 
             let head = fld.fold_expr(head);
-            let fl = ast::ExprKind::ForLoop(rewritten_pats.remove(0), head, body, opt_ident);
-            fld.cx.expr(expr.span, fl).with_attrs(fold_thin_attrs(expr.attrs, fld))
+            expr.node = ast::ExprKind::ForLoop(rewritten_pats.remove(0), head, body, opt_ident);
         }
 
         ast::ExprKind::IfLet(pat, sub_expr, body, else_opt) => {
@@ -172,25 +170,21 @@ pub fn expand_expr(expr: ast::Expr, fld: &mut MacroExpander) -> P<ast::Expr> {
 
             let else_opt = else_opt.map(|else_opt| fld.fold_expr(else_opt));
             let sub_expr = fld.fold_expr(sub_expr);
-            let il = ast::ExprKind::IfLet(rewritten_pats.remove(0), sub_expr, body, else_opt);
-            fld.cx.expr(expr.span, il).with_attrs(fold_thin_attrs(expr.attrs, fld))
+            expr.node = ast::ExprKind::IfLet(rewritten_pats.remove(0), sub_expr, body, else_opt);
         }
 
         ast::ExprKind::Closure(capture_clause, fn_decl, block, fn_decl_span) => {
             let (rewritten_fn_decl, rewritten_block)
                 = expand_and_rename_fn_decl_and_block(fn_decl, block, fld);
-            let new_node = ast::ExprKind::Closure(capture_clause,
-                                                  rewritten_fn_decl,
-                                                  rewritten_block,
-                                                  fn_decl_span);
-            P(ast::Expr{ id: expr.id,
-                         node: new_node,
-                         span: expr.span,
-                         attrs: fold_thin_attrs(expr.attrs, fld) })
+            expr.node = ast::ExprKind::Closure(capture_clause,
+                                               rewritten_fn_decl,
+                                               rewritten_block,
+                                               fn_decl_span);
         }
 
-        _ => P(noop_fold_expr(expr, fld)),
-    }
+        _ => expr = noop_fold_expr(expr, fld),
+    };
+    P(expr)
 }
 
 /// Expand a macro invocation. Returns the result of expansion.
@@ -249,7 +243,7 @@ fn expand_mac_invoc<T>(mac: ast::Mac, ident: Option<Ident>, attrs: Vec<ast::Attr
                     },
                 });
 
-                let marked_tts = mark_tts(&tts[..], mark);
+                let marked_tts = mark_tts(tts, mark);
                 Some(expandfun.expand(fld.cx, call_site, &marked_tts))
             }
 
@@ -269,7 +263,7 @@ fn expand_mac_invoc<T>(mac: ast::Mac, ident: Option<Ident>, attrs: Vec<ast::Attr
                     }
                 });
 
-                let marked_tts = mark_tts(&tts, mark);
+                let marked_tts = mark_tts(tts, mark);
                 Some(expander.expand(fld.cx, call_site, ident, marked_tts))
             }
 
@@ -445,25 +439,25 @@ fn expand_stmt(stmt: Stmt, fld: &mut MacroExpander) -> SmallVector<Stmt> {
     };
 
     let (mac, style, attrs) = match stmt.node {
-        StmtKind::Mac(mac, style, attrs) => (mac, style, attrs),
+        StmtKind::Mac(mac) => mac.unwrap(),
         _ => return expand_non_macro_stmt(stmt, fld)
     };
 
     let mut fully_expanded: SmallVector<ast::Stmt> =
-        expand_mac_invoc(mac.unwrap(), None, attrs.into_attr_vec(), stmt.span, fld);
+        expand_mac_invoc(mac, None, attrs.into(), stmt.span, fld);
 
     // If this is a macro invocation with a semicolon, then apply that
     // semicolon to the final statement produced by expansion.
     if style == MacStmtStyle::Semicolon {
         if let Some(stmt) = fully_expanded.pop() {
-            let new_stmt = Spanned {
+            fully_expanded.push(Stmt {
+                id: stmt.id,
                 node: match stmt.node {
-                    StmtKind::Expr(e, stmt_id) => StmtKind::Semi(e, stmt_id),
+                    StmtKind::Expr(expr) => StmtKind::Semi(expr),
                     _ => stmt.node /* might already have a semi */
                 },
-                span: stmt.span
-            };
-            fully_expanded.push(new_stmt);
+                span: stmt.span,
+            });
         }
     }
 
@@ -472,73 +466,53 @@ fn expand_stmt(stmt: Stmt, fld: &mut MacroExpander) -> SmallVector<Stmt> {
 
 // expand a non-macro stmt. this is essentially the fallthrough for
 // expand_stmt, above.
-fn expand_non_macro_stmt(Spanned {node, span: stmt_span}: Stmt, fld: &mut MacroExpander)
+fn expand_non_macro_stmt(stmt: Stmt, fld: &mut MacroExpander)
                          -> SmallVector<Stmt> {
     // is it a let?
-    match node {
-        StmtKind::Decl(decl, node_id) => decl.and_then(|Spanned {node: decl, span}| match decl {
-            DeclKind::Local(local) => {
-                // take it apart:
-                let rewritten_local = local.map(|Local {id, pat, ty, init, span, attrs}| {
-                    // expand the ty since TyKind::FixedLengthVec contains an Expr
-                    // and thus may have a macro use
-                    let expanded_ty = ty.map(|t| fld.fold_ty(t));
-                    // expand the pat (it might contain macro uses):
-                    let expanded_pat = fld.fold_pat(pat);
-                    // find the PatIdents in the pattern:
-                    // oh dear heaven... this is going to include the enum
-                    // names, as well... but that should be okay, as long as
-                    // the new names are gensyms for the old ones.
-                    // generate fresh names, push them to a new pending list
-                    let idents = pattern_bindings(&expanded_pat);
-                    let mut new_pending_renames =
-                        idents.iter().map(|ident| (*ident, fresh_name(*ident))).collect();
-                    // rewrite the pattern using the new names (the old
-                    // ones have already been applied):
-                    let rewritten_pat = {
-                        // nested binding to allow borrow to expire:
-                        let mut rename_fld = IdentRenamer{renames: &mut new_pending_renames};
-                        rename_fld.fold_pat(expanded_pat)
-                    };
-                    // add them to the existing pending renames:
-                    fld.cx.syntax_env.info().pending_renames
-                          .extend(new_pending_renames);
-                    Local {
-                        id: id,
-                        ty: expanded_ty,
-                        pat: rewritten_pat,
-                        // also, don't forget to expand the init:
-                        init: init.map(|e| fld.fold_expr(e)),
-                        span: span,
-                        attrs: fold::fold_thin_attrs(attrs, fld),
-                    }
-                });
-                SmallVector::one(Spanned {
-                    node: StmtKind::Decl(P(Spanned {
-                            node: DeclKind::Local(rewritten_local),
-                            span: span
-                        }),
-                        node_id),
-                    span: stmt_span
-                })
-            }
-            _ => {
-                noop_fold_stmt(Spanned {
-                    node: StmtKind::Decl(P(Spanned {
-                            node: decl,
-                            span: span
-                        }),
-                        node_id),
-                    span: stmt_span
-                }, fld)
-            }
-        }),
-        _ => {
-            noop_fold_stmt(Spanned {
-                node: node,
-                span: stmt_span
-            }, fld)
+    match stmt.node {
+        StmtKind::Local(local) => {
+            // take it apart:
+            let rewritten_local = local.map(|Local {id, pat, ty, init, span, attrs}| {
+                // expand the ty since TyKind::FixedLengthVec contains an Expr
+                // and thus may have a macro use
+                let expanded_ty = ty.map(|t| fld.fold_ty(t));
+                // expand the pat (it might contain macro uses):
+                let expanded_pat = fld.fold_pat(pat);
+                // find the PatIdents in the pattern:
+                // oh dear heaven... this is going to include the enum
+                // names, as well... but that should be okay, as long as
+                // the new names are gensyms for the old ones.
+                // generate fresh names, push them to a new pending list
+                let idents = pattern_bindings(&expanded_pat);
+                let mut new_pending_renames =
+                    idents.iter().map(|ident| (*ident, fresh_name(*ident))).collect();
+                // rewrite the pattern using the new names (the old
+                // ones have already been applied):
+                let rewritten_pat = {
+                    // nested binding to allow borrow to expire:
+                    let mut rename_fld = IdentRenamer{renames: &mut new_pending_renames};
+                    rename_fld.fold_pat(expanded_pat)
+                };
+                // add them to the existing pending renames:
+                fld.cx.syntax_env.info().pending_renames
+                      .extend(new_pending_renames);
+                Local {
+                    id: id,
+                    ty: expanded_ty,
+                    pat: rewritten_pat,
+                    // also, don't forget to expand the init:
+                    init: init.map(|e| fld.fold_expr(e)),
+                    span: span,
+                    attrs: fold::fold_thin_attrs(attrs, fld),
+                }
+            });
+            SmallVector::one(Stmt {
+                id: stmt.id,
+                node: StmtKind::Local(rewritten_local),
+                span: stmt.span,
+            })
         }
+        _ => noop_fold_stmt(stmt, fld),
     }
 }
 
@@ -596,7 +570,7 @@ struct PatIdentFinder {
     ident_accumulator: Vec<ast::Ident>
 }
 
-impl<'v> Visitor<'v> for PatIdentFinder {
+impl Visitor for PatIdentFinder {
     fn visit_pat(&mut self, pattern: &ast::Pat) {
         match *pattern {
             ast::Pat { id: _, node: PatKind::Ident(_, ref path1, ref inner), span: _ } => {
@@ -637,23 +611,14 @@ pub fn expand_block(blk: P<Block>, fld: &mut MacroExpander) -> P<Block> {
 
 // expand the elements of a block.
 pub fn expand_block_elts(b: P<Block>, fld: &mut MacroExpander) -> P<Block> {
-    b.map(|Block {id, stmts, expr, rules, span}| {
+    b.map(|Block {id, stmts, rules, span}| {
         let new_stmts = stmts.into_iter().flat_map(|x| {
             // perform pending renames and expand macros in the statement
             fld.fold_stmt(x).into_iter()
         }).collect();
-        let new_expr = expr.map(|x| {
-            let expr = {
-                let pending_renames = &mut fld.cx.syntax_env.info().pending_renames;
-                let mut rename_fld = IdentRenamer{renames:pending_renames};
-                rename_fld.fold_expr(x)
-            };
-            fld.fold_expr(expr)
-        });
         Block {
             id: fld.new_id(id),
             stmts: new_stmts,
-            expr: new_expr,
             rules: rules,
             span: span
         }
@@ -754,25 +719,10 @@ fn expand_multi_modified(a: Annotatable, fld: &mut MacroExpander) -> SmallVector
             _ => noop_fold_item(it, fld),
         }.into_iter().map(|i| Annotatable::Item(i)).collect(),
 
-        Annotatable::TraitItem(it) => match it.node {
-            ast::TraitItemKind::Method(_, Some(_)) => {
-                let ti = it.unwrap();
-                SmallVector::one(ast::TraitItem {
-                    id: ti.id,
-                    ident: ti.ident,
-                    attrs: ti.attrs,
-                    node: match ti.node  {
-                        ast::TraitItemKind::Method(sig, Some(body)) => {
-                            let (sig, body) = expand_and_rename_method(sig, body, fld);
-                            ast::TraitItemKind::Method(sig, Some(body))
-                        }
-                        _ => unreachable!()
-                    },
-                    span: ti.span,
-                })
-            }
-            _ => fold::noop_fold_trait_item(it.unwrap(), fld)
-        }.into_iter().map(|ti| Annotatable::TraitItem(P(ti))).collect(),
+        Annotatable::TraitItem(it) => {
+            expand_trait_item(it.unwrap(), fld).into_iter().
+                map(|it| Annotatable::TraitItem(P(it))).collect()
+        }
 
         Annotatable::ImplItem(ii) => {
             expand_impl_item(ii.unwrap(), fld).into_iter().
@@ -900,6 +850,31 @@ fn expand_impl_item(ii: ast::ImplItem, fld: &mut MacroExpander)
     }
 }
 
+fn expand_trait_item(ti: ast::TraitItem, fld: &mut MacroExpander)
+                     -> SmallVector<ast::TraitItem> {
+    match ti.node {
+        ast::TraitItemKind::Method(_, Some(_)) => {
+            SmallVector::one(ast::TraitItem {
+                id: ti.id,
+                ident: ti.ident,
+                attrs: ti.attrs,
+                node: match ti.node  {
+                    ast::TraitItemKind::Method(sig, Some(body)) => {
+                        let (sig, body) = expand_and_rename_method(sig, body, fld);
+                        ast::TraitItemKind::Method(sig, Some(body))
+                    }
+                    _ => unreachable!()
+                },
+                span: ti.span,
+            })
+        }
+        ast::TraitItemKind::Macro(mac) => {
+            expand_mac_invoc(mac, None, ti.attrs, ti.span, fld)
+        }
+        _ => fold::noop_fold_trait_item(ti, fld)
+    }
+}
+
 /// Given a fn_decl and a block and a MacroExpander, expand the fn_decl, then use the
 /// PatIdents in its arguments to perform renaming in the FnDecl and
 /// the block, returning both the new FnDecl and the new Block.
@@ -980,9 +955,9 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
             at_crate_root: bool,
         }
 
-        impl<'a, 'b, 'v> Visitor<'v> for MacroLoadingVisitor<'a, 'b> {
-            fn visit_mac(&mut self, _: &'v ast::Mac) {}
-            fn visit_item(&mut self, item: &'v ast::Item) {
+        impl<'a, 'b> Visitor for MacroLoadingVisitor<'a, 'b> {
+            fn visit_mac(&mut self, _: &ast::Mac) {}
+            fn visit_item(&mut self, item: &ast::Item) {
                 if let ast::ItemKind::ExternCrate(..) = item.node {
                     // We need to error on `#[macro_use] extern crate` when it isn't at the
                     // crate root, because `$crate` won't work properly.
@@ -995,7 +970,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                     self.at_crate_root = at_crate_root;
                 }
             }
-            fn visit_block(&mut self, block: &'v ast::Block) {
+            fn visit_block(&mut self, block: &ast::Block) {
                 let at_crate_root = ::std::mem::replace(&mut self.at_crate_root, false);
                 visit::walk_block(self, block);
                 self.at_crate_root = at_crate_root;
@@ -1022,7 +997,7 @@ impl<'a, 'b> Folder for MacroExpander<'a, 'b> {
     fn fold_opt_expr(&mut self, expr: P<ast::Expr>) -> Option<P<ast::Expr>> {
         expr.and_then(|expr| match expr.node {
             ast::ExprKind::Mac(mac) =>
-                expand_mac_invoc(mac, None, expr.attrs.into_attr_vec(), expr.span, self),
+                expand_mac_invoc(mac, None, expr.attrs.into(), expr.span, self),
             _ => Some(expand_expr(expr, self)),
         })
     }
@@ -1040,7 +1015,7 @@ impl<'a, 'b> Folder for MacroExpander<'a, 'b> {
                 result = expand_item(item, self);
                 self.pop_mod_path();
             } else {
-                let filename = if inner != codemap::DUMMY_SP {
+                let filename = if inner != syntax_pos::DUMMY_SP {
                     Some(self.cx.parse_sess.codemap().span_to_filename(inner))
                 } else { None };
                 let orig_filename = replace(&mut self.cx.filename, filename);
@@ -1202,8 +1177,7 @@ impl Folder for Marker {
         Spanned {
             node: Mac_ {
                 path: self.fold_path(node.path),
-                tts: self.fold_tts(&node.tts),
-                ctxt: mtwt::apply_mark(self.mark, node.ctxt),
+                tts: self.fold_tts(node.tts),
             },
             span: self.new_span(span),
         }
@@ -1218,7 +1192,7 @@ impl Folder for Marker {
 }
 
 // apply a given mark to the given token trees. Used prior to expansion of a macro.
-fn mark_tts(tts: &[TokenTree], m: Mrk) -> Vec<TokenTree> {
+fn mark_tts(tts: Vec<TokenTree>, m: Mrk) -> Vec<TokenTree> {
     noop_fold_tts(tts, &mut Marker{mark:m, expn_id: None})
 }
 
@@ -1229,7 +1203,7 @@ mod tests {
     use super::{PatIdentFinder, IdentRenamer, PatIdentRenamer, ExpansionConfig};
     use ast;
     use ast::Name;
-    use codemap;
+    use syntax_pos;
     use ext::base::{ExtCtxt, DummyMacroLoader};
     use ext::mtwt;
     use fold::Folder;
@@ -1248,7 +1222,7 @@ mod tests {
         path_accumulator: Vec<ast::Path> ,
     }
 
-    impl<'v> Visitor<'v> for PathExprFinderContext {
+    impl Visitor for PathExprFinderContext {
         fn visit_expr(&mut self, expr: &ast::Expr) {
             if let ast::ExprKind::Path(None, ref p) = expr.node {
                 self.path_accumulator.push(p.clone());
@@ -1270,8 +1244,8 @@ mod tests {
         ident_accumulator: Vec<ast::Ident>
     }
 
-    impl<'v> Visitor<'v> for IdentFinder {
-        fn visit_ident(&mut self, _: codemap::Span, id: ast::Ident){
+    impl Visitor for IdentFinder {
+        fn visit_ident(&mut self, _: syntax_pos::Span, id: ast::Ident){
             self.ident_accumulator.push(id);
         }
     }
diff --git a/src/libsyntax/ext/quote.rs b/src/libsyntax/ext/quote.rs
index 871b0d4b1c0..68527b0797d 100644
--- a/src/libsyntax/ext/quote.rs
+++ b/src/libsyntax/ext/quote.rs
@@ -8,8 +8,8 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use ast::{self, Arg, Arm, Block, Expr, Item, Pat, Stmt, TokenTree, Ty};
-use codemap::Span;
+use ast::{self, Arg, Arm, Block, Expr, Item, Pat, Stmt, Ty};
+use syntax_pos::Span;
 use ext::base::ExtCtxt;
 use ext::base;
 use ext::build::AstBuilder;
@@ -17,6 +17,7 @@ use parse::parser::{Parser, PathStyle};
 use parse::token::*;
 use parse::token;
 use ptr::P;
+use tokenstream::{self, TokenTree};
 
 /// Quasiquoting works via token trees.
 ///
@@ -31,12 +32,12 @@ pub mod rt {
     use ext::base::ExtCtxt;
     use parse::{self, token, classify};
     use ptr::P;
-    use std::rc::Rc;
 
-    use ast::TokenTree;
+    use tokenstream::{self, TokenTree};
 
     pub use parse::new_parser_from_tts;
-    pub use codemap::{BytePos, Span, dummy_spanned, DUMMY_SP};
+    pub use syntax_pos::{BytePos, Span, DUMMY_SP};
+    pub use codemap::{dummy_spanned};
 
     pub trait ToTokens {
         fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree>;
@@ -214,12 +215,12 @@ pub mod rt {
             if self.node.style == ast::AttrStyle::Inner {
                 r.push(TokenTree::Token(self.span, token::Not));
             }
-            r.push(TokenTree::Delimited(self.span, Rc::new(ast::Delimited {
+            r.push(TokenTree::Delimited(self.span, tokenstream::Delimited {
                 delim: token::Bracket,
                 open_span: self.span,
                 tts: self.node.value.to_tokens(cx),
                 close_span: self.span,
-            })));
+            }));
             r
         }
     }
@@ -234,12 +235,12 @@ pub mod rt {
 
     impl ToTokens for () {
         fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
-            vec![TokenTree::Delimited(DUMMY_SP, Rc::new(ast::Delimited {
+            vec![TokenTree::Delimited(DUMMY_SP, tokenstream::Delimited {
                 delim: token::Paren,
                 open_span: DUMMY_SP,
                 tts: vec![],
                 close_span: DUMMY_SP,
-            }))]
+            })]
         }
     }
 
@@ -250,7 +251,7 @@ pub mod rt {
                 id: ast::DUMMY_NODE_ID,
                 node: ast::ExprKind::Lit(P(self.clone())),
                 span: DUMMY_SP,
-                attrs: None,
+                attrs: ast::ThinVec::new(),
             }).to_tokens(cx)
         }
     }
@@ -281,7 +282,7 @@ pub mod rt {
                         id: ast::DUMMY_NODE_ID,
                         node: ast::ExprKind::Lit(P(dummy_spanned(lit))),
                         span: DUMMY_SP,
-                        attrs: None,
+                        attrs: ast::ThinVec::new(),
                     });
                     if *self >= 0 {
                         return lit.to_tokens(cx);
@@ -290,7 +291,7 @@ pub mod rt {
                         id: ast::DUMMY_NODE_ID,
                         node: ast::ExprKind::Unary(ast::UnOp::Neg, lit),
                         span: DUMMY_SP,
-                        attrs: None,
+                        attrs: ast::ThinVec::new(),
                     }).to_tokens(cx)
                 }
             }
@@ -512,10 +513,8 @@ pub fn expand_quote_matcher(cx: &mut ExtCtxt,
     let (cx_expr, tts) = parse_arguments_to_quote(cx, tts);
     let mut vector = mk_stmts_let(cx, sp);
     vector.extend(statements_mk_tts(cx, &tts[..], true));
-    let block = cx.expr_block(
-        cx.block_all(sp,
-                     vector,
-                     Some(cx.expr_ident(sp, id_ext("tt")))));
+    vector.push(cx.stmt_expr(cx.expr_ident(sp, id_ext("tt"))));
+    let block = cx.expr_block(cx.block(sp, vector));
 
     let expanded = expand_wrapper(cx, sp, cx_expr, block, &[&["syntax", "ext", "quote", "rt"]]);
     base::MacEager::expr(expanded)
@@ -548,7 +547,7 @@ fn mk_name(cx: &ExtCtxt, sp: Span, ident: ast::Ident) -> P<ast::Expr> {
 }
 
 fn mk_tt_path(cx: &ExtCtxt, sp: Span, name: &str) -> P<ast::Expr> {
-    let idents = vec!(id_ext("syntax"), id_ext("ast"), id_ext("TokenTree"), id_ext(name));
+    let idents = vec!(id_ext("syntax"), id_ext("tokenstream"), id_ext("TokenTree"), id_ext(name));
     cx.expr_path(cx.path_global(sp, idents))
 }
 
@@ -765,19 +764,20 @@ fn statements_mk_tt(cx: &ExtCtxt, tt: &TokenTree, matcher: bool) -> Vec<ast::Stm
             let stmt_let_tt = cx.stmt_let(sp, true, id_ext("tt"), cx.expr_vec_ng(sp));
             let mut tts_stmts = vec![stmt_let_tt];
             tts_stmts.extend(statements_mk_tts(cx, &seq.tts[..], matcher));
-            let e_tts = cx.expr_block(cx.block(sp, tts_stmts,
-                                                   Some(cx.expr_ident(sp, id_ext("tt")))));
+            tts_stmts.push(cx.stmt_expr(cx.expr_ident(sp, id_ext("tt"))));
+            let e_tts = cx.expr_block(cx.block(sp, tts_stmts));
+
             let e_separator = match seq.separator {
                 Some(ref sep) => cx.expr_some(sp, expr_mk_token(cx, sp, sep)),
                 None => cx.expr_none(sp),
             };
             let e_op = match seq.op {
-                ast::KleeneOp::ZeroOrMore => "ZeroOrMore",
-                ast::KleeneOp::OneOrMore => "OneOrMore",
+                tokenstream::KleeneOp::ZeroOrMore => "ZeroOrMore",
+                tokenstream::KleeneOp::OneOrMore => "OneOrMore",
             };
             let e_op_idents = vec![
                 id_ext("syntax"),
-                id_ext("ast"),
+                id_ext("tokenstream"),
                 id_ext("KleeneOp"),
                 id_ext(e_op),
             ];
@@ -787,16 +787,13 @@ fn statements_mk_tt(cx: &ExtCtxt, tt: &TokenTree, matcher: bool) -> Vec<ast::Stm
                               cx.field_imm(sp, id_ext("op"), e_op),
                               cx.field_imm(sp, id_ext("num_captures"),
                                                cx.expr_usize(sp, seq.num_captures))];
-            let seq_path = vec![id_ext("syntax"), id_ext("ast"), id_ext("SequenceRepetition")];
+            let seq_path = vec![id_ext("syntax"),
+                                id_ext("tokenstream"),
+                                id_ext("SequenceRepetition")];
             let e_seq_struct = cx.expr_struct(sp, cx.path_global(sp, seq_path), fields);
-            let e_rc_new = cx.expr_call_global(sp, vec![id_ext("std"),
-                                                        id_ext("rc"),
-                                                        id_ext("Rc"),
-                                                        id_ext("new")],
-                                                   vec![e_seq_struct]);
             let e_tok = cx.expr_call(sp,
                                      mk_tt_path(cx, sp, "Sequence"),
-                                     vec!(e_sp, e_rc_new));
+                                     vec!(e_sp, e_seq_struct));
             let e_push =
                 cx.expr_method_call(sp,
                                     cx.expr_ident(sp, id_ext("tt")),
@@ -884,10 +881,8 @@ fn expand_tts(cx: &ExtCtxt, sp: Span, tts: &[TokenTree])
 
     let mut vector = mk_stmts_let(cx, sp);
     vector.extend(statements_mk_tts(cx, &tts[..], false));
-    let block = cx.expr_block(
-        cx.block_all(sp,
-                     vector,
-                     Some(cx.expr_ident(sp, id_ext("tt")))));
+    vector.push(cx.stmt_expr(cx.expr_ident(sp, id_ext("tt"))));
+    let block = cx.expr_block(cx.block(sp, vector));
 
     (cx_expr, block)
 }
@@ -901,13 +896,14 @@ fn expand_wrapper(cx: &ExtCtxt,
     let cx_expr_borrow = cx.expr_addr_of(sp, cx.expr_deref(sp, cx_expr));
     let stmt_let_ext_cx = cx.stmt_let(sp, false, id_ext("ext_cx"), cx_expr_borrow);
 
-    let stmts = imports.iter().map(|path| {
+    let mut stmts = imports.iter().map(|path| {
         // make item: `use ...;`
         let path = path.iter().map(|s| s.to_string()).collect();
         cx.stmt_item(sp, cx.item_use_glob(sp, ast::Visibility::Inherited, ids_ext(path)))
-    }).chain(Some(stmt_let_ext_cx)).collect();
+    }).chain(Some(stmt_let_ext_cx)).collect::<Vec<_>>();
+    stmts.push(cx.stmt_expr(expr));
 
-    cx.expr_block(cx.block_all(sp, stmts, Some(expr)))
+    cx.expr_block(cx.block(sp, stmts))
 }
 
 fn expand_parse_call(cx: &ExtCtxt,
diff --git a/src/libsyntax/ext/source_util.rs b/src/libsyntax/ext/source_util.rs
index d297188a35c..97cb09991ec 100644
--- a/src/libsyntax/ext/source_util.rs
+++ b/src/libsyntax/ext/source_util.rs
@@ -9,8 +9,7 @@
 // except according to those terms.
 
 use ast;
-use codemap::{Pos, Span};
-use codemap;
+use syntax_pos::{self, Pos, Span};
 use ext::base::*;
 use ext::base;
 use ext::build::AstBuilder;
@@ -18,6 +17,7 @@ use parse::token;
 use parse;
 use print::pprust;
 use ptr::P;
+use tokenstream;
 use util::small_vector::SmallVector;
 
 use std::fs::File;
@@ -30,7 +30,7 @@ use std::rc::Rc;
 // a given file into the current one.
 
 /// line!(): expands to the current line number
-pub fn expand_line(cx: &mut ExtCtxt, sp: Span, tts: &[ast::TokenTree])
+pub fn expand_line(cx: &mut ExtCtxt, sp: Span, tts: &[tokenstream::TokenTree])
                    -> Box<base::MacResult+'static> {
     base::check_zero_tts(cx, sp, tts, "line!");
 
@@ -41,7 +41,7 @@ pub fn expand_line(cx: &mut ExtCtxt, sp: Span, tts: &[ast::TokenTree])
 }
 
 /* column!(): expands to the current column number */
-pub fn expand_column(cx: &mut ExtCtxt, sp: Span, tts: &[ast::TokenTree])
+pub fn expand_column(cx: &mut ExtCtxt, sp: Span, tts: &[tokenstream::TokenTree])
                   -> Box<base::MacResult+'static> {
     base::check_zero_tts(cx, sp, tts, "column!");
 
@@ -54,7 +54,7 @@ pub fn expand_column(cx: &mut ExtCtxt, sp: Span, tts: &[ast::TokenTree])
 /// file!(): expands to the current filename */
 /// The filemap (`loc.file`) contains a bunch more information we could spit
 /// out if we wanted.
-pub fn expand_file(cx: &mut ExtCtxt, sp: Span, tts: &[ast::TokenTree])
+pub fn expand_file(cx: &mut ExtCtxt, sp: Span, tts: &[tokenstream::TokenTree])
                    -> Box<base::MacResult+'static> {
     base::check_zero_tts(cx, sp, tts, "file!");
 
@@ -64,14 +64,14 @@ pub fn expand_file(cx: &mut ExtCtxt, sp: Span, tts: &[ast::TokenTree])
     base::MacEager::expr(cx.expr_str(topmost, filename))
 }
 
-pub fn expand_stringify(cx: &mut ExtCtxt, sp: Span, tts: &[ast::TokenTree])
+pub fn expand_stringify(cx: &mut ExtCtxt, sp: Span, tts: &[tokenstream::TokenTree])
                         -> Box<base::MacResult+'static> {
     let s = pprust::tts_to_string(tts);
     base::MacEager::expr(cx.expr_str(sp,
                                    token::intern_and_get_ident(&s[..])))
 }
 
-pub fn expand_mod(cx: &mut ExtCtxt, sp: Span, tts: &[ast::TokenTree])
+pub fn expand_mod(cx: &mut ExtCtxt, sp: Span, tts: &[tokenstream::TokenTree])
                   -> Box<base::MacResult+'static> {
     base::check_zero_tts(cx, sp, tts, "module_path!");
     let string = cx.mod_path()
@@ -87,7 +87,7 @@ pub fn expand_mod(cx: &mut ExtCtxt, sp: Span, tts: &[ast::TokenTree])
 /// include! : parse the given file as an expr
 /// This is generally a bad idea because it's going to behave
 /// unhygienically.
-pub fn expand_include<'cx>(cx: &'cx mut ExtCtxt, sp: Span, tts: &[ast::TokenTree])
+pub fn expand_include<'cx>(cx: &'cx mut ExtCtxt, sp: Span, tts: &[tokenstream::TokenTree])
                            -> Box<base::MacResult+'cx> {
     let file = match get_single_str_from_tts(cx, sp, tts, "include!") {
         Some(f) => f,
@@ -130,7 +130,7 @@ pub fn expand_include<'cx>(cx: &'cx mut ExtCtxt, sp: Span, tts: &[ast::TokenTree
 }
 
 // include_str! : read the given file, insert it as a literal string expr
-pub fn expand_include_str(cx: &mut ExtCtxt, sp: Span, tts: &[ast::TokenTree])
+pub fn expand_include_str(cx: &mut ExtCtxt, sp: Span, tts: &[tokenstream::TokenTree])
                           -> Box<base::MacResult+'static> {
     let file = match get_single_str_from_tts(cx, sp, tts, "include_str!") {
         Some(f) => f,
@@ -167,7 +167,7 @@ pub fn expand_include_str(cx: &mut ExtCtxt, sp: Span, tts: &[ast::TokenTree])
     }
 }
 
-pub fn expand_include_bytes(cx: &mut ExtCtxt, sp: Span, tts: &[ast::TokenTree])
+pub fn expand_include_bytes(cx: &mut ExtCtxt, sp: Span, tts: &[tokenstream::TokenTree])
                             -> Box<base::MacResult+'static> {
     let file = match get_single_str_from_tts(cx, sp, tts, "include_bytes!") {
         Some(f) => f,
@@ -194,7 +194,7 @@ pub fn expand_include_bytes(cx: &mut ExtCtxt, sp: Span, tts: &[ast::TokenTree])
 
 // resolve a file-system path to an absolute file-system path (if it
 // isn't already)
-fn res_rel_file(cx: &mut ExtCtxt, sp: codemap::Span, arg: &Path) -> PathBuf {
+fn res_rel_file(cx: &mut ExtCtxt, sp: syntax_pos::Span, arg: &Path) -> PathBuf {
     // NB: relative paths are resolved relative to the compilation unit
     if !arg.is_absolute() {
         let callsite = cx.codemap().source_callsite(sp);
diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs
index ca5eb8f8003..813afb93576 100644
--- a/src/libsyntax/ext/tt/macro_parser.rs
+++ b/src/libsyntax/ext/tt/macro_parser.rs
@@ -79,9 +79,9 @@ pub use self::ParseResult::*;
 use self::TokenTreeOrTokenTreeVec::*;
 
 use ast;
-use ast::{TokenTree, Name, Ident};
-use codemap::{BytePos, mk_sp, Span, Spanned};
-use codemap;
+use ast::{Name, Ident};
+use syntax_pos::{self, BytePos, mk_sp, Span};
+use codemap::Spanned;
 use errors::FatalError;
 use parse::lexer::*; //resolve bug?
 use parse::ParseSess;
@@ -91,6 +91,7 @@ use parse::token::{Token, Nonterminal};
 use parse::token;
 use print::pprust;
 use ptr::P;
+use tokenstream::{self, TokenTree};
 
 use std::mem;
 use std::rc::Rc;
@@ -102,8 +103,8 @@ use std::collections::hash_map::Entry::{Vacant, Occupied};
 
 #[derive(Clone)]
 enum TokenTreeOrTokenTreeVec {
-    Tt(ast::TokenTree),
-    TtSeq(Rc<Vec<ast::TokenTree>>),
+    Tt(tokenstream::TokenTree),
+    TtSeq(Rc<Vec<tokenstream::TokenTree>>),
 }
 
 impl TokenTreeOrTokenTreeVec {
@@ -196,7 +197,7 @@ pub fn initial_matcher_pos(ms: Rc<Vec<TokenTree>>, sep: Option<Token>, lo: ByteP
 /// token tree it was derived from.
 
 pub enum NamedMatch {
-    MatchedSeq(Vec<Rc<NamedMatch>>, codemap::Span),
+    MatchedSeq(Vec<Rc<NamedMatch>>, syntax_pos::Span),
     MatchedNonterminal(Nonterminal)
 }
 
@@ -204,7 +205,7 @@ pub fn nameize(p_s: &ParseSess, ms: &[TokenTree], res: &[Rc<NamedMatch>])
             -> ParseResult<HashMap<Name, Rc<NamedMatch>>> {
     fn n_rec(p_s: &ParseSess, m: &TokenTree, res: &[Rc<NamedMatch>],
              ret_val: &mut HashMap<Name, Rc<NamedMatch>>, idx: &mut usize)
-             -> Result<(), (codemap::Span, String)> {
+             -> Result<(), (syntax_pos::Span, String)> {
         match *m {
             TokenTree::Sequence(_, ref seq) => {
                 for next_m in &seq.tts {
@@ -251,9 +252,9 @@ pub fn nameize(p_s: &ParseSess, ms: &[TokenTree], res: &[Rc<NamedMatch>])
 pub enum ParseResult<T> {
     Success(T),
     /// Arm failed to match
-    Failure(codemap::Span, String),
+    Failure(syntax_pos::Span, String),
     /// Fatal error (malformed macro?). Abort compilation.
-    Error(codemap::Span, String)
+    Error(syntax_pos::Span, String)
 }
 
 pub type NamedParseResult = ParseResult<HashMap<Name, Rc<NamedMatch>>>;
@@ -374,7 +375,7 @@ pub fn parse(sess: &ParseSess,
                 match ei.top_elts.get_tt(idx) {
                     /* need to descend into sequence */
                     TokenTree::Sequence(sp, seq) => {
-                        if seq.op == ast::KleeneOp::ZeroOrMore {
+                        if seq.op == tokenstream::KleeneOp::ZeroOrMore {
                             let mut new_ei = ei.clone();
                             new_ei.match_cur += seq.num_captures;
                             new_ei.idx += 1;
diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs
index bbe989b0f40..23f0b1fff0a 100644
--- a/src/libsyntax/ext/tt/macro_rules.rs
+++ b/src/libsyntax/ext/tt/macro_rules.rs
@@ -8,8 +8,8 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use ast::{self, TokenTree};
-use codemap::{Span, DUMMY_SP};
+use ast;
+use syntax_pos::{Span, DUMMY_SP};
 use ext::base::{DummyResult, ExtCtxt, MacResult, SyntaxExtension};
 use ext::base::{NormalTT, TTMacroExpander};
 use ext::tt::macro_parser::{Success, Error, Failure};
@@ -21,13 +21,13 @@ use parse::token::{self, gensym_ident, NtTT, Token};
 use parse::token::Token::*;
 use print;
 use ptr::P;
+use tokenstream::{self, TokenTree};
 
 use util::small_vector::SmallVector;
 
 use std::cell::RefCell;
 use std::collections::{HashMap};
 use std::collections::hash_map::{Entry};
-use std::rc::Rc;
 
 struct ParserAnyMacro<'a> {
     parser: RefCell<Parser<'a>>,
@@ -100,6 +100,21 @@ impl<'a> MacResult for ParserAnyMacro<'a> {
         Some(ret)
     }
 
+    fn make_trait_items(self: Box<ParserAnyMacro<'a>>)
+                       -> Option<SmallVector<ast::TraitItem>> {
+        let mut ret = SmallVector::zero();
+        loop {
+            let mut parser = self.parser.borrow_mut();
+            match parser.token {
+                token::Eof => break,
+                _ => ret.push(panictry!(parser.parse_trait_item()))
+            }
+        }
+        self.ensure_complete_parse(false, "item");
+        Some(ret)
+    }
+
+
     fn make_stmts(self: Box<ParserAnyMacro<'a>>)
                  -> Option<SmallVector<ast::Stmt>> {
         let mut ret = SmallVector::zero();
@@ -246,27 +261,25 @@ pub fn compile<'cx>(cx: &'cx mut ExtCtxt,
     // These spans won't matter, anyways
     let match_lhs_tok = MatchNt(lhs_nm, token::str_to_ident("tt"));
     let match_rhs_tok = MatchNt(rhs_nm, token::str_to_ident("tt"));
-    let argument_gram = vec!(
-        TokenTree::Sequence(DUMMY_SP,
-                   Rc::new(ast::SequenceRepetition {
-                       tts: vec![
-                           TokenTree::Token(DUMMY_SP, match_lhs_tok),
-                           TokenTree::Token(DUMMY_SP, token::FatArrow),
-                           TokenTree::Token(DUMMY_SP, match_rhs_tok)],
-                       separator: Some(token::Semi),
-                       op: ast::KleeneOp::OneOrMore,
-                       num_captures: 2
-                   })),
-        //to phase into semicolon-termination instead of
-        //semicolon-separation
-        TokenTree::Sequence(DUMMY_SP,
-                   Rc::new(ast::SequenceRepetition {
-                       tts: vec![TokenTree::Token(DUMMY_SP, token::Semi)],
-                       separator: None,
-                       op: ast::KleeneOp::ZeroOrMore,
-                       num_captures: 0
-                   })));
-
+    let argument_gram = vec![
+        TokenTree::Sequence(DUMMY_SP, tokenstream::SequenceRepetition {
+            tts: vec![
+                TokenTree::Token(DUMMY_SP, match_lhs_tok),
+                TokenTree::Token(DUMMY_SP, token::FatArrow),
+                TokenTree::Token(DUMMY_SP, match_rhs_tok),
+            ],
+            separator: Some(token::Semi),
+            op: tokenstream::KleeneOp::OneOrMore,
+            num_captures: 2,
+        }),
+        // to phase into semicolon-termination instead of semicolon-separation
+        TokenTree::Sequence(DUMMY_SP, tokenstream::SequenceRepetition {
+            tts: vec![TokenTree::Token(DUMMY_SP, token::Semi)],
+            separator: None,
+            op: tokenstream::KleeneOp::ZeroOrMore,
+            num_captures: 0
+        }),
+    ];
 
     // Parse the macro_rules! invocation (`none` is for no interpolations):
     let arg_reader = new_tt_reader(&cx.parse_sess().span_diagnostic,
@@ -427,7 +440,7 @@ impl FirstSets {
                         }
 
                         // Reverse scan: Sequence comes before `first`.
-                        if subfirst.maybe_empty || seq_rep.op == ast::KleeneOp::ZeroOrMore {
+                        if subfirst.maybe_empty || seq_rep.op == tokenstream::KleeneOp::ZeroOrMore {
                             // If sequence is potentially empty, then
                             // union them (preserving first emptiness).
                             first.add_all(&TokenSet { maybe_empty: true, ..subfirst });
@@ -474,7 +487,8 @@ impl FirstSets {
 
                             assert!(first.maybe_empty);
                             first.add_all(subfirst);
-                            if subfirst.maybe_empty || seq_rep.op == ast::KleeneOp::ZeroOrMore {
+                            if subfirst.maybe_empty ||
+                               seq_rep.op == tokenstream::KleeneOp::ZeroOrMore {
                                 // continue scanning for more first
                                 // tokens, but also make sure we
                                 // restore empty-tracking state
diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/tt/transcribe.rs
index 6b3b5ce9de9..58328eb4246 100644
--- a/src/libsyntax/ext/tt/transcribe.rs
+++ b/src/libsyntax/ext/tt/transcribe.rs
@@ -9,15 +9,15 @@
 // except according to those terms.
 use self::LockstepIterSize::*;
 
-use ast;
-use ast::{TokenTree, Ident, Name};
-use codemap::{Span, DUMMY_SP};
+use ast::{Ident, Name};
+use syntax_pos::{Span, DUMMY_SP};
 use errors::{Handler, DiagnosticBuilder};
 use ext::tt::macro_parser::{NamedMatch, MatchedSeq, MatchedNonterminal};
 use parse::token::{DocComment, MatchNt, SubstNt};
 use parse::token::{Token, NtIdent, SpecialMacroVar};
 use parse::token;
 use parse::lexer::TokenAndSpan;
+use tokenstream::{self, TokenTree};
 
 use std::rc::Rc;
 use std::ops::Add;
@@ -59,7 +59,7 @@ pub struct TtReader<'a> {
 pub fn new_tt_reader(sp_diag: &Handler,
                      interp: Option<HashMap<Name, Rc<NamedMatch>>>,
                      imported_from: Option<Ident>,
-                     src: Vec<ast::TokenTree>)
+                     src: Vec<tokenstream::TokenTree>)
                      -> TtReader {
     new_tt_reader_with_doc_flag(sp_diag, interp, imported_from, src, false)
 }
@@ -73,17 +73,17 @@ pub fn new_tt_reader(sp_diag: &Handler,
 pub fn new_tt_reader_with_doc_flag(sp_diag: &Handler,
                                    interp: Option<HashMap<Name, Rc<NamedMatch>>>,
                                    imported_from: Option<Ident>,
-                                   src: Vec<ast::TokenTree>,
+                                   src: Vec<tokenstream::TokenTree>,
                                    desugar_doc_comments: bool)
                                    -> TtReader {
     let mut r = TtReader {
         sp_diag: sp_diag,
         stack: vec!(TtFrame {
-            forest: TokenTree::Sequence(DUMMY_SP, Rc::new(ast::SequenceRepetition {
+            forest: TokenTree::Sequence(DUMMY_SP, tokenstream::SequenceRepetition {
                 tts: src,
                 // doesn't matter. This merely holds the root unzipping.
-                separator: None, op: ast::KleeneOp::ZeroOrMore, num_captures: 0
-            })),
+                separator: None, op: tokenstream::KleeneOp::ZeroOrMore, num_captures: 0
+            }),
             idx: 0,
             dotdotdoted: false,
             sep: None,
@@ -259,7 +259,7 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan {
                     }
                     LisConstraint(len, _) => {
                         if len == 0 {
-                            if seq.op == ast::KleeneOp::OneOrMore {
+                            if seq.op == tokenstream::KleeneOp::OneOrMore {
                                 // FIXME #2887 blame invoker
                                 panic!(r.sp_diag.span_fatal(sp.clone(),
                                                      "this must repeat at least once"));