Merge branch 'master' into issue-32540

10 files changed, 155 insertions, 307 deletions

diff --git a/src/libsyntax/ext/base.rs b/src/libsyntax/ext/base.rs
index dc7e7673eb0..fda026fec64 100644
--- a/src/libsyntax/ext/base.rs
+++ b/src/libsyntax/ext/base.rs
@@ -12,11 +12,11 @@ pub use self::SyntaxExtension::{MultiDecorator, MultiModifier, NormalTT, IdentTT
 
 use ast::{self, Attribute, Name, PatKind, MetaItem};
 use attr::HasAttrs;
-use codemap::{self, CodeMap, ExpnInfo, Spanned, respan};
-use syntax_pos::{Span, ExpnId, NO_EXPANSION};
-use errors::{DiagnosticBuilder, FatalError};
+use codemap::{self, CodeMap, Spanned, respan};
+use syntax_pos::{Span, DUMMY_SP};
+use errors::DiagnosticBuilder;
 use ext::expand::{self, Expansion, Invocation};
-use ext::hygiene::Mark;
+use ext::hygiene::{Mark, SyntaxContext};
 use fold::{self, Folder};
 use parse::{self, parser, DirectoryOwnership};
 use parse::token;
@@ -56,6 +56,14 @@ impl HasAttrs for Annotatable {
 }
 
 impl Annotatable {
+    pub fn span(&self) -> Span {
+        match *self {
+            Annotatable::Item(ref item) => item.span,
+            Annotatable::TraitItem(ref trait_item) => trait_item.span,
+            Annotatable::ImplItem(ref impl_item) => impl_item.span,
+        }
+    }
+
     pub fn expect_item(self) -> P<ast::Item> {
         match self {
             Annotatable::Item(i) => i,
@@ -201,7 +209,26 @@ impl<F> TTMacroExpander for F
 {
     fn expand<'cx>(&self, ecx: &'cx mut ExtCtxt, span: Span, input: TokenStream)
                    -> Box<MacResult+'cx> {
-        (*self)(ecx, span, &input.trees().collect::<Vec<_>>())
+        struct AvoidInterpolatedIdents;
+
+        impl Folder for AvoidInterpolatedIdents {
+            fn fold_tt(&mut self, tt: tokenstream::TokenTree) -> tokenstream::TokenTree {
+                if let tokenstream::TokenTree::Token(_, token::Interpolated(ref nt)) = tt {
+                    if let token::NtIdent(ident) = **nt {
+                        return tokenstream::TokenTree::Token(ident.span, token::Ident(ident.node));
+                    }
+                }
+                fold::noop_fold_tt(tt, self)
+            }
+
+            fn fold_mac(&mut self, mac: ast::Mac) -> ast::Mac {
+                fold::noop_fold_mac(mac, self)
+            }
+        }
+
+        let input: Vec<_> =
+            input.trees().map(|tt| AvoidInterpolatedIdents.fold_tt(tt)).collect();
+        (*self)(ecx, span, &input)
     }
 }
 
@@ -602,7 +629,6 @@ pub struct ModuleData {
 pub struct ExpansionData {
     pub mark: Mark,
     pub depth: usize,
-    pub backtrace: ExpnId,
     pub module: Rc<ModuleData>,
     pub directory_ownership: DirectoryOwnership,
 }
@@ -633,7 +659,6 @@ impl<'a> ExtCtxt<'a> {
             current_expansion: ExpansionData {
                 mark: Mark::root(),
                 depth: 0,
-                backtrace: NO_EXPANSION,
                 module: Rc::new(ModuleData { mod_path: Vec::new(), directory: PathBuf::new() }),
                 directory_ownership: DirectoryOwnership::Owned,
             },
@@ -658,30 +683,30 @@ impl<'a> ExtCtxt<'a> {
     pub fn parse_sess(&self) -> &'a parse::ParseSess { self.parse_sess }
     pub fn cfg(&self) -> &ast::CrateConfig { &self.parse_sess.config }
     pub fn call_site(&self) -> Span {
-        self.codemap().with_expn_info(self.backtrace(), |ei| match ei {
+        match self.current_expansion.mark.expn_info() {
             Some(expn_info) => expn_info.call_site,
-            None => self.bug("missing top span")
-        })
+            None => DUMMY_SP,
+        }
+    }
+    pub fn backtrace(&self) -> SyntaxContext {
+        SyntaxContext::empty().apply_mark(self.current_expansion.mark)
     }
-    pub fn backtrace(&self) -> ExpnId { self.current_expansion.backtrace }
 
     /// Returns span for the macro which originally caused the current expansion to happen.
     ///
     /// Stops backtracing at include! boundary.
     pub fn expansion_cause(&self) -> Span {
-        let mut expn_id = self.backtrace();
+        let mut ctxt = self.backtrace();
         let mut last_macro = None;
         loop {
-            if self.codemap().with_expn_info(expn_id, |info| {
-                info.map_or(None, |i| {
-                    if i.callee.name() == "include" {
-                        // Stop going up the backtrace once include! is encountered
-                        return None;
-                    }
-                    expn_id = i.call_site.expn_id;
-                    last_macro = Some(i.call_site);
-                    return Some(());
-                })
+            if ctxt.outer().expn_info().map_or(None, |info| {
+                if info.callee.name() == "include" {
+                    // Stop going up the backtrace once include! is encountered
+                    return None;
+                }
+                ctxt = info.call_site.ctxt;
+                last_macro = Some(info.call_site);
+                return Some(());
             }).is_none() {
                 break
             }
@@ -689,28 +714,6 @@ impl<'a> ExtCtxt<'a> {
         last_macro.expect("missing expansion backtrace")
     }
 
-    pub fn bt_push(&mut self, ei: ExpnInfo) {
-        if self.current_expansion.depth > self.ecfg.recursion_limit {
-            let suggested_limit = self.ecfg.recursion_limit * 2;
-            let mut err = self.struct_span_fatal(ei.call_site,
-                &format!("recursion limit reached while expanding the macro `{}`",
-                         ei.callee.name()));
-            err.help(&format!(
-                "consider adding a `#![recursion_limit=\"{}\"]` attribute to your crate",
-                suggested_limit));
-            err.emit();
-            panic!(FatalError);
-        }
-
-        let mut call_site = ei.call_site;
-        call_site.expn_id = self.backtrace();
-        self.current_expansion.backtrace = self.codemap().record_expansion(ExpnInfo {
-            call_site: call_site,
-            callee: ei.callee
-        });
-    }
-    pub fn bt_pop(&mut self) {}
-
     pub fn struct_span_warn(&self,
                             sp: Span,
                             msg: &str)
@@ -792,9 +795,9 @@ impl<'a> ExtCtxt<'a> {
 /// compilation on error, merely emits a non-fatal error and returns None.
 pub fn expr_to_spanned_string(cx: &mut ExtCtxt, expr: P<ast::Expr>, err_msg: &str)
                               -> Option<Spanned<(Symbol, ast::StrStyle)>> {
-    // Update `expr.span`'s expn_id now in case expr is an `include!` macro invocation.
+    // Update `expr.span`'s ctxt now in case expr is an `include!` macro invocation.
     let expr = expr.map(|mut expr| {
-        expr.span.expn_id = cx.backtrace();
+        expr.span.ctxt = expr.span.ctxt.apply_mark(cx.current_expansion.mark);
         expr
     });
 
diff --git a/src/libsyntax/ext/derive.rs b/src/libsyntax/ext/derive.rs
index 1569d9f540b..c79040424f6 100644
--- a/src/libsyntax/ext/derive.rs
+++ b/src/libsyntax/ext/derive.rs
@@ -9,13 +9,16 @@
 // except according to those terms.
 
 use attr::HasAttrs;
-use {ast, codemap};
+use ast;
+use codemap::{ExpnInfo, NameAndSpan, ExpnFormat};
 use ext::base::ExtCtxt;
 use ext::build::AstBuilder;
 use parse::parser::PathStyle;
 use symbol::Symbol;
 use syntax_pos::Span;
 
+use std::collections::HashSet;
+
 pub fn collect_derives(cx: &mut ExtCtxt, attrs: &mut Vec<ast::Attribute>) -> Vec<ast::Path> {
     let mut result = Vec::new();
     attrs.retain(|attr| {
@@ -41,36 +44,35 @@ pub fn collect_derives(cx: &mut ExtCtxt, attrs: &mut Vec<ast::Attribute>) -> Vec
     result
 }
 
-fn allow_unstable(cx: &mut ExtCtxt, span: Span, attr_name: &str) -> Span {
-    Span {
-        expn_id: cx.codemap().record_expansion(codemap::ExpnInfo {
-            call_site: span,
-            callee: codemap::NameAndSpan {
-                format: codemap::MacroAttribute(Symbol::intern(attr_name)),
-                span: Some(span),
-                allow_internal_unstable: true,
-            },
-        }),
-        ..span
+pub fn add_derived_markers<T>(cx: &mut ExtCtxt, span: Span, traits: &[ast::Path], item: T) -> T
+    where T: HasAttrs,
+{
+    let (mut names, mut pretty_name) = (HashSet::new(), "derive(".to_owned());
+    for (i, path) in traits.iter().enumerate() {
+        if i > 0 {
+            pretty_name.push_str(", ");
+        }
+        pretty_name.push_str(&path.to_string());
+        names.insert(unwrap_or!(path.segments.get(0), continue).identifier.name);
     }
-}
+    pretty_name.push(')');
 
-pub fn add_derived_markers<T: HasAttrs>(cx: &mut ExtCtxt, traits: &[ast::Path], item: T) -> T {
-    let span = match traits.get(0) {
-        Some(path) => path.span,
-        None => return item,
-    };
+    cx.current_expansion.mark.set_expn_info(ExpnInfo {
+        call_site: span,
+        callee: NameAndSpan {
+            format: ExpnFormat::MacroAttribute(Symbol::intern(&pretty_name)),
+            span: None,
+            allow_internal_unstable: true,
+        },
+    });
 
+    let span = Span { ctxt: cx.backtrace(), ..span };
     item.map_attrs(|mut attrs| {
-        if traits.iter().any(|path| *path == "PartialEq") &&
-           traits.iter().any(|path| *path == "Eq") {
-            let span = allow_unstable(cx, span, "derive(PartialEq, Eq)");
+        if names.contains(&Symbol::intern("Eq")) && names.contains(&Symbol::intern("PartialEq")) {
             let meta = cx.meta_word(span, Symbol::intern("structural_match"));
             attrs.push(cx.attribute(span, meta));
         }
-        if traits.iter().any(|path| *path == "Copy") &&
-           traits.iter().any(|path| *path == "Clone") {
-            let span = allow_unstable(cx, span, "derive(Copy, Clone)");
+        if names.contains(&Symbol::intern("Copy")) && names.contains(&Symbol::intern("Clone")) {
             let meta = cx.meta_word(span, Symbol::intern("rustc_copy_clone_marker"));
             attrs.push(cx.attribute(span, meta));
         }
diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index 6abeb4b0b28..1b3352f73ad 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -8,11 +8,12 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use ast::{self, Block, Ident, PatKind, Path};
+use ast::{self, Block, Ident, NodeId, PatKind, Path};
 use ast::{MacStmtStyle, StmtKind, ItemKind};
 use attr::{self, HasAttrs};
 use codemap::{ExpnInfo, NameAndSpan, MacroBang, MacroAttribute};
 use config::{is_test_or_bench, StripUnconfigured};
+use errors::FatalError;
 use ext::base::*;
 use ext::derive::{add_derived_markers, collect_derives};
 use ext::hygiene::Mark;
@@ -27,7 +28,7 @@ use ptr::P;
 use std_inject;
 use symbol::Symbol;
 use symbol::keywords;
-use syntax_pos::{Span, ExpnId, DUMMY_SP};
+use syntax_pos::{Span, DUMMY_SP};
 use tokenstream::TokenStream;
 use util::small_vector::SmallVector;
 use visit::Visitor;
@@ -273,7 +274,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                     let item = item
                         .map_attrs(|mut attrs| { attrs.retain(|a| a.path != "derive"); attrs });
                     let item_with_markers =
-                        add_derived_markers(&mut self.cx, &traits, item.clone());
+                        add_derived_markers(&mut self.cx, item.span(), &traits, item.clone());
                     let derives = derives.entry(invoc.expansion_data.mark).or_insert_with(Vec::new);
 
                     for path in &traits {
@@ -321,7 +322,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
         while let Some(expansions) = expansions.pop() {
             for (mark, expansion) in expansions.into_iter().rev() {
                 let derives = derives.remove(&mark).unwrap_or_else(Vec::new);
-                placeholder_expander.add(mark.as_placeholder_id(), expansion, derives);
+                placeholder_expander.add(NodeId::placeholder_from_mark(mark), expansion, derives);
             }
         }
 
@@ -363,11 +364,26 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
     }
 
     fn expand_invoc(&mut self, invoc: Invocation, ext: Rc<SyntaxExtension>) -> Expansion {
-        match invoc.kind {
+        let result = match invoc.kind {
             InvocationKind::Bang { .. } => self.expand_bang_invoc(invoc, ext),
             InvocationKind::Attr { .. } => self.expand_attr_invoc(invoc, ext),
             InvocationKind::Derive { .. } => self.expand_derive_invoc(invoc, ext),
+        };
+
+        if self.cx.current_expansion.depth > self.cx.ecfg.recursion_limit {
+            let info = self.cx.current_expansion.mark.expn_info().unwrap();
+            let suggested_limit = self.cx.ecfg.recursion_limit * 2;
+            let mut err = self.cx.struct_span_fatal(info.call_site,
+                &format!("recursion limit reached while expanding the macro `{}`",
+                         info.callee.name()));
+            err.help(&format!(
+                "consider adding a `#![recursion_limit=\"{}\"]` attribute to your crate",
+                suggested_limit));
+            err.emit();
+            panic!(FatalError);
         }
+
+        result
     }
 
     fn expand_attr_invoc(&mut self, invoc: Invocation, ext: Rc<SyntaxExtension>) -> Expansion {
@@ -378,11 +394,11 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
         };
 
         attr::mark_used(&attr);
-        self.cx.bt_push(ExpnInfo {
+        invoc.expansion_data.mark.set_expn_info(ExpnInfo {
             call_site: attr.span,
             callee: NameAndSpan {
                 format: MacroAttribute(Symbol::intern(&format!("{}", attr.path))),
-                span: Some(attr.span),
+                span: None,
                 allow_internal_unstable: false,
             }
         });
@@ -403,19 +419,8 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
             SyntaxExtension::AttrProcMacro(ref mac) => {
                 let item_toks = stream_for_item(&item, &self.cx.parse_sess);
 
-                let span = Span {
-                    expn_id: self.cx.codemap().record_expansion(ExpnInfo {
-                        call_site: attr.span,
-                        callee: NameAndSpan {
-                            format: MacroAttribute(Symbol::intern(&format!("{}", attr.path))),
-                            span: None,
-                            allow_internal_unstable: false,
-                        },
-                    }),
-                    ..attr.span
-                };
-
-                let tok_result = mac.expand(self.cx, attr.span, attr.tokens.clone(), item_toks);
+                let span = Span { ctxt: self.cx.backtrace(), ..attr.span };
+                let tok_result = mac.expand(self.cx, attr.span, attr.tokens, item_toks);
                 self.parse_expansion(tok_result, kind, &attr.path, span)
             }
             SyntaxExtension::ProcMacroDerive(..) | SyntaxExtension::BuiltinDerive(..) => {
@@ -440,8 +445,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
         let path = &mac.node.path;
 
         let ident = ident.unwrap_or(keywords::Invalid.ident());
-        let marked_tts =
-            noop_fold_tts(mac.node.stream(), &mut Marker { mark: mark, expn_id: None });
+        let marked_tts = noop_fold_tts(mac.node.stream(), &mut Marker(mark));
         let opt_expanded = match *ext {
             NormalTT(ref expandfun, exp_span, allow_internal_unstable) => {
                 if ident.name != keywords::Invalid.name() {
@@ -451,7 +455,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                     return kind.dummy(span);
                 }
 
-                self.cx.bt_push(ExpnInfo {
+                invoc.expansion_data.mark.set_expn_info(ExpnInfo {
                     call_site: span,
                     callee: NameAndSpan {
                         format: MacroBang(Symbol::intern(&format!("{}", path))),
@@ -470,7 +474,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                     return kind.dummy(span);
                 };
 
-                self.cx.bt_push(ExpnInfo {
+                invoc.expansion_data.mark.set_expn_info(ExpnInfo {
                     call_site: span,
                     callee: NameAndSpan {
                         format: MacroBang(Symbol::intern(&format!("{}", path))),
@@ -502,7 +506,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                     return kind.dummy(span);
                 }
 
-                self.cx.bt_push(ExpnInfo {
+                invoc.expansion_data.mark.set_expn_info(ExpnInfo {
                     call_site: span,
                     callee: NameAndSpan {
                         format: MacroBang(Symbol::intern(&format!("{}", path))),
@@ -528,10 +532,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
             return kind.dummy(span);
         };
 
-        expanded.fold_with(&mut Marker {
-            mark: mark,
-            expn_id: Some(self.cx.backtrace()),
-        })
+        expanded.fold_with(&mut Marker(mark))
     }
 
     /// Expand a derive invocation. Returns the result of expansion.
@@ -550,50 +551,33 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
             id: ast::AttrId(0), style: ast::AttrStyle::Outer, is_sugared_doc: false,
         };
 
-        self.cx.bt_push(ExpnInfo {
+        let mut expn_info = ExpnInfo {
             call_site: span,
             callee: NameAndSpan {
                 format: MacroAttribute(pretty_name),
                 span: None,
                 allow_internal_unstable: false,
             }
-        });
+        };
 
         match *ext {
             SyntaxExtension::ProcMacroDerive(ref ext, _) => {
-                let span = Span {
-                    expn_id: self.cx.codemap().record_expansion(ExpnInfo {
-                        call_site: span,
-                        callee: NameAndSpan {
-                            format: MacroAttribute(pretty_name),
-                            span: None,
-                            allow_internal_unstable: false,
-                        },
-                    }),
-                    ..span
-                };
+                invoc.expansion_data.mark.set_expn_info(expn_info);
+                let span = Span { ctxt: self.cx.backtrace(), ..span };
                 let dummy = ast::MetaItem { // FIXME(jseyfried) avoid this
                     name: keywords::Invalid.name(),
                     span: DUMMY_SP,
                     node: ast::MetaItemKind::Word,
                 };
-                return kind.expect_from_annotatables(ext.expand(self.cx, span, &dummy, item));
+                kind.expect_from_annotatables(ext.expand(self.cx, span, &dummy, item))
             }
             SyntaxExtension::BuiltinDerive(func) => {
-                let span = Span {
-                    expn_id: self.cx.codemap().record_expansion(ExpnInfo {
-                        call_site: span,
-                        callee: NameAndSpan {
-                            format: MacroAttribute(pretty_name),
-                            span: None,
-                            allow_internal_unstable: true,
-                        },
-                    }),
-                    ..span
-                };
+                expn_info.callee.allow_internal_unstable = true;
+                invoc.expansion_data.mark.set_expn_info(expn_info);
+                let span = Span { ctxt: self.cx.backtrace(), ..span };
                 let mut items = Vec::new();
                 func(self.cx, span, &attr.meta().unwrap(), &item, &mut |a| items.push(a));
-                return kind.expect_from_annotatables(items);
+                kind.expect_from_annotatables(items)
             }
             _ => {
                 let msg = &format!("macro `{}` may not be used for derive attributes", attr.path);
@@ -703,7 +687,7 @@ impl<'a, 'b> InvocationCollector<'a, 'b> {
                 ..self.cx.current_expansion.clone()
             },
         });
-        placeholder(expansion_kind, mark.as_placeholder_id())
+        placeholder(expansion_kind, NodeId::placeholder_from_mark(mark))
     }
 
     fn collect_bang(&mut self, mac: ast::Mac, span: Span, kind: ExpansionKind) -> Expansion {
@@ -753,10 +737,9 @@ impl<'a, 'b> InvocationCollector<'a, 'b> {
     // Detect use of feature-gated or invalid attributes on macro invocations
     // since they will not be detected after macro expansion.
     fn check_attributes(&mut self, attrs: &[ast::Attribute]) {
-        let codemap = &self.cx.parse_sess.codemap();
         let features = self.cx.ecfg.features.unwrap();
         for attr in attrs.iter() {
-            feature_gate::check_attribute(&attr, &self.cx.parse_sess, codemap, features);
+            feature_gate::check_attribute(&attr, &self.cx.parse_sess, features);
         }
     }
 }
@@ -1065,23 +1048,21 @@ impl<'feat> ExpansionConfig<'feat> {
     }
 }
 
-// A Marker adds the given mark to the syntax context and
-// sets spans' `expn_id` to the given expn_id (unless it is `None`).
-struct Marker { mark: Mark, expn_id: Option<ExpnId> }
+// A Marker adds the given mark to the syntax context.
+struct Marker(Mark);
 
 impl Folder for Marker {
     fn fold_ident(&mut self, mut ident: Ident) -> Ident {
-        ident.ctxt = ident.ctxt.apply_mark(self.mark);
+        ident.ctxt = ident.ctxt.apply_mark(self.0);
         ident
     }
-    fn fold_mac(&mut self, mac: ast::Mac) -> ast::Mac {
-        noop_fold_mac(mac, self)
-    }
 
     fn new_span(&mut self, mut span: Span) -> Span {
-        if let Some(expn_id) = self.expn_id {
-            span.expn_id = expn_id;
-        }
+        span.ctxt = span.ctxt.apply_mark(self.0);
         span
     }
+
+    fn fold_mac(&mut self, mac: ast::Mac) -> ast::Mac {
+        noop_fold_mac(mac, self)
+    }
 }
diff --git a/src/libsyntax/ext/hygiene.rs b/src/libsyntax/ext/hygiene.rs
deleted file mode 100644
index 57f5ab73d37..00000000000
--- a/src/libsyntax/ext/hygiene.rs
+++ /dev/null
@@ -1,127 +0,0 @@
-// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
-// file at the top-level directory of this distribution and at
-// http://rust-lang.org/COPYRIGHT.
-//
-// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
-// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
-// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
-// option. This file may not be copied, modified, or distributed
-// except according to those terms.
-
-//! Machinery for hygienic macros, inspired by the MTWT[1] paper.
-//!
-//! [1] Matthew Flatt, Ryan Culpepper, David Darais, and Robert Bruce Findler.
-//! 2012. *Macros that work together: Compile-time bindings, partial expansion,
-//! and definition contexts*. J. Funct. Program. 22, 2 (March 2012), 181-216.
-//! DOI=10.1017/S0956796812000093 http://dx.doi.org/10.1017/S0956796812000093
-
-use ast::NodeId;
-use std::cell::RefCell;
-use std::collections::HashMap;
-use std::fmt;
-
-/// A SyntaxContext represents a chain of macro expansions (represented by marks).
-#[derive(Clone, Copy, PartialEq, Eq, Hash, RustcEncodable, RustcDecodable, Default)]
-pub struct SyntaxContext(u32);
-
-#[derive(Copy, Clone)]
-pub struct SyntaxContextData {
-    pub outer_mark: Mark,
-    pub prev_ctxt: SyntaxContext,
-}
-
-/// A mark is a unique id associated with a macro expansion.
-#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, Default, RustcEncodable, RustcDecodable)]
-pub struct Mark(u32);
-
-impl Mark {
-    pub fn fresh() -> Self {
-        HygieneData::with(|data| {
-            let next_mark = Mark(data.next_mark.0 + 1);
-            ::std::mem::replace(&mut data.next_mark, next_mark)
-        })
-    }
-
-    /// The mark of the theoretical expansion that generates freshly parsed, unexpanded AST.
-    pub fn root() -> Self {
-        Mark(0)
-    }
-
-    pub fn from_placeholder_id(id: NodeId) -> Self {
-        Mark(id.as_u32())
-    }
-
-    pub fn as_placeholder_id(self) -> NodeId {
-        NodeId::from_u32(self.0)
-    }
-
-    pub fn as_u32(self) -> u32 {
-        self.0
-    }
-}
-
-struct HygieneData {
-    syntax_contexts: Vec<SyntaxContextData>,
-    markings: HashMap<(SyntaxContext, Mark), SyntaxContext>,
-    next_mark: Mark,
-}
-
-impl HygieneData {
-    fn new() -> Self {
-        HygieneData {
-            syntax_contexts: vec![SyntaxContextData {
-                outer_mark: Mark::root(),
-                prev_ctxt: SyntaxContext::empty(),
-            }],
-            markings: HashMap::new(),
-            next_mark: Mark(1),
-        }
-    }
-
-    fn with<T, F: FnOnce(&mut HygieneData) -> T>(f: F) -> T {
-        thread_local! {
-            static HYGIENE_DATA: RefCell<HygieneData> = RefCell::new(HygieneData::new());
-        }
-        HYGIENE_DATA.with(|data| f(&mut *data.borrow_mut()))
-    }
-}
-
-pub fn reset_hygiene_data() {
-    HygieneData::with(|data| *data = HygieneData::new())
-}
-
-impl SyntaxContext {
-    pub const fn empty() -> Self {
-        SyntaxContext(0)
-    }
-
-    pub fn data(self) -> SyntaxContextData {
-        HygieneData::with(|data| data.syntax_contexts[self.0 as usize])
-    }
-
-    /// Extend a syntax context with a given mark
-    pub fn apply_mark(self, mark: Mark) -> SyntaxContext {
-        // Applying the same mark twice is a no-op
-        let ctxt_data = self.data();
-        if mark == ctxt_data.outer_mark {
-            return ctxt_data.prev_ctxt;
-        }
-
-        HygieneData::with(|data| {
-            let syntax_contexts = &mut data.syntax_contexts;
-            *data.markings.entry((self, mark)).or_insert_with(|| {
-                syntax_contexts.push(SyntaxContextData {
-                    outer_mark: mark,
-                    prev_ctxt: self,
-                });
-                SyntaxContext(syntax_contexts.len() as u32 - 1)
-            })
-        })
-    }
-}
-
-impl fmt::Debug for SyntaxContext {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "#{}", self.0)
-    }
-}
diff --git a/src/libsyntax/ext/placeholders.rs b/src/libsyntax/ext/placeholders.rs
index f60b1d17a5e..4fb138d506a 100644
--- a/src/libsyntax/ext/placeholders.rs
+++ b/src/libsyntax/ext/placeholders.rs
@@ -8,7 +8,7 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use ast;
+use ast::{self, NodeId};
 use codemap::{DUMMY_SP, dummy_spanned};
 use ext::base::ExtCtxt;
 use ext::expand::{Expansion, ExpansionKind};
@@ -88,7 +88,7 @@ impl<'a, 'b> PlaceholderExpander<'a, 'b> {
         let mut expansion = expansion.fold_with(self);
         if let Expansion::Items(mut items) = expansion {
             for derive in derives {
-                match self.remove(derive.as_placeholder_id()) {
+                match self.remove(NodeId::placeholder_from_mark(derive)) {
                     Expansion::Items(derived_items) => items.extend(derived_items),
                     _ => unreachable!(),
                 }
@@ -106,8 +106,8 @@ impl<'a, 'b> PlaceholderExpander<'a, 'b> {
 impl<'a, 'b> Folder for PlaceholderExpander<'a, 'b> {
     fn fold_item(&mut self, item: P<ast::Item>) -> SmallVector<P<ast::Item>> {
         match item.node {
-            ast::ItemKind::Mac(ref mac) if !mac.node.path.segments.is_empty() => {}
             ast::ItemKind::Mac(_) => return self.remove(item.id).make_items(),
+            ast::ItemKind::MacroDef(_) => return SmallVector::one(item),
             _ => {}
         }
 
@@ -178,17 +178,9 @@ impl<'a, 'b> Folder for PlaceholderExpander<'a, 'b> {
             block.stmts = block.stmts.move_flat_map(|mut stmt| {
                 remaining_stmts -= 1;
 
-                match stmt.node {
-                    // Avoid wasting a node id on a trailing expression statement,
-                    // which shares a HIR node with the expression itself.
-                    ast::StmtKind::Expr(ref expr) if remaining_stmts == 0 => stmt.id = expr.id,
-
-                    _ if self.monotonic => {
-                        assert_eq!(stmt.id, ast::DUMMY_NODE_ID);
-                        stmt.id = self.cx.resolver.next_node_id();
-                    }
-
-                    _ => {}
+                if self.monotonic {
+                    assert_eq!(stmt.id, ast::DUMMY_NODE_ID);
+                    stmt.id = self.cx.resolver.next_node_id();
                 }
 
                 Some(stmt)
diff --git a/src/libsyntax/ext/source_util.rs b/src/libsyntax/ext/source_util.rs
index 39b92c7d007..0103d6ea959 100644
--- a/src/libsyntax/ext/source_util.rs
+++ b/src/libsyntax/ext/source_util.rs
@@ -185,7 +185,7 @@ pub fn expand_include_bytes(cx: &mut ExtCtxt, sp: Span, tts: &[tokenstream::Toke
 fn res_rel_file(cx: &mut ExtCtxt, sp: syntax_pos::Span, arg: &Path) -> PathBuf {
     // NB: relative paths are resolved relative to the compilation unit
     if !arg.is_absolute() {
-        let callsite = cx.codemap().source_callsite(sp);
+        let callsite = sp.source_callsite();
         let mut cu = PathBuf::from(&cx.codemap().span_to_filename(callsite));
         cu.pop();
         cu.push(arg);
diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs
index ed17f0f956c..6cd1fea2e75 100644
--- a/src/libsyntax/ext/tt/macro_parser.rs
+++ b/src/libsyntax/ext/tt/macro_parser.rs
@@ -79,7 +79,7 @@ pub use self::ParseResult::*;
 use self::TokenTreeOrTokenTreeVec::*;
 
 use ast::Ident;
-use syntax_pos::{self, BytePos, mk_sp, Span};
+use syntax_pos::{self, BytePos, Span};
 use codemap::Spanned;
 use errors::FatalError;
 use ext::tt::quoted::{self, TokenTree};
@@ -285,7 +285,7 @@ fn inner_parse_loop(sess: &ParseSess,
                     eof_eis: &mut SmallVector<Box<MatcherPos>>,
                     bb_eis: &mut SmallVector<Box<MatcherPos>>,
                     token: &Token,
-                    span: &syntax_pos::Span)
+                    span: syntax_pos::Span)
                     -> ParseResult<()> {
     while let Some(mut ei) = cur_eis.pop() {
         // When unzipped trees end, remove them
@@ -323,8 +323,7 @@ fn inner_parse_loop(sess: &ParseSess,
                     for idx in ei.match_lo..ei.match_hi {
                         let sub = ei.matches[idx].clone();
                         new_pos.matches[idx]
-                            .push(Rc::new(MatchedSeq(sub, mk_sp(ei.sp_lo,
-                                                                span.hi))));
+                            .push(Rc::new(MatchedSeq(sub, Span { lo: ei.sp_lo, ..span })));
                     }
 
                     new_pos.match_cur = ei.match_hi;
@@ -426,7 +425,7 @@ pub fn parse(sess: &ParseSess, tts: TokenStream, ms: &[TokenTree], directory: Op
         assert!(next_eis.is_empty());
 
         match inner_parse_loop(sess, &mut cur_eis, &mut next_eis, &mut eof_eis, &mut bb_eis,
-                               &parser.token, &parser.span) {
+                               &parser.token, parser.span) {
             Success(_) => {},
             Failure(sp, tok) => return Failure(sp, tok),
             Error(sp, msg) => return Error(sp, msg),
@@ -493,7 +492,7 @@ fn parse_nt<'a>(p: &mut Parser<'a>, sp: Span, name: &str) -> Nonterminal {
         _ => {}
     }
     // check at the beginning and the parser checks after each bump
-    p.check_unknown_macro_variable();
+    p.process_potential_macro_variable();
     match name {
         "item" => match panictry!(p.parse_item()) {
             Some(i) => token::NtItem(i),
diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs
index 021c5398a42..93348c8f083 100644
--- a/src/libsyntax/ext/tt/macro_rules.rs
+++ b/src/libsyntax/ext/tt/macro_rules.rs
@@ -119,9 +119,9 @@ fn generic_extension<'cx>(cx: &'cx ExtCtxt,
                 };
                 let mut p = Parser::new(cx.parse_sess(), tts, Some(directory), false);
                 p.root_module_name = cx.current_expansion.module.mod_path.last()
-                    .map(|id| (*id.name.as_str()).to_owned());
+                    .map(|id| id.name.as_str().to_string());
 
-                p.check_unknown_macro_variable();
+                p.process_potential_macro_variable();
                 // Let the context choose how to interpret the result.
                 // Weird, but useful for X-macros.
                 return Box::new(ParserAnyMacro {
diff --git a/src/libsyntax/ext/tt/quoted.rs b/src/libsyntax/ext/tt/quoted.rs
index d56859d805c..d216effbd45 100644
--- a/src/libsyntax/ext/tt/quoted.rs
+++ b/src/libsyntax/ext/tt/quoted.rs
@@ -34,17 +34,19 @@ impl Delimited {
     }
 
     pub fn open_tt(&self, span: Span) -> TokenTree {
-        let open_span = match span {
-            DUMMY_SP => DUMMY_SP,
-            _ => Span { hi: span.lo + BytePos(self.delim.len() as u32), ..span },
+        let open_span = if span == DUMMY_SP {
+            DUMMY_SP
+        } else {
+            Span { hi: span.lo + BytePos(self.delim.len() as u32), ..span }
         };
         TokenTree::Token(open_span, self.open_token())
     }
 
     pub fn close_tt(&self, span: Span) -> TokenTree {
-        let close_span = match span {
-            DUMMY_SP => DUMMY_SP,
-            _ => Span { lo: span.hi - BytePos(self.delim.len() as u32), ..span },
+        let close_span = if span == DUMMY_SP {
+            DUMMY_SP
+        } else {
+            Span { lo: span.hi - BytePos(self.delim.len() as u32), ..span }
         };
         TokenTree::Token(close_span, self.close_token())
     }
@@ -134,11 +136,14 @@ pub fn parse(input: tokenstream::TokenStream, expect_matchers: bool, sess: &Pars
             TokenTree::Token(start_sp, token::SubstNt(ident)) if expect_matchers => {
                 let span = match trees.next() {
                     Some(tokenstream::TokenTree::Token(span, token::Colon)) => match trees.next() {
-                        Some(tokenstream::TokenTree::Token(end_sp, token::Ident(kind))) => {
-                            let span = Span { lo: start_sp.lo, ..end_sp };
-                            result.push(TokenTree::MetaVarDecl(span, ident, kind));
-                            continue
-                        }
+                        Some(tokenstream::TokenTree::Token(end_sp, ref tok)) => match tok.ident() {
+                            Some(kind) => {
+                                let span = Span { lo: start_sp.lo, ..end_sp };
+                                result.push(TokenTree::MetaVarDecl(span, ident, kind));
+                                continue
+                            }
+                            _ => end_sp,
+                        },
                         tree @ _ => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span),
                     },
                     tree @ _ => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(start_sp),
diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/tt/transcribe.rs
index 24004492be2..947089b0b9a 100644
--- a/src/libsyntax/ext/tt/transcribe.rs
+++ b/src/libsyntax/ext/tt/transcribe.rs
@@ -12,7 +12,7 @@ use ast::Ident;
 use errors::Handler;
 use ext::tt::macro_parser::{NamedMatch, MatchedSeq, MatchedNonterminal};
 use ext::tt::quoted;
-use parse::token::{self, SubstNt, Token, NtIdent, NtTT};
+use parse::token::{self, SubstNt, Token, NtTT};
 use syntax_pos::{Span, DUMMY_SP};
 use tokenstream::{TokenStream, TokenTree, Delimited};
 use util::small_vector::SmallVector;
@@ -154,13 +154,6 @@ pub fn transcribe(sp_diag: &Handler,
                     None => result.push(TokenTree::Token(sp, SubstNt(ident)).into()),
                     Some(cur_matched) => if let MatchedNonterminal(ref nt) = *cur_matched {
                         match **nt {
-                            // sidestep the interpolation tricks for ident because
-                            // (a) idents can be in lots of places, so it'd be a pain
-                            // (b) we actually can, since it's a token.
-                            NtIdent(ref sn) => {
-                                let token = TokenTree::Token(sn.span, token::Ident(sn.node));
-                                result.push(token.into());
-                            }
                             NtTT(ref tt) => result.push(tt.clone().into()),
                             _ => {
                                 let token = TokenTree::Token(sp, token::Interpolated(nt.clone()));