auto merge of #9026 : jbclements/rust/let-var-hygiene, r=jbclements

This is a rebase of my approved pull request from ... the end of June? It introduces hygiene for let-bound variables.

5 files changed, 808 insertions, 215 deletions

diff --git a/src/libsyntax/ext/base.rs b/src/libsyntax/ext/base.rs
index 50683358f87..3d5d62aeadf 100644
--- a/src/libsyntax/ext/base.rs
+++ b/src/libsyntax/ext/base.rs
@@ -22,8 +22,7 @@ use std::hashmap::HashMap;
 
 // new-style macro! tt code:
 //
-//    SyntaxExpanderTT, SyntaxExpanderTTItem, MacResult,
-//    NormalTT, IdentTT
+//    MacResult, NormalTT, IdentTT
 //
 // also note that ast::mac used to have a bunch of extraneous cases and
 // is now probably a redundant AST node, can be merged with
@@ -34,33 +33,46 @@ pub struct MacroDef {
     ext: SyntaxExtension
 }
 
-pub type ItemDecorator = @fn(@ExtCtxt,
+// No context arg for an Item Decorator macro, simply because
+// adding it would require adding a ctxt field to all items.
+// we could do this if it turns out to be useful.
+
+pub type ItemDecoratorFun = @fn(@ExtCtxt,
                              Span,
                              @ast::MetaItem,
                              ~[@ast::item])
                           -> ~[@ast::item];
 
-pub struct SyntaxExpanderTT {
-    expander: SyntaxExpanderTTFun,
-    span: Option<Span>
-}
-
 pub type SyntaxExpanderTTFun = @fn(@ExtCtxt,
                                    Span,
-                                   &[ast::token_tree])
+                                   &[ast::token_tree],
+                                   ast::SyntaxContext)
                                 -> MacResult;
 
-pub struct SyntaxExpanderTTItem {
-    expander: SyntaxExpanderTTItemFun,
-    span: Option<Span>
-}
-
 pub type SyntaxExpanderTTItemFun = @fn(@ExtCtxt,
                                        Span,
                                        ast::Ident,
+                                       ~[ast::token_tree],
+                                       ast::SyntaxContext)
+                                    -> MacResult;
+
+// oog... in order to make the presentation of builtin_normal_tt_no_ctxt
+// and builtin_ident_tt_no_ctxt palatable, we need one-off types for
+// functions that don't consume a ctxt:
+
+pub type SyntaxExpanderTTFunNoCtxt = @fn(@ExtCtxt,
+                                   Span,
+                                   &[ast::token_tree])
+                                -> MacResult;
+
+pub type SyntaxExpanderTTItemFunNoCtxt = @fn(@ExtCtxt,
+                                       Span,
+                                       ast::Ident,
                                        ~[ast::token_tree])
                                     -> MacResult;
 
+
+
 pub enum MacResult {
     MRExpr(@ast::Expr),
     MRItem(@ast::item),
@@ -73,10 +85,10 @@ pub enum MacResult {
 pub enum SyntaxExtension {
 
     // #[auto_encode] and such
-    ItemDecorator(ItemDecorator),
+    ItemDecorator(ItemDecoratorFun),
 
     // Token-tree expanders
-    NormalTT(SyntaxExpanderTT),
+    NormalTT(SyntaxExpanderTTFun, Option<Span>),
 
     // An IdentTT is a macro that has an
     // identifier in between the name of the
@@ -86,9 +98,10 @@ pub enum SyntaxExtension {
 
     // perhaps macro_rules! will lose its odd special identifier argument,
     // and this can go away also
-    IdentTT(SyntaxExpanderTTItem),
+    IdentTT(SyntaxExpanderTTItemFun, Option<Span>),
 }
 
+
 // The SyntaxEnv is the environment that's threaded through the expansion
 // of macros. It contains bindings for macros, and also a special binding
 // for " block" (not a legal identifier) that maps to a BlockInfo
@@ -120,12 +133,16 @@ type RenameList = ~[(ast::Ident,Name)];
 // AST nodes into full ASTs
 pub fn syntax_expander_table() -> SyntaxEnv {
     // utility function to simplify creating NormalTT syntax extensions
-    fn builtin_normal_tt(f: SyntaxExpanderTTFun) -> @Transformer {
-        @SE(NormalTT(SyntaxExpanderTT{expander: f, span: None}))
+    // that ignore their contexts
+    fn builtin_normal_tt_no_ctxt(f: SyntaxExpanderTTFunNoCtxt) -> @Transformer {
+        let wrapped_expander : SyntaxExpanderTTFun = |a,b,c,_d|{f(a,b,c)};
+        @SE(NormalTT(wrapped_expander, None))
     }
     // utility function to simplify creating IdentTT syntax extensions
-    fn builtin_item_tt(f: SyntaxExpanderTTItemFun) -> @Transformer {
-        @SE(IdentTT(SyntaxExpanderTTItem{expander: f, span: None}))
+    // that ignore their contexts
+    fn builtin_item_tt_no_ctxt(f: SyntaxExpanderTTItemFunNoCtxt) -> @Transformer {
+        let wrapped_expander : SyntaxExpanderTTItemFun = |a,b,c,d,_e|{f(a,b,c,d)};
+        @SE(IdentTT(wrapped_expander, None))
     }
     let mut syntax_expanders = HashMap::new();
     // NB identifier starts with space, and can't conflict with legal idents
@@ -135,16 +152,15 @@ pub fn syntax_expander_table() -> SyntaxEnv {
                                 pending_renames : @mut ~[]
                             }));
     syntax_expanders.insert(intern(&"macro_rules"),
-                            builtin_item_tt(
-                                ext::tt::macro_rules::add_new_extension));
+                            @SE(IdentTT(ext::tt::macro_rules::add_new_extension, None)));
     syntax_expanders.insert(intern(&"fmt"),
-                            builtin_normal_tt(ext::fmt::expand_syntax_ext));
+                            builtin_normal_tt_no_ctxt(ext::fmt::expand_syntax_ext));
     syntax_expanders.insert(intern(&"format"),
-                            builtin_normal_tt(ext::ifmt::expand_format));
+                            builtin_normal_tt_no_ctxt(ext::ifmt::expand_format));
     syntax_expanders.insert(intern(&"write"),
-                            builtin_normal_tt(ext::ifmt::expand_write));
+                            builtin_normal_tt_no_ctxt(ext::ifmt::expand_write));
     syntax_expanders.insert(intern(&"writeln"),
-                            builtin_normal_tt(ext::ifmt::expand_writeln));
+                            builtin_normal_tt_no_ctxt(ext::ifmt::expand_writeln));
     syntax_expanders.insert(
         intern(&"auto_encode"),
         @SE(ItemDecorator(ext::auto_encode::expand_auto_encode)));
@@ -152,16 +168,16 @@ pub fn syntax_expander_table() -> SyntaxEnv {
         intern(&"auto_decode"),
         @SE(ItemDecorator(ext::auto_encode::expand_auto_decode)));
     syntax_expanders.insert(intern(&"env"),
-                            builtin_normal_tt(ext::env::expand_env));
+                            builtin_normal_tt_no_ctxt(ext::env::expand_env));
     syntax_expanders.insert(intern(&"option_env"),
-                            builtin_normal_tt(ext::env::expand_option_env));
+                            builtin_normal_tt_no_ctxt(ext::env::expand_option_env));
     syntax_expanders.insert(intern("bytes"),
-                            builtin_normal_tt(ext::bytes::expand_syntax_ext));
+                            builtin_normal_tt_no_ctxt(ext::bytes::expand_syntax_ext));
     syntax_expanders.insert(intern("concat_idents"),
-                            builtin_normal_tt(
+                            builtin_normal_tt_no_ctxt(
                                 ext::concat_idents::expand_syntax_ext));
     syntax_expanders.insert(intern(&"log_syntax"),
-                            builtin_normal_tt(
+                            builtin_normal_tt_no_ctxt(
                                 ext::log_syntax::expand_syntax_ext));
     syntax_expanders.insert(intern(&"deriving"),
                             @SE(ItemDecorator(
@@ -169,49 +185,50 @@ pub fn syntax_expander_table() -> SyntaxEnv {
 
     // Quasi-quoting expanders
     syntax_expanders.insert(intern(&"quote_tokens"),
-                       builtin_normal_tt(ext::quote::expand_quote_tokens));
+                            builtin_normal_tt_no_ctxt(
+                                ext::quote::expand_quote_tokens));
     syntax_expanders.insert(intern(&"quote_expr"),
-                       builtin_normal_tt(ext::quote::expand_quote_expr));
+                            builtin_normal_tt_no_ctxt(ext::quote::expand_quote_expr));
     syntax_expanders.insert(intern(&"quote_ty"),
-                       builtin_normal_tt(ext::quote::expand_quote_ty));
+                            builtin_normal_tt_no_ctxt(ext::quote::expand_quote_ty));
     syntax_expanders.insert(intern(&"quote_item"),
-                       builtin_normal_tt(ext::quote::expand_quote_item));
+                            builtin_normal_tt_no_ctxt(ext::quote::expand_quote_item));
     syntax_expanders.insert(intern(&"quote_pat"),
-                       builtin_normal_tt(ext::quote::expand_quote_pat));
+                            builtin_normal_tt_no_ctxt(ext::quote::expand_quote_pat));
     syntax_expanders.insert(intern(&"quote_stmt"),
-                       builtin_normal_tt(ext::quote::expand_quote_stmt));
+                            builtin_normal_tt_no_ctxt(ext::quote::expand_quote_stmt));
 
     syntax_expanders.insert(intern(&"line"),
-                            builtin_normal_tt(
+                            builtin_normal_tt_no_ctxt(
                                 ext::source_util::expand_line));
     syntax_expanders.insert(intern(&"col"),
-                            builtin_normal_tt(
+                            builtin_normal_tt_no_ctxt(
                                 ext::source_util::expand_col));
     syntax_expanders.insert(intern(&"file"),
-                            builtin_normal_tt(
+                            builtin_normal_tt_no_ctxt(
                                 ext::source_util::expand_file));
     syntax_expanders.insert(intern(&"stringify"),
-                            builtin_normal_tt(
+                            builtin_normal_tt_no_ctxt(
                                 ext::source_util::expand_stringify));
     syntax_expanders.insert(intern(&"include"),
-                            builtin_normal_tt(
+                            builtin_normal_tt_no_ctxt(
                                 ext::source_util::expand_include));
     syntax_expanders.insert(intern(&"include_str"),
-                            builtin_normal_tt(
+                            builtin_normal_tt_no_ctxt(
                                 ext::source_util::expand_include_str));
     syntax_expanders.insert(intern(&"include_bin"),
-                            builtin_normal_tt(
+                            builtin_normal_tt_no_ctxt(
                                 ext::source_util::expand_include_bin));
     syntax_expanders.insert(intern(&"module_path"),
-                            builtin_normal_tt(
+                            builtin_normal_tt_no_ctxt(
                                 ext::source_util::expand_mod));
     syntax_expanders.insert(intern(&"asm"),
-                            builtin_normal_tt(ext::asm::expand_asm));
+                            builtin_normal_tt_no_ctxt(ext::asm::expand_asm));
     syntax_expanders.insert(intern(&"cfg"),
-                            builtin_normal_tt(ext::cfg::expand_cfg));
+                            builtin_normal_tt_no_ctxt(ext::cfg::expand_cfg));
     syntax_expanders.insert(
         intern(&"trace_macros"),
-        builtin_normal_tt(ext::trace_macros::expand_trace_macros));
+        builtin_normal_tt_no_ctxt(ext::trace_macros::expand_trace_macros));
     MapChain::new(~syntax_expanders)
 }
 
diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index ea277a8a625..cde418ca991 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -8,11 +8,12 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use ast::{Block, Crate, NodeId, Expr_, ExprMac, Ident, mac_invoc_tt};
-use ast::{item_mac, Stmt_, StmtMac, StmtExpr, StmtSemi};
-use ast::{ILLEGAL_CTXT};
+use ast::{Block, Crate, NodeId, DeclLocal, Expr_, ExprMac, SyntaxContext};
+use ast::{Local, Ident, mac_invoc_tt};
+use ast::{item_mac, Mrk, Stmt_, StmtDecl, StmtMac, StmtExpr, StmtSemi};
+use ast::{token_tree};
 use ast;
-use ast_util::{new_rename, new_mark, resolve};
+use ast_util::{mtwt_outer_mark, new_rename, new_mark};
 use attr;
 use attr::AttrMetaMethods;
 use codemap;
@@ -23,7 +24,7 @@ use opt_vec;
 use parse;
 use parse::{parse_item_from_source_str};
 use parse::token;
-use parse::token::{ident_to_str, intern};
+use parse::token::{fresh_mark, fresh_name, ident_to_str, intern};
 use visit;
 use visit::Visitor;
 
@@ -41,8 +42,13 @@ pub fn expand_expr(extsbox: @mut SyntaxEnv,
         // entry-point for all syntax extensions.
         ExprMac(ref mac) => {
             match (*mac).node {
+                // it would almost certainly be cleaner to pass the whole
+                // macro invocation in, rather than pulling it apart and
+                // marking the tts and the ctxt separately. This also goes
+                // for the other three macro invocation chunks of code
+                // in this file.
                 // Token-tree macros:
-                mac_invoc_tt(ref pth, ref tts) => {
+                mac_invoc_tt(ref pth, ref tts, ctxt) => {
                     if (pth.segments.len() > 1u) {
                         cx.span_fatal(
                             pth.span,
@@ -58,35 +64,38 @@ pub fn expand_expr(extsbox: @mut SyntaxEnv,
                                 pth.span,
                                 fmt!("macro undefined: '%s'", extnamestr))
                         }
-                        Some(@SE(NormalTT(SyntaxExpanderTT{
-                            expander: exp,
-                            span: exp_sp
-                        }))) => {
+                        Some(@SE(NormalTT(expandfun, exp_span))) => {
                             cx.bt_push(ExpnInfo {
                                 call_site: s,
                                 callee: NameAndSpan {
                                     name: extnamestr,
-                                    span: exp_sp,
+                                    span: exp_span,
                                 },
                             });
-
-                            let expanded = match exp(cx, mac.span, *tts) {
-                                MRExpr(e) => e,
-                                MRAny(expr_maker,_,_) => expr_maker(),
-                                _ => {
-                                    cx.span_fatal(
-                                        pth.span,
-                                        fmt!(
-                                            "non-expr macro in expr pos: %s",
-                                            extnamestr
+                            let fm = fresh_mark();
+                            // mark before:
+                            let marked_before = mark_tts(*tts,fm);
+                            let marked_ctxt = new_mark(fm, ctxt);
+                            let expanded =
+                                match expandfun(cx, mac.span, marked_before, marked_ctxt) {
+                                    MRExpr(e) => e,
+                                    MRAny(expr_maker,_,_) => expr_maker(),
+                                    _ => {
+                                        cx.span_fatal(
+                                            pth.span,
+                                            fmt!(
+                                                "non-expr macro in expr pos: %s",
+                                                extnamestr
+                                            )
                                         )
-                                    )
-                                }
-                            };
+                                    }
+                                };
+                            // mark after:
+                            let marked_after = mark_expr(expanded,fm);
 
                             //keep going, outside-in
                             let fully_expanded =
-                                fld.fold_expr(expanded).node.clone();
+                                fld.fold_expr(marked_after).node.clone();
                             cx.bt_pop();
 
                             (fully_expanded, s)
@@ -309,7 +318,6 @@ pub fn expand_mod_items(extsbox: @mut SyntaxEnv,
     ast::_mod { items: new_items, ..module_ }
 }
 
-
 // eval $e with a new exts frame:
 macro_rules! with_exts_frame (
     ($extsboxexpr:expr,$macros_escape:expr,$e:expr) =>
@@ -333,25 +341,16 @@ pub fn expand_item(extsbox: @mut SyntaxEnv,
                    fld: @ast_fold,
                    orig: @fn(@ast::item, @ast_fold) -> Option<@ast::item>)
                 -> Option<@ast::item> {
-    // need to do expansion first... it might turn out to be a module.
-    let maybe_it = match it.node {
-      ast::item_mac(*) => expand_item_mac(extsbox, cx, it, fld),
-      _ => Some(it)
-    };
-    match maybe_it {
-      Some(it) => {
-          match it.node {
-              ast::item_mod(_) | ast::item_foreign_mod(_) => {
-                  cx.mod_push(it.ident);
-                  let macro_escape = contains_macro_escape(it.attrs);
-                  let result = with_exts_frame!(extsbox,macro_escape,orig(it,fld));
-                  cx.mod_pop();
-                  result
-              }
-              _ => orig(it,fld)
-          }
-      }
-      None => None
+    match it.node {
+        ast::item_mac(*) => expand_item_mac(extsbox, cx, it, fld),
+        ast::item_mod(_) | ast::item_foreign_mod(_) => {
+            cx.mod_push(it.ident);
+            let macro_escape = contains_macro_escape(it.attrs);
+            let result = with_exts_frame!(extsbox,macro_escape,orig(it,fld));
+            cx.mod_pop();
+            result
+        },
+        _ => orig(it,fld)
     }
 }
 
@@ -366,21 +365,22 @@ pub fn expand_item_mac(extsbox: @mut SyntaxEnv,
                        cx: @ExtCtxt, it: @ast::item,
                        fld: @ast_fold)
                     -> Option<@ast::item> {
-    let (pth, tts) = match it.node {
-        item_mac(codemap::Spanned { node: mac_invoc_tt(ref pth, ref tts), _}) => {
-            (pth, (*tts).clone())
+    let (pth, tts, ctxt) = match it.node {
+        item_mac(codemap::Spanned { node: mac_invoc_tt(ref pth, ref tts, ctxt), _}) => {
+            (pth, (*tts).clone(), ctxt)
         }
         _ => cx.span_bug(it.span, "invalid item macro invocation")
     };
 
     let extname = &pth.segments[0].identifier;
     let extnamestr = ident_to_str(extname);
+    let fm = fresh_mark();
     let expanded = match (*extsbox).find(&extname.name) {
         None => cx.span_fatal(pth.span,
                               fmt!("macro undefined: '%s!'", extnamestr)),
 
-        Some(@SE(NormalTT(ref expand))) => {
-            if it.ident != parse::token::special_idents::invalid {
+        Some(@SE(NormalTT(expander, span))) => {
+            if it.ident.name != parse::token::special_idents::invalid.name {
                 cx.span_fatal(pth.span,
                               fmt!("macro %s! expects no ident argument, \
                                     given '%s'", extnamestr,
@@ -390,13 +390,16 @@ pub fn expand_item_mac(extsbox: @mut SyntaxEnv,
                 call_site: it.span,
                 callee: NameAndSpan {
                     name: extnamestr,
-                    span: expand.span
+                    span: span
                 }
             });
-            ((*expand).expander)(cx, it.span, tts)
+            // mark before expansion:
+            let marked_before = mark_tts(tts,fm);
+            let marked_ctxt = new_mark(fm,ctxt);
+            expander(cx, it.span, marked_before, marked_ctxt)
         }
-        Some(@SE(IdentTT(ref expand))) => {
-            if it.ident == parse::token::special_idents::invalid {
+        Some(@SE(IdentTT(expander, span))) => {
+            if it.ident.name == parse::token::special_idents::invalid.name {
                 cx.span_fatal(pth.span,
                               fmt!("macro %s! expects an ident argument",
                                    extnamestr));
@@ -405,21 +408,27 @@ pub fn expand_item_mac(extsbox: @mut SyntaxEnv,
                 call_site: it.span,
                 callee: NameAndSpan {
                     name: extnamestr,
-                    span: expand.span
+                    span: span
                 }
             });
-            ((*expand).expander)(cx, it.span, it.ident, tts)
+            // mark before expansion:
+            let marked_tts = mark_tts(tts,fm);
+            let marked_ctxt = new_mark(fm,ctxt);
+            expander(cx, it.span, it.ident, marked_tts, marked_ctxt)
         }
         _ => cx.span_fatal(
             it.span, fmt!("%s! is not legal in item position", extnamestr))
     };
 
     let maybe_it = match expanded {
-        MRItem(it) => fld.fold_item(it),
+        MRItem(it) => mark_item(it,fm).chain(|i| {fld.fold_item(i)}),
         MRExpr(_) => cx.span_fatal(pth.span,
                                    fmt!("expr macro in item position: %s", extnamestr)),
-        MRAny(_, item_maker, _) => item_maker().chain(|i| {fld.fold_item(i)}),
+        MRAny(_, item_maker, _) => item_maker().chain(|i| {mark_item(i,fm)})
+                                      .chain(|i| {fld.fold_item(i)}),
         MRDef(ref mdef) => {
+            // yikes... no idea how to apply the mark to this. I'm afraid
+            // we're going to have to wait-and-see on this one.
             insert_macro(*extsbox,intern(mdef.name), @SE((*mdef).ext));
             None
         }
@@ -454,15 +463,17 @@ pub fn expand_stmt(extsbox: @mut SyntaxEnv,
                    orig: @fn(&Stmt_, Span, @ast_fold)
                              -> (Option<Stmt_>, Span))
                 -> (Option<Stmt_>, Span) {
-    let (mac, pth, tts, semi) = match *s {
+    // why the copying here and not in expand_expr?
+    // looks like classic changed-in-only-one-place
+    let (mac, pth, tts, semi, ctxt) = match *s {
         StmtMac(ref mac, semi) => {
             match mac.node {
-                mac_invoc_tt(ref pth, ref tts) => {
-                    ((*mac).clone(), pth, (*tts).clone(), semi)
+                mac_invoc_tt(ref pth, ref tts, ctxt) => {
+                    ((*mac).clone(), pth, (*tts).clone(), semi, ctxt)
                 }
             }
         }
-        _ => return orig(s, sp, fld)
+        _ => return expand_non_macro_stmt(*extsbox,s,sp,fld,orig)
     };
     if (pth.segments.len() > 1u) {
         cx.span_fatal(
@@ -476,13 +487,16 @@ pub fn expand_stmt(extsbox: @mut SyntaxEnv,
         None =>
             cx.span_fatal(pth.span, fmt!("macro undefined: '%s'", extnamestr)),
 
-        Some(@SE(NormalTT(
-            SyntaxExpanderTT{expander: exp, span: exp_sp}))) => {
+        Some(@SE(NormalTT(expandfun, exp_span))) => {
             cx.bt_push(ExpnInfo {
                 call_site: sp,
-                callee: NameAndSpan { name: extnamestr, span: exp_sp }
+                callee: NameAndSpan { name: extnamestr, span: exp_span }
             });
-            let expanded = match exp(cx, mac.span, tts) {
+            let fm = fresh_mark();
+            // mark before expansion:
+            let marked_tts = mark_tts(tts,fm);
+            let marked_ctxt = new_mark(fm,ctxt);
+            let expanded = match expandfun(cx, mac.span, marked_tts, marked_ctxt) {
                 MRExpr(e) =>
                     @codemap::Spanned { node: StmtExpr(e, cx.next_id()),
                                     span: e.span},
@@ -491,9 +505,10 @@ pub fn expand_stmt(extsbox: @mut SyntaxEnv,
                     pth.span,
                     fmt!("non-stmt macro in stmt pos: %s", extnamestr))
             };
+            let marked_after = mark_stmt(expanded,fm);
 
             //keep going, outside-in
-            let fully_expanded = match fld.fold_stmt(expanded) {
+            let fully_expanded = match fld.fold_stmt(marked_after) {
                 Some(stmt) => {
                     let fully_expanded = &stmt.node;
                     cx.bt_pop();
@@ -521,6 +536,73 @@ pub fn expand_stmt(extsbox: @mut SyntaxEnv,
 
 }
 
+// expand a non-macro stmt. this is essentially the fallthrough for
+// expand_stmt, above.
+fn expand_non_macro_stmt (exts: SyntaxEnv,
+                          s: &Stmt_,
+                          sp: Span,
+                          fld: @ast_fold,
+                          orig: @fn(&Stmt_, Span, @ast_fold) -> (Option<Stmt_>, Span))
+    -> (Option<Stmt_>,Span) {
+    // is it a let?
+    match *s {
+        StmtDecl(@Spanned{node: DeclLocal(ref local), span: stmt_span}, node_id) => {
+            let block_info = get_block_info(exts);
+            let pending_renames = block_info.pending_renames;
+
+            // take it apart:
+            let @Local{is_mutbl:is_mutbl,
+                       ty:_,
+                       pat:pat,
+                       init:init,
+                       id:id,
+                       span:span
+                      } = *local;
+            // types can't be copied automatically because of the owned ptr in ty_tup...
+            let ty = local.ty.clone();
+            // expand the pat (it might contain exprs... #:(o)>
+            let expanded_pat = fld.fold_pat(pat);
+            // find the pat_idents in the pattern:
+            // oh dear heaven... this is going to include the enum names, as well....
+            // ... but that should be okay, as long as the new names are gensyms
+            // for the old ones.
+            let idents = @mut ~[];
+            let name_finder = new_name_finder(idents);
+            name_finder.visit_pat(expanded_pat,());
+            // generate fresh names, push them to a new pending list
+            let new_pending_renames = @mut ~[];
+            for ident in idents.iter() {
+                let new_name = fresh_name(ident);
+                new_pending_renames.push((*ident,new_name));
+            }
+            let rename_fld = renames_to_fold(new_pending_renames);
+            // rewrite the pattern using the new names (the old ones
+            // have already been applied):
+            let rewritten_pat = rename_fld.fold_pat(expanded_pat);
+            // add them to the existing pending renames:
+            for pr in new_pending_renames.iter() {pending_renames.push(*pr)}
+            // also, don't forget to expand the init:
+            let new_init_opt = init.map(|e| fld.fold_expr(*e));
+            let rewritten_local =
+                @Local{is_mutbl:is_mutbl,
+                       ty:ty,
+                       pat:rewritten_pat,
+                       init:new_init_opt,
+                       id:id,
+                       span:span};
+            (Some(StmtDecl(@Spanned{node:DeclLocal(rewritten_local),
+                                     span: stmt_span},node_id)),
+             sp)
+        },
+        _ => {
+            orig(s, sp, fld)
+        }
+    }
+}
+
+// a visitor that extracts the pat_ident paths
+// from a given thingy and puts them in a mutable
+// array (passed in to the traversal)
 #[deriving(Clone)]
 struct NewNameFinderContext {
     ident_accumulator: @mut ~[ast::Ident],
@@ -664,8 +746,133 @@ impl Visitor<()> for NewNameFinderContext {
     }
 }
 
+// a visitor that extracts the paths
+// from a given thingy and puts them in a mutable
+// array (passed in to the traversal)
+#[deriving(Clone)]
+struct NewPathExprFinderContext {
+    path_accumulator: @mut ~[ast::Path],
+}
+
+// XXX : YIKES a lot of boilerplate again....
+impl Visitor<()> for NewPathExprFinderContext {
+
+    fn visit_expr(&mut self, expr: @ast::Expr, _: ()) {
+        match *expr {
+            ast::Expr{id:_,span:_,node:ast::ExprPath(ref p)} => {
+                self.path_accumulator.push(p.clone());
+                // not calling visit_path, should be fine.
+            }
+            _ => visit::walk_expr(self,expr,())
+        }
+    }
+
+
+    // XXX: Methods below can become default methods.
+
+    fn visit_pat(&mut self, pattern: @ast::Pat, _: ()) {
+        visit::walk_pat(self,pattern,())
+    }
+
+    fn visit_mod(&mut self, module: &ast::_mod, _: Span, _: NodeId, _: ()) {
+        visit::walk_mod(self, module, ())
+    }
+
+    fn visit_view_item(&mut self, view_item: &ast::view_item, _: ()) {
+        visit::walk_view_item(self, view_item, ())
+    }
+
+    fn visit_item(&mut self, item: @ast::item, _: ()) {
+        visit::walk_item(self, item, ())
+    }
+
+    fn visit_foreign_item(&mut self,
+                          foreign_item: @ast::foreign_item,
+                          _: ()) {
+        visit::walk_foreign_item(self, foreign_item, ())
+    }
+
+    fn visit_local(&mut self, local: @ast::Local, _: ()) {
+        visit::walk_local(self, local, ())
+    }
+
+    fn visit_block(&mut self, block: &ast::Block, _: ()) {
+        visit::walk_block(self, block, ())
+    }
+
+    fn visit_stmt(&mut self, stmt: @ast::Stmt, _: ()) {
+        visit::walk_stmt(self, stmt, ())
+    }
+
+    fn visit_arm(&mut self, arm: &ast::Arm, _: ()) {
+        visit::walk_arm(self, arm, ())
+    }
+
+    fn visit_decl(&mut self, decl: @ast::Decl, _: ()) {
+        visit::walk_decl(self, decl, ())
+    }
+
+    fn visit_expr_post(&mut self, _: @ast::Expr, _: ()) {
+        // Empty!
+    }
+
+    fn visit_ty(&mut self, typ: &ast::Ty, _: ()) {
+        visit::walk_ty(self, typ, ())
+    }
+
+    fn visit_generics(&mut self, generics: &ast::Generics, _: ()) {
+        visit::walk_generics(self, generics, ())
+    }
+
+    fn visit_fn(&mut self,
+                function_kind: &visit::fn_kind,
+                function_declaration: &ast::fn_decl,
+                block: &ast::Block,
+                span: Span,
+                node_id: NodeId,
+                _: ()) {
+        visit::walk_fn(self,
+                        function_kind,
+                        function_declaration,
+                        block,
+                        span,
+                        node_id,
+                        ())
+    }
+
+    fn visit_ty_method(&mut self, ty_method: &ast::TypeMethod, _: ()) {
+        visit::walk_ty_method(self, ty_method, ())
+    }
+
+    fn visit_trait_method(&mut self,
+                          trait_method: &ast::trait_method,
+                          _: ()) {
+        visit::walk_trait_method(self, trait_method, ())
+    }
+
+    fn visit_struct_def(&mut self,
+                        struct_def: @ast::struct_def,
+                        ident: Ident,
+                        generics: &ast::Generics,
+                        node_id: NodeId,
+                        _: ()) {
+        visit::walk_struct_def(self,
+                                struct_def,
+                                ident,
+                                generics,
+                                node_id,
+                                ())
+    }
+
+    fn visit_struct_field(&mut self,
+                          struct_field: @ast::struct_field,
+                          _: ()) {
+        visit::walk_struct_field(self, struct_field, ())
+    }
+}
+
 // return a visitor that extracts the pat_ident paths
-// from a given pattern and puts them in a mutable
+// from a given thingy and puts them in a mutable
 // array (passed in to the traversal)
 pub fn new_name_finder(idents: @mut ~[ast::Ident]) -> @mut Visitor<()> {
     let context = @mut NewNameFinderContext {
@@ -674,16 +881,60 @@ pub fn new_name_finder(idents: @mut ~[ast::Ident]) -> @mut Visitor<()> {
     context as @mut Visitor<()>
 }
 
+// return a visitor that extracts the paths
+// from a given pattern and puts them in a mutable
+// array (passed in to the traversal)
+pub fn new_path_finder(paths: @mut ~[ast::Path]) -> @mut Visitor<()> {
+    let context = @mut NewPathExprFinderContext {
+        path_accumulator: paths,
+    };
+    context as @mut Visitor<()>
+}
+
+// expand a block. pushes a new exts_frame, then calls expand_block_elts
 pub fn expand_block(extsbox: @mut SyntaxEnv,
                     _cx: @ExtCtxt,
                     blk: &Block,
                     fld: @ast_fold,
-                    orig: @fn(&Block, @ast_fold) -> Block)
+                    _orig: @fn(&Block, @ast_fold) -> Block)
                  -> Block {
     // see note below about treatment of exts table
-    with_exts_frame!(extsbox,false,orig(blk,fld))
+    with_exts_frame!(extsbox,false,
+                     expand_block_elts(*extsbox, blk, fld))
+}
+
+// expand the elements of a block.
+pub fn expand_block_elts(exts: SyntaxEnv, b: &Block, fld: @ast_fold) -> Block {
+    let block_info = get_block_info(exts);
+    let pending_renames = block_info.pending_renames;
+    let rename_fld = renames_to_fold(pending_renames);
+    let new_view_items = b.view_items.map(|x| fld.fold_view_item(x));
+    let mut new_stmts = ~[];
+    for x in b.stmts.iter() {
+        match fld.fold_stmt(mustbesome(rename_fld.fold_stmt(*x))) {
+            Some(s) => new_stmts.push(s),
+            None => ()
+        }
+    }
+    let new_expr = b.expr.map(|x| fld.fold_expr(rename_fld.fold_expr(*x)));
+    Block{
+        view_items: new_view_items,
+        stmts: new_stmts,
+        expr: new_expr,
+        id: fld.new_id(b.id),
+        rules: b.rules,
+        span: b.span,
+    }
 }
 
+// rename_fold should never return "None".
+// (basically, just .get() with a better message...)
+fn mustbesome<T>(val : Option<T>) -> T {
+    match val {
+        Some(v) => v,
+        None => fail!("rename_fold returned None")
+    }
+}
 
 // get the (innermost) BlockInfo from an exts stack
 fn get_block_info(exts : SyntaxEnv) -> BlockInfo {
@@ -694,43 +945,14 @@ fn get_block_info(exts : SyntaxEnv) -> BlockInfo {
     }
 }
 
-
-// given a mutable list of renames, return a tree-folder that applies those
-// renames.
-fn renames_to_fold(renames : @mut ~[(ast::Ident,ast::Name)]) -> @ast_fold {
-    let afp = default_ast_fold();
-    let f_pre = @AstFoldFns {
-        fold_ident: |id,_| {
-            // the individual elements are memoized... it would
-            // also be possible to memoize on the whole list at once.
-            let new_ctxt = renames.iter().fold(id.ctxt,|ctxt,&(from,to)| {
-                new_rename(from,to,ctxt)
-            });
-            ast::Ident{name:id.name,ctxt:new_ctxt}
-        },
-        .. *afp
-    };
-    make_fold(f_pre)
-}
-
-// perform a bunch of renames
-fn apply_pending_renames(folder : @ast_fold, stmt : ast::Stmt) -> @ast::Stmt {
-    match folder.fold_stmt(&stmt) {
-        Some(s) => s,
-        None => fail!(fmt!("renaming of stmt produced None"))
-    }
-}
-
-
-
 pub fn new_span(cx: @ExtCtxt, sp: Span) -> Span {
     /* this discards information in the case of macro-defining macros */
     return Span {lo: sp.lo, hi: sp.hi, expn_info: cx.backtrace()};
 }
 
 // FIXME (#2247): this is a moderately bad kludge to inject some macros into
-// the default compilation environment. It would be much nicer to use
-// a mechanism like syntax_quote to ensure hygiene.
+// the default compilation environment in that it injects strings, rather than
+// syntax elements.
 
 pub fn std_macros() -> @str {
     return
@@ -1179,62 +1401,165 @@ pub fn expand_crate(parse_sess: @mut parse::ParseSess,
     return ret;
 }
 
-// given a function from idents to idents, produce
-// an ast_fold that applies that function:
-pub fn fun_to_ident_folder(f: @fn(ast::Ident)->ast::Ident) -> @ast_fold{
-    let afp = default_ast_fold();
-    let f_pre = @AstFoldFns{
-        fold_ident : |id, _| f(id),
-        .. *afp
-    };
-    make_fold(f_pre)
+// HYGIENIC CONTEXT EXTENSION:
+// all of these functions are for walking over
+// ASTs and making some change to the context of every
+// element that has one. a CtxtFn is a trait-ified
+// version of a closure in (SyntaxContext -> SyntaxContext).
+// the ones defined here include:
+// Renamer - add a rename to a context
+// MultiRenamer - add a set of renames to a context
+// Marker - add a mark to a context
+// Repainter - replace a context (maybe Replacer would be a better name?)
+
+// a function in SyntaxContext -> SyntaxContext
+pub trait CtxtFn{
+    fn f(&self, ast::SyntaxContext) -> ast::SyntaxContext;
 }
 
-// update the ctxts in a path to get a rename node
-pub fn new_ident_renamer(from: ast::Ident,
-                      to: ast::Name) ->
-    @fn(ast::Ident)->ast::Ident {
-    |id : ast::Ident|
-    ast::Ident{
-        name: id.name,
-        ctxt: new_rename(from,to,id.ctxt)
+// a renamer adds a rename to the syntax context
+pub struct Renamer {
+    from : ast::Ident,
+    to : ast::Name
+}
+
+impl CtxtFn for Renamer {
+    fn f(&self, ctxt : ast::SyntaxContext) -> ast::SyntaxContext {
+        new_rename(self.from,self.to,ctxt)
+    }
+}
+
+// a renamer that performs a whole bunch of renames
+pub struct MultiRenamer {
+    renames : @mut ~[(ast::Ident,ast::Name)]
+}
+
+impl CtxtFn for MultiRenamer {
+    fn f(&self, starting_ctxt : ast::SyntaxContext) -> ast::SyntaxContext {
+        // the individual elements are memoized... it would
+        // also be possible to memoize on the whole list at once.
+        self.renames.iter().fold(starting_ctxt,|ctxt,&(from,to)| {
+            new_rename(from,to,ctxt)
+        })
+    }
+}
+
+// a marker adds the given mark to the syntax context
+pub struct Marker { mark : Mrk }
+
+impl CtxtFn for Marker {
+    fn f(&self, ctxt : ast::SyntaxContext) -> ast::SyntaxContext {
+        new_mark(self.mark,ctxt)
     }
 }
 
+// a repainter just replaces the given context with the one it's closed over
+pub struct Repainter { ctxt : SyntaxContext }
 
-// update the ctxts in a path to get a mark node
-pub fn new_ident_marker(mark: uint) ->
-    @fn(ast::Ident)->ast::Ident {
-    |id : ast::Ident|
-    ast::Ident{
-        name: id.name,
-        ctxt: new_mark(mark,id.ctxt)
+impl CtxtFn for Repainter {
+    fn f(&self, _ctxt : ast::SyntaxContext) -> ast::SyntaxContext {
+        self.ctxt
     }
 }
 
-// perform resolution (in the MTWT sense) on all of the
-// idents in the tree. This is the final step in expansion.
-pub fn new_ident_resolver() ->
-    @fn(ast::Ident)->ast::Ident {
-    |id : ast::Ident|
-    ast::Ident {
-        name : resolve(id),
-        ctxt : ILLEGAL_CTXT
+// given a function from ctxts to ctxts, produce
+// an ast_fold that applies that function to all ctxts:
+pub fn fun_to_ctxt_folder<T : 'static + CtxtFn>(cf: @T) -> @AstFoldFns {
+    let afp = default_ast_fold();
+    let fi : @fn(ast::Ident, @ast_fold) -> ast::Ident =
+        |ast::Ident{name, ctxt}, _| {
+        ast::Ident{name:name,ctxt:cf.f(ctxt)}
+    };
+    let fm : @fn(&ast::mac_, Span, @ast_fold) -> (ast::mac_,Span) =
+        |m, sp, fld| {
+        match *m {
+            mac_invoc_tt(ref path, ref tts, ctxt) =>
+            (mac_invoc_tt(fld.fold_path(path),
+                         fold_tts(*tts,fld),
+                         cf.f(ctxt)),
+            sp)
+        }
+
+    };
+    @AstFoldFns{
+        fold_ident : fi,
+        fold_mac : fm,
+        .. *afp
     }
 }
 
 
+
+// given a mutable list of renames, return a tree-folder that applies those
+// renames.
+// FIXME #4536: currently pub to allow testing
+pub fn renames_to_fold(renames : @mut ~[(ast::Ident,ast::Name)]) -> @AstFoldFns {
+    fun_to_ctxt_folder(@MultiRenamer{renames : renames})
+}
+
+// just a convenience:
+pub fn new_mark_folder(m : Mrk) -> @AstFoldFns {
+    fun_to_ctxt_folder(@Marker{mark:m})
+}
+
+pub fn new_rename_folder(from : ast::Ident, to : ast::Name) -> @AstFoldFns {
+    fun_to_ctxt_folder(@Renamer{from:from,to:to})
+}
+
+// apply a given mark to the given token trees. Used prior to expansion of a macro.
+fn mark_tts(tts : &[token_tree], m : Mrk) -> ~[token_tree] {
+    fold_tts(tts,new_mark_folder(m) as @ast_fold)
+}
+
+// apply a given mark to the given expr. Used following the expansion of a macro.
+fn mark_expr(expr : @ast::Expr, m : Mrk) -> @ast::Expr {
+    new_mark_folder(m).fold_expr(expr)
+}
+
+// apply a given mark to the given stmt. Used following the expansion of a macro.
+fn mark_stmt(expr : &ast::Stmt, m : Mrk) -> @ast::Stmt {
+    new_mark_folder(m).fold_stmt(expr).unwrap()
+}
+
+// apply a given mark to the given item. Used following the expansion of a macro.
+fn mark_item(expr : @ast::item, m : Mrk) -> Option<@ast::item> {
+    new_mark_folder(m).fold_item(expr)
+}
+
+// replace all contexts in a given expr with the given mark. Used
+// for capturing macros
+pub fn replace_ctxts(expr : @ast::Expr, ctxt : SyntaxContext) -> @ast::Expr {
+    fun_to_ctxt_folder(@Repainter{ctxt:ctxt}).fold_expr(expr)
+}
+
+// take the mark from the given ctxt (that has a mark at the outside),
+// and apply it to everything in the token trees, thereby cancelling
+// that mark.
+pub fn mtwt_cancel_outer_mark(tts: &[ast::token_tree], ctxt: ast::SyntaxContext)
+    -> ~[ast::token_tree] {
+    let outer_mark = mtwt_outer_mark(ctxt);
+    mark_tts(tts,outer_mark)
+}
+
+
 #[cfg(test)]
 mod test {
     use super::*;
     use ast;
     use ast::{Attribute_, AttrOuter, MetaWord, EMPTY_CTXT};
+    use ast_util::{get_sctable, mtwt_marksof, mtwt_resolve, new_rename};
+    use ast_util;
     use codemap;
     use codemap::Spanned;
+    use fold;
     use parse;
-    use parse::token::{intern, get_ident_interner};
+    use parse::token::{fresh_mark, gensym, intern, get_ident_interner, ident_to_str};
+    use parse::token;
     use print::pprust;
-    use util::parser_testing::{string_to_item, string_to_pat, strs_to_idents};
+    use std;
+    use util::parser_testing::{string_to_crate, string_to_crate_and_sess};
+    use util::parser_testing::{string_to_pat, string_to_tts, strs_to_idents};
+    use visit;
 
     // make sure that fail! is present
     #[test] fn fail_exists_test () {
@@ -1332,28 +1657,251 @@ mod test {
         }
     }
 
+    #[test] fn cancel_outer_mark_test(){
+        let invalid_name = token::special_idents::invalid.name;
+        let ident_str = @"x";
+        let tts = string_to_tts(ident_str);
+        let fm = fresh_mark();
+        let marked_once = fold::fold_tts(tts,new_mark_folder(fm) as @fold::ast_fold);
+        assert_eq!(marked_once.len(),1);
+        let marked_once_ctxt =
+            match marked_once[0] {
+                ast::tt_tok(_,token::IDENT(id,_)) => id.ctxt,
+                _ => fail!(fmt!("unexpected shape for marked tts: %?",marked_once[0]))
+            };
+        assert_eq!(mtwt_marksof(marked_once_ctxt,invalid_name),~[fm]);
+        let remarked = mtwt_cancel_outer_mark(marked_once,marked_once_ctxt);
+        assert_eq!(remarked.len(),1);
+        match remarked[0] {
+            ast::tt_tok(_,token::IDENT(id,_)) =>
+            assert_eq!(mtwt_marksof(id.ctxt,invalid_name),~[]),
+            _ => fail!(fmt!("unexpected shape for marked tts: %?",remarked[0]))
+        }
+    }
+
     #[test]
     fn renaming () {
-        let maybe_item_ast = string_to_item(@"fn a() -> int { let b = 13; b }");
-        let item_ast = match maybe_item_ast {
-            Some(x) => x,
-            None => fail!("test case fail")
-        };
+        let item_ast = string_to_crate(@"fn f() -> int { a }");
         let a_name = intern("a");
-        let a2_name = intern("a2");
-        let renamer = new_ident_renamer(ast::Ident{name:a_name,ctxt:EMPTY_CTXT},
+        let a2_name = gensym("a2");
+        let renamer = new_rename_folder(ast::Ident{name:a_name,ctxt:EMPTY_CTXT},
                                         a2_name);
-        let renamed_ast = fun_to_ident_folder(renamer).fold_item(item_ast).unwrap();
-        let resolver = new_ident_resolver();
-        let resolved_ast = fun_to_ident_folder(resolver).fold_item(renamed_ast).unwrap();
-        let resolved_as_str = pprust::item_to_str(resolved_ast,
-                                                  get_ident_interner());
-        assert_eq!(resolved_as_str,~"fn a2() -> int { let b = 13; b }");
+        let renamed_ast = renamer.fold_crate(item_ast);
+        let varrefs = @mut ~[];
+        visit::walk_crate(&mut new_path_finder(varrefs), &renamed_ast, ());
+        match varrefs {
+            @[Path{segments:[ref seg],_}] => assert_eq!(mtwt_resolve(seg.identifier),a2_name),
+            _ => assert_eq!(0,1)
+        }
+
+        // try a double-rename, with pending_renames.
+        let a3_name = gensym("a3");
+        // a context that renames from ("a",empty) to "a2" :
+        let ctxt2 = new_rename(ast::Ident::new(a_name),a2_name,EMPTY_CTXT);
+        let pending_renames = @mut ~[(ast::Ident::new(a_name),a2_name),
+                                     (ast::Ident{name:a_name,ctxt:ctxt2},a3_name)];
+        let double_renamed = renames_to_fold(pending_renames).fold_crate(item_ast);
+        let varrefs = @mut ~[];
+        visit::walk_crate(&mut new_path_finder(varrefs), &double_renamed, ());
+        match varrefs {
+            @[Path{segments:[ref seg],_}] => assert_eq!(mtwt_resolve(seg.identifier),a3_name),
+            _ => assert_eq!(0,1)
+        }
+    }
 
+    fn fake_print_crate(crate: @ast::Crate) {
+        let s = pprust::rust_printer(std::io::stderr(),get_ident_interner());
+        pprust::print_crate_(s, crate);
+    }
 
+    fn expand_crate_str(crate_str: @str) -> @ast::Crate {
+        let (crate_ast,ps) = string_to_crate_and_sess(crate_str);
+        // the cfg argument actually does matter, here...
+        expand_crate(ps,~[],crate_ast)
     }
 
-    // sigh... it looks like I have two different renaming mechanisms, now...
+    //fn expand_and_resolve(crate_str: @str) -> ast::crate {
+        //let expanded_ast = expand_crate_str(crate_str);
+        // std::io::println(fmt!("expanded: %?\n",expanded_ast));
+        //mtwt_resolve_crate(expanded_ast)
+    //}
+    //fn expand_and_resolve_and_pretty_print (crate_str : @str) -> ~str {
+        //let resolved_ast = expand_and_resolve(crate_str);
+        //pprust::to_str(&resolved_ast,fake_print_crate,get_ident_interner())
+    //}
+
+    #[test] fn macro_tokens_should_match(){
+        expand_crate_str(@"macro_rules! m((a)=>(13)) fn main(){m!(a);}");
+    }
+
+    // renaming tests expand a crate and then check that the bindings match
+    // the right varrefs. The specification of the test case includes the
+    // text of the crate, and also an array of arrays.  Each element in the
+    // outer array corresponds to a binding in the traversal of the AST
+    // induced by visit.  Each of these arrays contains a list of indexes,
+    // interpreted as the varrefs in the varref traversal that this binding
+    // should match.  So, for instance, in a program with two bindings and
+    // three varrefs, the array ~[~[1,2],~[0]] would indicate that the first
+    // binding should match the second two varrefs, and the second binding
+    // should match the first varref.
+    //
+    // The comparisons are done post-mtwt-resolve, so we're comparing renamed
+    // names; differences in marks don't matter any more.
+    //
+    // oog... I also want tests that check "binding-identifier-=?". That is,
+    // not just "do these have the same name", but "do they have the same
+    // name *and* the same marks"? Understanding this is really pretty painful.
+    // in principle, you might want to control this boolean on a per-varref basis,
+    // but that would make things even harder to understand, and might not be
+    // necessary for thorough testing.
+    type renaming_test = (&'static str, ~[~[uint]], bool);
+
+    #[test]
+    fn automatic_renaming () {
+        let tests : ~[renaming_test] =
+            ~[// b & c should get new names throughout, in the expr too:
+                ("fn a() -> int { let b = 13; let c = b; b+c }",
+                 ~[~[0,1],~[2]], false),
+                // both x's should be renamed (how is this causing a bug?)
+                ("fn main () {let x : int = 13;x;}",
+                 ~[~[0]], false),
+                // the use of b after the + should be renamed, the other one not:
+                ("macro_rules! f (($x:ident) => (b + $x)) fn a() -> int { let b = 13; f!(b)}",
+                 ~[~[1]], false),
+                // the b before the plus should not be renamed (requires marks)
+                ("macro_rules! f (($x:ident) => ({let b=9; ($x + b)})) fn a() -> int { f!(b)}",
+                 ~[~[1]], false),
+                // the marks going in and out of letty should cancel, allowing that $x to
+                // capture the one following the semicolon.
+                // this was an awesome test case, and caught a *lot* of bugs.
+                ("macro_rules! letty(($x:ident) => (let $x = 15;))
+                  macro_rules! user(($x:ident) => ({letty!($x); $x}))
+                  fn main() -> int {user!(z)}",
+                 ~[~[0]], false),
+                // FIXME #8062: this test exposes a *potential* bug; our system does
+                // not behave exactly like MTWT, but I haven't thought of a way that
+                // this could cause a bug in Rust, yet.
+                // ("fn main() {let hrcoo = 19; macro_rules! getx(()=>(hrcoo)); getx!();}",
+                // ~[~[0]], true)
+                // FIXME #6994: the next string exposes the bug referred to in issue 6994, so I'm
+                // commenting it out.
+                // the z flows into and out of two macros (g & f) along one path, and one
+                // (just g) along the other, so the result of the whole thing should
+                // be "let z_123 = 3; z_123"
+                //"macro_rules! g (($x:ident) =>
+                //   ({macro_rules! f(($y:ident)=>({let $y=3;$x}));f!($x)}))
+                //   fn a(){g!(z)}"
+                // create a really evil test case where a $x appears inside a binding of $x
+                // but *shouldnt* bind because it was inserted by a different macro....
+            ];
+        for (idx,s) in tests.iter().enumerate() {
+            run_renaming_test(s,idx);
+        }
+    }
+
+    // run one of the renaming tests
+    fn run_renaming_test(t : &renaming_test, test_idx: uint) {
+        let invalid_name = token::special_idents::invalid.name;
+        let (teststr, bound_connections, bound_ident_check) = match *t {
+            (ref str,ref conns, bic) => (str.to_managed(), conns.clone(), bic)
+        };
+        let cr = expand_crate_str(teststr.to_managed());
+        // find the bindings:
+        let bindings = @mut ~[];
+        visit::walk_crate(&mut new_name_finder(bindings),cr,());
+        // find the varrefs:
+        let varrefs = @mut ~[];
+        visit::walk_crate(&mut new_path_finder(varrefs),cr,());
+        // must be one check clause for each binding:
+        assert_eq!(bindings.len(),bound_connections.len());
+        for (binding_idx,shouldmatch) in bound_connections.iter().enumerate() {
+            let binding_name = mtwt_resolve(bindings[binding_idx]);
+            let binding_marks = mtwt_marksof(bindings[binding_idx].ctxt,invalid_name);
+            // shouldmatch can't name varrefs that don't exist:
+            assert!((shouldmatch.len() == 0) ||
+                    (varrefs.len() > *shouldmatch.iter().max().unwrap()));
+            for (idx,varref) in varrefs.iter().enumerate() {
+                if shouldmatch.contains(&idx) {
+                    // it should be a path of length 1, and it should
+                    // be free-identifier=? or bound-identifier=? to the given binding
+                    assert_eq!(varref.segments.len(),1);
+                    let varref_name = mtwt_resolve(varref.segments[0].identifier);
+                    let varref_marks = mtwt_marksof(varref.segments[0].identifier.ctxt,
+                                                    invalid_name);
+                    if (!(varref_name==binding_name)){
+                        std::io::println("uh oh, should match but doesn't:");
+                        std::io::println(fmt!("varref: %?",varref));
+                        std::io::println(fmt!("binding: %?", bindings[binding_idx]));
+                        ast_util::display_sctable(get_sctable());
+                    }
+                    assert_eq!(varref_name,binding_name);
+                    if (bound_ident_check) {
+                        // we're checking bound-identifier=?, and the marks
+                        // should be the same, too:
+                        assert_eq!(varref_marks,binding_marks.clone());
+                    }
+                } else {
+                    let fail = (varref.segments.len() == 1)
+                        && (mtwt_resolve(varref.segments[0].identifier) == binding_name);
+                    // temp debugging:
+                    if (fail) {
+                        println!("failure on test {}",test_idx);
+                        println!("text of test case: \"{}\"", teststr);
+                        println!("");
+                        println!("uh oh, matches but shouldn't:");
+                        std::io::println(fmt!("varref: %?",varref));
+                        // good lord, you can't make a path with 0 segments, can you?
+                        println!("varref's first segment's uint: {}, and string: \"{}\"",
+                                 varref.segments[0].identifier.name,
+                                 ident_to_str(&varref.segments[0].identifier));
+                        std::io::println(fmt!("binding: %?", bindings[binding_idx]));
+                        ast_util::display_sctable(get_sctable());
+                    }
+                    assert!(!fail);
+                }
+            }
+        }
+    }
+
+    #[test] fn fmt_in_macro_used_inside_module_macro() {
+        let crate_str = @"macro_rules! fmt_wrap(($b:expr)=>(fmt!(\"left: %?\", $b)))
+macro_rules! foo_module (() => (mod generated { fn a() { let xx = 147; fmt_wrap!(xx);}}))
+foo_module!()
+";
+        let cr = expand_crate_str(crate_str);
+        // find the xx binding
+        let bindings = @mut ~[];
+        visit::walk_crate(&mut new_name_finder(bindings), cr, ());
+        let cxbinds : ~[&ast::Ident] =
+            bindings.iter().filter(|b|{@"xx" == (ident_to_str(*b))}).collect();
+        let cxbind = match cxbinds {
+            [b] => b,
+            _ => fail!("expected just one binding for ext_cx")
+        };
+        let resolved_binding = mtwt_resolve(*cxbind);
+        // find all the xx varrefs:
+        let varrefs = @mut ~[];
+        visit::walk_crate(&mut new_path_finder(varrefs), cr, ());
+        // the xx binding should bind all of the xx varrefs:
+        for (idx,v) in varrefs.iter().filter(|p|{ p.segments.len() == 1
+                                          && (@"xx" == (ident_to_str(&p.segments[0].identifier)))
+                                     }).enumerate() {
+            if (mtwt_resolve(v.segments[0].identifier) != resolved_binding) {
+                std::io::println("uh oh, xx binding didn't match xx varref:");
+                std::io::println(fmt!("this is xx varref # %?",idx));
+                std::io::println(fmt!("binding: %?",cxbind));
+                std::io::println(fmt!("resolves to: %?",resolved_binding));
+                std::io::println(fmt!("varref: %?",v.segments[0].identifier));
+                std::io::println(fmt!("resolves to: %?",mtwt_resolve(v.segments[0].identifier)));
+                let table = get_sctable();
+                std::io::println("SC table:");
+                for (idx,val) in table.table.iter().enumerate() {
+                    std::io::println(fmt!("%4u : %?",idx,val));
+                }
+            }
+            assert_eq!(mtwt_resolve(v.segments[0].identifier),resolved_binding);
+        };
+    }
 
     #[test]
     fn pat_idents(){
@@ -1363,4 +1911,5 @@ mod test {
         pat_idents.visit_pat(pat, ());
         assert_eq!(idents, @mut strs_to_idents(~["a","c","b","d"]));
     }
+
 }
diff --git a/src/libsyntax/ext/quote.rs b/src/libsyntax/ext/quote.rs
index 4e8b7467c5c..6527b083cc1 100644
--- a/src/libsyntax/ext/quote.rs
+++ b/src/libsyntax/ext/quote.rs
@@ -293,45 +293,51 @@ pub fn expand_quote_tokens(cx: @ExtCtxt,
                            sp: Span,
                            tts: &[ast::token_tree]) -> base::MacResult {
     let (cx_expr, expr) = expand_tts(cx, sp, tts);
-    base::MRExpr(expand_wrapper(cx, sp, cx_expr, expr))
+    let expanded = expand_wrapper(cx, sp, cx_expr, expr);
+    base::MRExpr(expanded)
 }
 
 pub fn expand_quote_expr(cx: @ExtCtxt,
                          sp: Span,
                          tts: &[ast::token_tree]) -> base::MacResult {
-    base::MRExpr(expand_parse_call(cx, sp, "parse_expr", ~[], tts))
+    let expanded = expand_parse_call(cx, sp, "parse_expr", ~[], tts);
+    base::MRExpr(expanded)
 }
 
 pub fn expand_quote_item(cx: @ExtCtxt,
                          sp: Span,
                          tts: &[ast::token_tree]) -> base::MacResult {
     let e_attrs = cx.expr_vec_uniq(sp, ~[]);
-    base::MRExpr(expand_parse_call(cx, sp, "parse_item",
-                                    ~[e_attrs], tts))
+    let expanded = expand_parse_call(cx, sp, "parse_item",
+                                    ~[e_attrs], tts);
+    base::MRExpr(expanded)
 }
 
 pub fn expand_quote_pat(cx: @ExtCtxt,
                         sp: Span,
                         tts: &[ast::token_tree]) -> base::MacResult {
     let e_refutable = cx.expr_lit(sp, ast::lit_bool(true));
-    base::MRExpr(expand_parse_call(cx, sp, "parse_pat",
-                                    ~[e_refutable], tts))
+    let expanded = expand_parse_call(cx, sp, "parse_pat",
+                                    ~[e_refutable], tts);
+    base::MRExpr(expanded)
 }
 
 pub fn expand_quote_ty(cx: @ExtCtxt,
                        sp: Span,
                        tts: &[ast::token_tree]) -> base::MacResult {
     let e_param_colons = cx.expr_lit(sp, ast::lit_bool(false));
-    base::MRExpr(expand_parse_call(cx, sp, "parse_ty",
-                                    ~[e_param_colons], tts))
+    let expanded = expand_parse_call(cx, sp, "parse_ty",
+                                     ~[e_param_colons], tts);
+    base::MRExpr(expanded)
 }
 
 pub fn expand_quote_stmt(cx: @ExtCtxt,
                          sp: Span,
                          tts: &[ast::token_tree]) -> base::MacResult {
     let e_attrs = cx.expr_vec_uniq(sp, ~[]);
-    base::MRExpr(expand_parse_call(cx, sp, "parse_stmt",
-                                    ~[e_attrs], tts))
+    let expanded = expand_parse_call(cx, sp, "parse_stmt",
+                                    ~[e_attrs], tts);
+    base::MRExpr(expanded)
 }
 
 fn ids_ext(strs: ~[~str]) -> ~[ast::Ident] {
diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs
index 7a9e916421c..aa4183837e3 100644
--- a/src/libsyntax/ext/tt/macro_parser.rs
+++ b/src/libsyntax/ext/tt/macro_parser.rs
@@ -234,6 +234,15 @@ pub fn parse_or_else(
     }
 }
 
+// perform a token equality check, ignoring syntax context (that is, an unhygienic comparison)
+pub fn token_name_eq(t1 : &Token, t2 : &Token) -> bool {
+    match (t1,t2) {
+        (&token::IDENT(id1,_),&token::IDENT(id2,_)) =>
+        id1.name == id2.name,
+        _ => *t1 == *t2
+    }
+}
+
 pub fn parse(
     sess: @mut ParseSess,
     cfg: ast::CrateConfig,
@@ -297,7 +306,10 @@ pub fn parse(
                     // the *_t vars are workarounds for the lack of unary move
                     match ei.sep {
                       Some(ref t) if idx == len => { // we need a separator
-                        if tok == (*t) { //pass the separator
+                        // i'm conflicted about whether this should be hygienic....
+                        // though in this case, if the separators are never legal
+                        // idents, it shouldn't matter.
+                        if token_name_eq(&tok, t) { //pass the separator
                             let mut ei_t = ei.clone();
                             ei_t.idx += 1;
                             next_eis.push(ei_t);
@@ -343,7 +355,8 @@ pub fn parse(
                   match_nonterminal(_,_,_) => { bb_eis.push(ei) }
                   match_tok(ref t) => {
                     let mut ei_t = ei.clone();
-                    if (*t) == tok {
+                    //if (token_name_eq(t,&tok)) {
+                    if (token::mtwt_token_eq(t,&tok)) {
                         ei_t.idx += 1;
                         next_eis.push(ei_t);
                     }
@@ -353,7 +366,7 @@ pub fn parse(
         }
 
         /* error messages here could be improved with links to orig. rules */
-        if tok == EOF {
+        if token_name_eq(&tok, &EOF) {
             if eof_eis.len() == 1u {
                 let mut v = ~[];
                 for dv in eof_eis[0u].matches.mut_iter() {
diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs
index 732ae2ccb96..74de8eaa09e 100644
--- a/src/libsyntax/ext/tt/macro_rules.rs
+++ b/src/libsyntax/ext/tt/macro_rules.rs
@@ -14,6 +14,7 @@ use ast;
 use codemap::{Span, Spanned, dummy_sp};
 use ext::base::{ExtCtxt, MacResult, MRAny, MRDef, MacroDef, NormalTT};
 use ext::base;
+use ext::expand;
 use ext::tt::macro_parser::{error};
 use ext::tt::macro_parser::{named_match, matched_seq, matched_nonterminal};
 use ext::tt::macro_parser::{parse, parse_or_else, success, failure};
@@ -23,11 +24,17 @@ use parse::token::{get_ident_interner, special_idents, gensym_ident, ident_to_st
 use parse::token::{FAT_ARROW, SEMI, nt_matchers, nt_tt};
 use print;
 
+// this procedure performs the expansion of the
+// macro_rules! macro. It parses the RHS and adds
+// an extension to the current context.
 pub fn add_new_extension(cx: @ExtCtxt,
                          sp: Span,
                          name: Ident,
-                         arg: ~[ast::token_tree])
+                         arg: ~[ast::token_tree],
+                         stx_ctxt: ast::SyntaxContext)
                       -> base::MacResult {
+    let arg = expand::mtwt_cancel_outer_mark(arg,stx_ctxt);
+    // Wrap a matcher_ in a spanned to produce a matcher.
     // these spans won't matter, anyways
     fn ms(m: matcher_) -> matcher {
         Spanned {
@@ -39,6 +46,7 @@ pub fn add_new_extension(cx: @ExtCtxt,
     let lhs_nm =  gensym_ident("lhs");
     let rhs_nm =  gensym_ident("rhs");
 
+    // The pattern that macro_rules matches.
     // The grammar for macro_rules! is:
     // $( $lhs:mtcs => $rhs:tt );+
     // ...quasiquoting this would be nice.
@@ -144,11 +152,11 @@ pub fn add_new_extension(cx: @ExtCtxt,
         cx.span_fatal(best_fail_spot, best_fail_msg);
     }
 
-    let exp: @fn(@ExtCtxt, Span, &[ast::token_tree]) -> MacResult =
-        |cx, sp, arg| generic_extension(cx, sp, name, arg, *lhses, *rhses);
+    let exp: @fn(@ExtCtxt, Span, &[ast::token_tree], ctxt: ast::SyntaxContext) -> MacResult =
+        |cx, sp, arg, _ctxt| generic_extension(cx, sp, name, arg, *lhses, *rhses);
 
     return MRDef(MacroDef{
         name: ident_to_str(&name),
-        ext: NormalTT(base::SyntaxExpanderTT{expander: exp, span: Some(sp)})
+        ext: NormalTT(exp, Some(sp))
     });
 }