From 77c96d754e3a5d4b8dd59269b0cb0324350ccbec Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Tue, 9 Jul 2013 22:52:34 -0700
Subject: disallow ident equality checks when contexts are not equal

---
 src/libsyntax/ast.rs | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs
index c7ebc344a9c..8a9bd5192ad 100644
--- a/src/libsyntax/ast.rs
+++ b/src/libsyntax/ast.rs
@@ -24,7 +24,7 @@ use extra::serialize::{Encodable, Decodable, Encoder, Decoder};
 // table) and a SyntaxContext to track renaming and
 // macro expansion per Flatt et al., "Macros
 // That Work Together"
-#[deriving(Clone, Eq, IterBytes, ToStr)]
+#[deriving(Clone, IterBytes, ToStr)]
 pub struct Ident { name: Name, ctxt: SyntaxContext }
 
 impl Ident {
@@ -32,6 +32,19 @@ impl Ident {
     pub fn new(name: Name) -> Ident { Ident {name: name, ctxt: EMPTY_CTXT}}
 }
 
+impl Eq for Ident {
+    fn eq(&self, other: &Ident) -> bool {
+        if (self.ctxt == other.ctxt) {
+            self.name == other.name
+        } else {
+            fail!(fmt!("not allowed to compare these idents: %?, %?", self, other));
+        }
+    }
+    fn ne(&self, other: &Ident) -> bool {
+        ! self.eq(other)
+    }
+}
+
 /// A SyntaxContext represents a chain of macro-expandings
 /// and renamings. Each macro expansion corresponds to
 /// a fresh uint
-- 
cgit 1.4.1-3-g733a5


From acad9f35b3aef8cc0d82c6a43c662ddc0f946568 Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Fri, 6 Sep 2013 13:34:58 -0700
Subject: commenting out special Eq implementation for now

See the comments for details on why I'm leaving this code
in, though commented out.
---
 src/libsyntax/ast.rs | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs
index 8a9bd5192ad..c2087928e8d 100644
--- a/src/libsyntax/ast.rs
+++ b/src/libsyntax/ast.rs
@@ -24,7 +24,7 @@ use extra::serialize::{Encodable, Decodable, Encoder, Decoder};
 // table) and a SyntaxContext to track renaming and
 // macro expansion per Flatt et al., "Macros
 // That Work Together"
-#[deriving(Clone, IterBytes, ToStr)]
+#[deriving(Clone, Eq, IterBytes, ToStr)]
 pub struct Ident { name: Name, ctxt: SyntaxContext }
 
 impl Ident {
@@ -32,7 +32,17 @@ impl Ident {
     pub fn new(name: Name) -> Ident { Ident {name: name, ctxt: EMPTY_CTXT}}
 }
 
-impl Eq for Ident {
+// defining eq in this way is a way of guaranteeing that later stages of the
+// compiler don't compare identifiers unhygienically. Unfortunately, some tests
+// (specifically debuginfo in no-opt) want to do these comparisons, and that
+// seems fine.  If only I could find a nice way to statically ensure that
+// the compiler "proper" never compares identifiers.... I'm leaving this
+// code here (commented out) for potential use in debugging. Specifically, if
+// there's a bug where "identifiers aren't matching", it may be because
+// they should be compared using mtwt_resolve. In such a case, re-enabling this
+// code (and disabling deriving(Eq) for Idents) could help to isolate the
+// problem
+/* impl Eq for Ident {
     fn eq(&self, other: &Ident) -> bool {
         if (self.ctxt == other.ctxt) {
             self.name == other.name
@@ -44,6 +54,7 @@ impl Eq for Ident {
         ! self.eq(other)
     }
 }
+*/
 
 /// A SyntaxContext represents a chain of macro-expandings
 /// and renamings. Each macro expansion corresponds to
-- 
cgit 1.4.1-3-g733a5


From e1734f6d38a84bea8a4fd3a8a5ed09ae2666cf14 Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Tue, 9 Jul 2013 14:09:30 -0700
Subject: rename resolve to mtwt_resolve

---
 src/libsyntax/ast_util.rs   | 2 +-
 src/libsyntax/ext/expand.rs | 5 ++---
 2 files changed, 3 insertions(+), 4 deletions(-)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/ast_util.rs b/src/libsyntax/ast_util.rs
index 5ee8537750e..0e402731c66 100644
--- a/src/libsyntax/ast_util.rs
+++ b/src/libsyntax/ast_util.rs
@@ -915,7 +915,7 @@ fn idx_push<T>(vec: &mut ~[T], val: T) -> uint {
 }
 
 /// Resolve a syntax object to a name, per MTWT.
-pub fn resolve(id : Ident) -> Name {
+pub fn mtwt_resolve(id : Ident) -> Name {
     resolve_internal(id, get_sctable())
 }
 
diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index ea277a8a625..8781dd1feff 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -12,7 +12,7 @@ use ast::{Block, Crate, NodeId, Expr_, ExprMac, Ident, mac_invoc_tt};
 use ast::{item_mac, Stmt_, StmtMac, StmtExpr, StmtSemi};
 use ast::{ILLEGAL_CTXT};
 use ast;
-use ast_util::{new_rename, new_mark, resolve};
+use ast_util::{new_rename, new_mark, mtwt_resolve};
 use attr;
 use attr::AttrMetaMethods;
 use codemap;
@@ -1218,12 +1218,11 @@ pub fn new_ident_resolver() ->
     @fn(ast::Ident)->ast::Ident {
     |id : ast::Ident|
     ast::Ident {
-        name : resolve(id),
+        name : mtwt_resolve(id),
         ctxt : ILLEGAL_CTXT
     }
 }
 
-
 #[cfg(test)]
 mod test {
     use super::*;
-- 
cgit 1.4.1-3-g733a5


From 72ee6af4d4e4d16a81e3f518f9c75f513a9a7dea Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Thu, 5 Sep 2013 14:14:31 -0700
Subject: compare macro tokens hygienically (commented out)

---
 src/libsyntax/ext/tt/macro_parser.rs | 16 +++++++++++++++-
 src/libsyntax/parse/parser.rs        |  5 ++++-
 2 files changed, 19 insertions(+), 2 deletions(-)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs
index 7a9e916421c..f859a656d1f 100644
--- a/src/libsyntax/ext/tt/macro_parser.rs
+++ b/src/libsyntax/ext/tt/macro_parser.rs
@@ -234,6 +234,19 @@ pub fn parse_or_else(
     }
 }
 
+// temporary for testing
+pub fn token_name_eq(t1 : &Token, t2 : &Token) -> bool {
+    if (*t1 == *t2) {
+        true
+    } else {
+        match (t1,t2) {
+            (&token::IDENT(id1,_),&token::IDENT(id2,_)) =>
+            id1.name == id2.name,
+            _ => false
+        }
+    }
+}
+
 pub fn parse(
     sess: @mut ParseSess,
     cfg: ast::CrateConfig,
@@ -343,7 +356,8 @@ pub fn parse(
                   match_nonterminal(_,_,_) => { bb_eis.push(ei) }
                   match_tok(ref t) => {
                     let mut ei_t = ei.clone();
-                    if (*t) == tok {
+                    if (token_name_eq(t,&tok)) {
+                    //if (token::mtwt_token_eq(t,&tok)) {
                         ei_t.idx += 1;
                         next_eis.push(ei_t);
                     }
diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs
index 51c5522ae2f..62abe3850c9 100644
--- a/src/libsyntax/parse/parser.rs
+++ b/src/libsyntax/parse/parser.rs
@@ -3518,7 +3518,10 @@ impl Parser {
     }
 
     fn is_self_ident(&self) -> bool {
-        *self.token == token::IDENT(special_idents::self_, false)
+        match *self.token {
+          token::IDENT(id, false) => id.name == special_idents::self_.name,
+          _ => false
+        }
     }
 
     fn expect_self_ident(&self) {
-- 
cgit 1.4.1-3-g733a5


From fa6c981606b89ebed80b0dd5e829d86cdb3078d8 Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Tue, 25 Jun 2013 11:40:51 -0700
Subject: add hygiene support fns, move them around.

also adds test cases
---
 src/libsyntax/ast.rs                 |   2 +-
 src/libsyntax/ast_util.rs            |   3 -
 src/libsyntax/ext/expand.rs          | 205 ++++++++++++++++++++++++++++-------
 src/libsyntax/parse/token.rs         |  10 +-
 src/libsyntax/util/parser_testing.rs |  15 ++-
 5 files changed, 175 insertions(+), 60 deletions(-)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs
index c2087928e8d..ef5282551a1 100644
--- a/src/libsyntax/ast.rs
+++ b/src/libsyntax/ast.rs
@@ -571,7 +571,7 @@ pub enum token_tree {
     // These only make sense for right-hand-sides of MBE macros:
 
     // a kleene-style repetition sequence with a span, a tt_forest,
-    // an optional separator (?), and a boolean where true indicates
+    // an optional separator, and a boolean where true indicates
     // zero or more (*), and false indicates one or more (+).
     tt_seq(Span, @mut ~[token_tree], Option<::parse::token::Token>, bool),
 
diff --git a/src/libsyntax/ast_util.rs b/src/libsyntax/ast_util.rs
index 0e402731c66..aeca145ea18 100644
--- a/src/libsyntax/ast_util.rs
+++ b/src/libsyntax/ast_util.rs
@@ -825,9 +825,6 @@ pub fn pat_is_ident(pat: @ast::Pat) -> bool {
 
 // HYGIENE FUNCTIONS
 
-/// Construct an identifier with the given name and an empty context:
-pub fn new_ident(name: Name) -> Ident { Ident {name: name, ctxt: 0}}
-
 /// Extend a syntax context with a given mark
 pub fn new_mark(m:Mrk, tail:SyntaxContext) -> SyntaxContext {
     new_mark_internal(m,tail,get_sctable())
diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index 8781dd1feff..81a47e0e485 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -8,9 +8,9 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use ast::{Block, Crate, NodeId, Expr_, ExprMac, Ident, mac_invoc_tt};
-use ast::{item_mac, Stmt_, StmtMac, StmtExpr, StmtSemi};
-use ast::{ILLEGAL_CTXT};
+use ast::{Block, Crate, NodeId, DeclLocal, Expr_, ExprMac, Local, Ident, mac_invoc_tt};
+use ast::{item_mac, Mrk, Stmt_, StmtDecl, StmtMac, StmtExpr, StmtSemi};
+use ast::{ILLEGAL_CTXT, SCTable, token_tree};
 use ast;
 use ast_util::{new_rename, new_mark, mtwt_resolve};
 use attr;
@@ -23,7 +23,7 @@ use opt_vec;
 use parse;
 use parse::{parse_item_from_source_str};
 use parse::token;
-use parse::token::{ident_to_str, intern};
+use parse::token::{fresh_name, ident_to_str, intern};
 use visit;
 use visit::Visitor;
 
@@ -521,6 +521,71 @@ pub fn expand_stmt(extsbox: @mut SyntaxEnv,
 
 }
 
+// expand a non-macro stmt. this is essentially the fallthrough for
+// expand_stmt, above.
+fn expand_non_macro_stmt (exts: SyntaxEnv,
+                          s: &Stmt_,
+                          sp: Span,
+                          fld: @ast_fold,
+                          orig: @fn(&Stmt_, Span, @ast_fold) -> (Option<Stmt_>, Span))
+    -> (Option<Stmt_>,Span) {
+    // is it a let?
+    match *s {
+        StmtDecl(@Spanned{node: DeclLocal(ref local), span: stmt_span}, node_id) => {
+            let block_info = get_block_info(exts);
+            let pending_renames = block_info.pending_renames;
+
+            // take it apart:
+            let @Local{is_mutbl:is_mutbl,
+                       ty:_,
+                       pat:pat,
+                       init:init,
+                       id:id,
+                       span:span
+                      } = *local;
+            // types can't be copied automatically because of the owned ptr in ty_tup...
+            let ty = local.ty.clone();
+            // expand the pat (it might contain exprs... #:(o)>
+            let expanded_pat = fld.fold_pat(pat);
+            // find the pat_idents in the pattern:
+            // oh dear heaven... this is going to include the enum names, as well....
+            let idents = @mut ~[];
+            let name_finder = new_name_finder(idents);
+            name_finder.visit_pat(expanded_pat,());
+            // generate fresh names, push them to a new pending list
+            let new_pending_renames = @mut ~[];
+            for ident in idents.iter() {
+                let new_name = fresh_name(ident);
+                new_pending_renames.push((*ident,new_name));
+            }
+            let mut rename_fld = renames_to_fold(new_pending_renames);
+            // rewrite the pattern using the new names (the old ones
+            // have already been applied):
+            let rewritten_pat = rename_fld.fold_pat(expanded_pat);
+            // add them to the existing pending renames:
+            for pr in new_pending_renames.iter() {pending_renames.push(*pr)}
+            // also, don't forget to expand the init:
+            let new_init_opt = init.map(|e| fld.fold_expr(*e));
+            let rewritten_local =
+                @Local{is_mutbl:is_mutbl,
+                       ty:ty,
+                       pat:rewritten_pat,
+                       init:new_init_opt,
+                       id:id,
+                       span:span};
+            (Some(StmtDecl(@Spanned{node:DeclLocal(rewritten_local),
+                                     span: stmt_span},node_id)),
+             sp)
+        },
+        _ => {
+            orig(s, sp, fld)
+        }
+    }
+}
+
+// a visitor that extracts the pat_ident paths
+// from a given pattern and puts them in a mutable
+// array (passed in to the traversal)
 #[deriving(Clone)]
 struct NewNameFinderContext {
     ident_accumulator: @mut ~[ast::Ident],
@@ -674,30 +739,10 @@ pub fn new_name_finder(idents: @mut ~[ast::Ident]) -> @mut Visitor<()> {
     context as @mut Visitor<()>
 }
 
-pub fn expand_block(extsbox: @mut SyntaxEnv,
-                    _cx: @ExtCtxt,
-                    blk: &Block,
-                    fld: @ast_fold,
-                    orig: @fn(&Block, @ast_fold) -> Block)
-                 -> Block {
-    // see note below about treatment of exts table
-    with_exts_frame!(extsbox,false,orig(blk,fld))
-}
-
-
-// get the (innermost) BlockInfo from an exts stack
-fn get_block_info(exts : SyntaxEnv) -> BlockInfo {
-    match exts.find_in_topmost_frame(&intern(special_block_name)) {
-        Some(@BlockInfo(bi)) => bi,
-        _ => fail!(fmt!("special identifier %? was bound to a non-BlockInfo",
-                       @" block"))
-    }
-}
-
-
 // given a mutable list of renames, return a tree-folder that applies those
 // renames.
-fn renames_to_fold(renames : @mut ~[(ast::Ident,ast::Name)]) -> @ast_fold {
+// FIXME #4536: currently pub to allow testing
+pub fn renames_to_fold(renames : @mut ~[(ast::Ident,ast::Name)]) -> @ast_fold {
     let afp = default_ast_fold();
     let f_pre = @AstFoldFns {
         fold_ident: |id,_| {
@@ -713,15 +758,56 @@ fn renames_to_fold(renames : @mut ~[(ast::Ident,ast::Name)]) -> @ast_fold {
     make_fold(f_pre)
 }
 
-// perform a bunch of renames
-fn apply_pending_renames(folder : @ast_fold, stmt : ast::Stmt) -> @ast::Stmt {
-    match folder.fold_stmt(&stmt) {
-        Some(s) => s,
-        None => fail!(fmt!("renaming of stmt produced None"))
+pub fn expand_block(extsbox: @mut SyntaxEnv,
+                    _cx: @ExtCtxt,
+                    blk: &Block,
+                    fld: @ast_fold,
+                    orig: @fn(&Block, @ast_fold) -> Block)
+                 -> Block {
+    // see note below about treatment of exts table
+    with_exts_frame!(extsbox,false,orig(blk,fld))
+}
+
+
+pub fn expand_block_elts(exts: SyntaxEnv, b: &Block, fld: @ast_fold) -> Block {
+    let block_info = get_block_info(exts);
+    let pending_renames = block_info.pending_renames;
+    let mut rename_fld = renames_to_fold(pending_renames);
+    let new_view_items = b.view_items.map(|x| fld.fold_view_item(x));
+    let mut new_stmts = ~[];
+    for x in b.stmts.iter() {
+        match fld.fold_stmt(mustbesome(rename_fld.fold_stmt(*x))) {
+            Some(s) => new_stmts.push(s),
+            None => ()
+        }
+    }
+    let new_expr = b.expr.map(|x| fld.fold_expr(rename_fld.fold_expr(*x)));
+    Block{
+        view_items: new_view_items,
+        stmts: new_stmts,
+        expr: new_expr,
+        id: fld.new_id(b.id),
+        rules: b.rules,
+        span: b.span,
     }
 }
 
+// rename_fold should never return "None".
+fn mustbesome<T>(val : Option<T>) -> T {
+    match val {
+        Some(v) => v,
+        None => fail!("rename_fold returned None")
+    }
+}
 
+// get the (innermost) BlockInfo from an exts stack
+fn get_block_info(exts : SyntaxEnv) -> BlockInfo {
+    match exts.find_in_topmost_frame(&intern(special_block_name)) {
+        Some(@BlockInfo(bi)) => bi,
+        _ => fail!(fmt!("special identifier %? was bound to a non-BlockInfo",
+                       @" block"))
+    }
+}
 
 pub fn new_span(cx: @ExtCtxt, sp: Span) -> Span {
     /* this discards information in the case of macro-defining macros */
@@ -1228,12 +1314,15 @@ mod test {
     use super::*;
     use ast;
     use ast::{Attribute_, AttrOuter, MetaWord, EMPTY_CTXT};
+    use ast_util::{get_sctable, new_rename};
     use codemap;
     use codemap::Spanned;
     use parse;
-    use parse::token::{intern, get_ident_interner};
+    use parse::token::{gensym, intern, get_ident_interner};
     use print::pprust;
-    use util::parser_testing::{string_to_item, string_to_pat, strs_to_idents};
+    use std;
+    use util::parser_testing::{string_to_crate_and_sess, string_to_item, string_to_pat};
+    use util::parser_testing::{strs_to_idents};
 
     // make sure that fail! is present
     #[test] fn fail_exists_test () {
@@ -1333,26 +1422,60 @@ mod test {
 
     #[test]
     fn renaming () {
-        let maybe_item_ast = string_to_item(@"fn a() -> int { let b = 13; b }");
-        let item_ast = match maybe_item_ast {
-            Some(x) => x,
-            None => fail!("test case fail")
-        };
+        let item_ast = string_to_item(@"fn a() -> int { let b = 13; b }").unwrap();
         let a_name = intern("a");
-        let a2_name = intern("a2");
+        let a2_name = gensym("a2");
         let renamer = new_ident_renamer(ast::Ident{name:a_name,ctxt:EMPTY_CTXT},
                                         a2_name);
         let renamed_ast = fun_to_ident_folder(renamer).fold_item(item_ast).unwrap();
         let resolver = new_ident_resolver();
-        let resolved_ast = fun_to_ident_folder(resolver).fold_item(renamed_ast).unwrap();
+        let resolver_fold = fun_to_ident_folder(resolver);
+        let resolved_ast = resolver_fold.fold_item(renamed_ast).unwrap();
         let resolved_as_str = pprust::item_to_str(resolved_ast,
                                                   get_ident_interner());
         assert_eq!(resolved_as_str,~"fn a2() -> int { let b = 13; b }");
 
+        // try a double-rename, with pending_renames.
+        let a3_name = gensym("a3");
+        let ctxt2 = new_rename(ast::Ident::new(a_name),a2_name,EMPTY_CTXT);
+        let pending_renames = @mut ~[(ast::Ident::new(a_name),a2_name),
+                                     (ast::Ident{name:a_name,ctxt:ctxt2},a3_name)];
+        let double_renamed = renames_to_fold(pending_renames).fold_item(item_ast).unwrap();
+        let resolved_again = resolver_fold.fold_item(double_renamed).unwrap();
+        let double_renamed_as_str = pprust::item_to_str(resolved_again,
+                                                        get_ident_interner());
+        assert_eq!(double_renamed_as_str,~"fn a3() -> int { let b = 13; b }");
+
+    }
 
+    fn fake_print_crate(s: @pprust::ps, crate: &ast::Crate) {
+        pprust::print_mod(s, &crate.module, crate.attrs);
     }
 
-    // sigh... it looks like I have two different renaming mechanisms, now...
+    // "fn a() -> int { let b = 13; let c = b; b+c }" --> b & c should get new names, in the expr too.
+    // "macro_rules! f (($x:ident) => ($x + b)) fn a() -> int { let b = 13; f!(b)}" --> one should
+    //     be renamed, one should not.
+
+    fn expand_and_resolve_and_pretty_print (crate_str : @str) -> ~str {
+        let resolver = new_ident_resolver();
+        let resolver_fold = fun_to_ident_folder(resolver);
+        let (crate_ast,ps) = string_to_crate_and_sess(crate_str);
+        // the cfg argument actually does matter, here...
+        let expanded_ast = expand_crate(ps,~[],crate_ast);
+        // std::io::println(fmt!("expanded: %?\n",expanded_ast));
+        let resolved_ast = resolver_fold.fold_crate(expanded_ast);
+        pprust::to_str(&resolved_ast,fake_print_crate,get_ident_interner())
+    }
+
+    #[test]
+    fn automatic_renaming () {
+        let teststrs =
+            ~[@"fn a() -> int { let b = 13; let c = b; b+c }",
+              @"macro_rules! f (($x:ident) => ($x + b)) fn a() -> int { let b = 13; f!(b)}"];
+        for s in teststrs.iter() {
+            std::io::println(expand_and_resolve_and_pretty_print(*s));
+        }
+    }
 
     #[test]
     fn pat_idents(){
diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs
index 591b4b10bd3..8de597733ae 100644
--- a/src/libsyntax/parse/token.rs
+++ b/src/libsyntax/parse/token.rs
@@ -552,9 +552,9 @@ pub fn gensym_ident(str : &str) -> ast::Ident {
 // by using a gensym with a name that has a random number
 // at the end. So, the gensym guarantees the uniqueness,
 // and the int helps to avoid confusion.
-pub fn fresh_name(src_name : &str) -> Name {
+pub fn fresh_name(src_name : &ast::Ident) -> Name {
     let num = rand::rng().gen_uint_range(0,0xffff);
-   gensym(fmt!("%s_%u",src_name,num))
+    gensym(fmt!("%s_%u",ident_to_str(src_name),num))
 }
 
 /**
@@ -697,9 +697,5 @@ pub fn is_reserved_keyword(tok: &Token) -> bool {
 #[cfg(test)]
 mod test {
     use super::*;
-    #[test] fn t1() {
-        let a = fresh_name("ghi");
-        printfln!("interned name: %u,\ntextual name: %s\n",
-                  a, interner_get(a));
-    }
+
 }
diff --git a/src/libsyntax/util/parser_testing.rs b/src/libsyntax/util/parser_testing.rs
index ca1e53f7fcd..51fd5be71ab 100644
--- a/src/libsyntax/util/parser_testing.rs
+++ b/src/libsyntax/util/parser_testing.rs
@@ -40,12 +40,19 @@ fn with_error_checking_parse<T>(s: @str, f: &fn(&mut Parser) -> T) -> T {
     x
 }
 
+// parse a string, return a crate.
 pub fn string_to_crate (source_str : @str) -> @ast::Crate {
     do with_error_checking_parse(source_str) |p| {
         p.parse_crate_mod()
     }
 }
 
+// parse a string, return a crate and the ParseSess
+pub fn string_to_crate_and_sess (source_str : @str) -> (@ast::Crate,@mut ParseSess) {
+    let (p,ps) = string_to_parser_and_sess(source_str);
+    (p.parse_crate_mod(),ps)
+}
+
 // parse a string, return an expr
 pub fn string_to_expr (source_str : @str) -> @ast::Expr {
     do with_error_checking_parse(source_str) |p| {
@@ -60,14 +67,6 @@ pub fn string_to_item (source_str : @str) -> Option<@ast::item> {
     }
 }
 
-// parse a string, return an item and the ParseSess
-pub fn string_to_item_and_sess (source_str : @str) -> (Option<@ast::item>,@mut ParseSess) {
-    let (p,ps) = string_to_parser_and_sess(source_str);
-    let io = p.parse_item(~[]);
-    p.abort_if_errors();
-    (io,ps)
-}
-
 // parse a string, return a stmt
 pub fn string_to_stmt(source_str : @str) -> @ast::Stmt {
     do with_error_checking_parse(source_str) |p| {
-- 
cgit 1.4.1-3-g733a5


From 2f6498f7b4be2584d1164d68fc4d5ae485da2ea6 Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Thu, 6 Jun 2013 11:42:34 -0700
Subject: flip the switch on let renaming

---
 src/libsyntax/ext/expand.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index 81a47e0e485..716c43450d7 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -462,7 +462,7 @@ pub fn expand_stmt(extsbox: @mut SyntaxEnv,
                 }
             }
         }
-        _ => return orig(s, sp, fld)
+        _ => return expand_non_macro_stmt(*extsbox,s,sp,fld,orig)
     };
     if (pth.segments.len() > 1u) {
         cx.span_fatal(
-- 
cgit 1.4.1-3-g733a5


From 6c6d053b0177ed91071128aa5a2f57539e746987 Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Thu, 6 Jun 2013 11:21:02 -0700
Subject: renaming test cases

---
 src/libsyntax/ext/expand.rs | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index 716c43450d7..60eecd1b183 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -1452,10 +1452,6 @@ mod test {
         pprust::print_mod(s, &crate.module, crate.attrs);
     }
 
-    // "fn a() -> int { let b = 13; let c = b; b+c }" --> b & c should get new names, in the expr too.
-    // "macro_rules! f (($x:ident) => ($x + b)) fn a() -> int { let b = 13; f!(b)}" --> one should
-    //     be renamed, one should not.
-
     fn expand_and_resolve_and_pretty_print (crate_str : @str) -> ~str {
         let resolver = new_ident_resolver();
         let resolver_fold = fun_to_ident_folder(resolver);
@@ -1469,10 +1465,20 @@ mod test {
 
     #[test]
     fn automatic_renaming () {
+        // "fn a() -> int { let b = 13; let c = b; b+c }"
+        //    --> b & c should get new names, in the expr too.
+        // "macro_rules! f (($x:ident) => ($x + b)) fn a() -> int { let b = 13; f!(b)}"
+        //    --> one should be renamed, one should not.
+
         let teststrs =
-            ~[@"fn a() -> int { let b = 13; let c = b; b+c }",
-              @"macro_rules! f (($x:ident) => ($x + b)) fn a() -> int { let b = 13; f!(b)}"];
+            ~[// b & c should get new names throughout, in the expr too:
+                @"fn a() -> int { let b = 13; let c = b; b+c }",
+                // the use of b before the + should be renamed, the other one not:
+                @"macro_rules! f (($x:ident) => ($x + b)) fn a() -> int { let b = 13; f!(b)}",
+                // the b before the plus should not be renamed (requires marks)
+                @"macro_rules! f (($x:ident) => ({let b=9; ($x + b)})) fn a() -> int { f!(b)}"];
         for s in teststrs.iter() {
+            // we need regexps to test these!
             std::io::println(expand_and_resolve_and_pretty_print(*s));
         }
     }
-- 
cgit 1.4.1-3-g733a5


From d8276e75f048955b496b1ef9ca1b3e11ca5bc306 Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Thu, 6 Jun 2013 11:14:29 -0700
Subject: comments in ast.rs

---
 src/libsyntax/ast.rs | 5 +++++
 1 file changed, 5 insertions(+)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs
index ef5282551a1..8e974cae86e 100644
--- a/src/libsyntax/ast.rs
+++ b/src/libsyntax/ast.rs
@@ -568,6 +568,7 @@ pub enum token_tree {
     // a delimited sequence (the delimiters appear as the first
     // and last elements of the vector)
     tt_delim(@mut ~[token_tree]),
+
     // These only make sense for right-hand-sides of MBE macros:
 
     // a kleene-style repetition sequence with a span, a tt_forest,
@@ -646,6 +647,10 @@ pub enum matcher_ {
 
 pub type mac = Spanned<mac_>;
 
+// represents a macro invocation. The Path indicates which macro
+// is being invoked, and the vector of token-trees contains the source
+// of the macro invocation.
+// There's only one flavor, now, so this could presumably be simplified.
 #[deriving(Clone, Eq, Encodable, Decodable, IterBytes)]
 pub enum mac_ {
     mac_invoc_tt(Path,~[token_tree]),   // new macro-invocation
-- 
cgit 1.4.1-3-g733a5


From 431ede40dfd66c34656d3e58fabaa36e772e33c5 Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Tue, 28 May 2013 14:53:38 -0700
Subject: removed unneccessary SyntaxExpander structs

---
 src/libsyntax/ext/base.rs           | 21 +++++----------------
 src/libsyntax/ext/expand.rs         | 28 ++++++++++++----------------
 src/libsyntax/ext/tt/macro_rules.rs |  2 +-
 3 files changed, 18 insertions(+), 33 deletions(-)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/ext/base.rs b/src/libsyntax/ext/base.rs
index 50683358f87..c71cff08ba8 100644
--- a/src/libsyntax/ext/base.rs
+++ b/src/libsyntax/ext/base.rs
@@ -22,8 +22,7 @@ use std::hashmap::HashMap;
 
 // new-style macro! tt code:
 //
-//    SyntaxExpanderTT, SyntaxExpanderTTItem, MacResult,
-//    NormalTT, IdentTT
+//    MacResult, NormalTT, IdentTT
 //
 // also note that ast::mac used to have a bunch of extraneous cases and
 // is now probably a redundant AST node, can be merged with
@@ -40,21 +39,11 @@ pub type ItemDecorator = @fn(@ExtCtxt,
                              ~[@ast::item])
                           -> ~[@ast::item];
 
-pub struct SyntaxExpanderTT {
-    expander: SyntaxExpanderTTFun,
-    span: Option<Span>
-}
-
 pub type SyntaxExpanderTTFun = @fn(@ExtCtxt,
                                    Span,
                                    &[ast::token_tree])
                                 -> MacResult;
 
-pub struct SyntaxExpanderTTItem {
-    expander: SyntaxExpanderTTItemFun,
-    span: Option<Span>
-}
-
 pub type SyntaxExpanderTTItemFun = @fn(@ExtCtxt,
                                        Span,
                                        ast::Ident,
@@ -76,7 +65,7 @@ pub enum SyntaxExtension {
     ItemDecorator(ItemDecorator),
 
     // Token-tree expanders
-    NormalTT(SyntaxExpanderTT),
+    NormalTT(SyntaxExpanderTTFun, Option<Span>),
 
     // An IdentTT is a macro that has an
     // identifier in between the name of the
@@ -86,7 +75,7 @@ pub enum SyntaxExtension {
 
     // perhaps macro_rules! will lose its odd special identifier argument,
     // and this can go away also
-    IdentTT(SyntaxExpanderTTItem),
+    IdentTT(SyntaxExpanderTTItemFun, Option<Span>),
 }
 
 // The SyntaxEnv is the environment that's threaded through the expansion
@@ -121,11 +110,11 @@ type RenameList = ~[(ast::Ident,Name)];
 pub fn syntax_expander_table() -> SyntaxEnv {
     // utility function to simplify creating NormalTT syntax extensions
     fn builtin_normal_tt(f: SyntaxExpanderTTFun) -> @Transformer {
-        @SE(NormalTT(SyntaxExpanderTT{expander: f, span: None}))
+        @SE(NormalTT(f, None))
     }
     // utility function to simplify creating IdentTT syntax extensions
     fn builtin_item_tt(f: SyntaxExpanderTTItemFun) -> @Transformer {
-        @SE(IdentTT(SyntaxExpanderTTItem{expander: f, span: None}))
+        @SE(IdentTT(f, None))
     }
     let mut syntax_expanders = HashMap::new();
     // NB identifier starts with space, and can't conflict with legal idents
diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index 60eecd1b183..fbaeab03e8a 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -58,19 +58,16 @@ pub fn expand_expr(extsbox: @mut SyntaxEnv,
                                 pth.span,
                                 fmt!("macro undefined: '%s'", extnamestr))
                         }
-                        Some(@SE(NormalTT(SyntaxExpanderTT{
-                            expander: exp,
-                            span: exp_sp
-                        }))) => {
+                        Some(@SE(NormalTT(expandfun, exp_span))) => {
                             cx.bt_push(ExpnInfo {
                                 call_site: s,
                                 callee: NameAndSpan {
                                     name: extnamestr,
-                                    span: exp_sp,
+                                    span: exp_span,
                                 },
                             });
 
-                            let expanded = match exp(cx, mac.span, *tts) {
+                            let expanded = match expandfun(cx, mac.span, *tts) {
                                 MRExpr(e) => e,
                                 MRAny(expr_maker,_,_) => expr_maker(),
                                 _ => {
@@ -379,7 +376,7 @@ pub fn expand_item_mac(extsbox: @mut SyntaxEnv,
         None => cx.span_fatal(pth.span,
                               fmt!("macro undefined: '%s!'", extnamestr)),
 
-        Some(@SE(NormalTT(ref expand))) => {
+        Some(@SE(NormalTT(expander, span))) => {
             if it.ident != parse::token::special_idents::invalid {
                 cx.span_fatal(pth.span,
                               fmt!("macro %s! expects no ident argument, \
@@ -390,12 +387,12 @@ pub fn expand_item_mac(extsbox: @mut SyntaxEnv,
                 call_site: it.span,
                 callee: NameAndSpan {
                     name: extnamestr,
-                    span: expand.span
+                    span: span
                 }
             });
-            ((*expand).expander)(cx, it.span, tts)
+             expander(cx, it.span, tts)
         }
-        Some(@SE(IdentTT(ref expand))) => {
+        Some(@SE(IdentTT(expander, span))) => {
             if it.ident == parse::token::special_idents::invalid {
                 cx.span_fatal(pth.span,
                               fmt!("macro %s! expects an ident argument",
@@ -405,10 +402,10 @@ pub fn expand_item_mac(extsbox: @mut SyntaxEnv,
                 call_site: it.span,
                 callee: NameAndSpan {
                     name: extnamestr,
-                    span: expand.span
+                    span: span
                 }
             });
-            ((*expand).expander)(cx, it.span, it.ident, tts)
+            expander(cx, it.span, it.ident, tts)
         }
         _ => cx.span_fatal(
             it.span, fmt!("%s! is not legal in item position", extnamestr))
@@ -476,13 +473,12 @@ pub fn expand_stmt(extsbox: @mut SyntaxEnv,
         None =>
             cx.span_fatal(pth.span, fmt!("macro undefined: '%s'", extnamestr)),
 
-        Some(@SE(NormalTT(
-            SyntaxExpanderTT{expander: exp, span: exp_sp}))) => {
+        Some(@SE(NormalTT(expandfun, exp_span))) => {
             cx.bt_push(ExpnInfo {
                 call_site: sp,
-                callee: NameAndSpan { name: extnamestr, span: exp_sp }
+                callee: NameAndSpan { name: extnamestr, span: exp_span }
             });
-            let expanded = match exp(cx, mac.span, tts) {
+            let expanded = match expandfun(cx, mac.span, tts) {
                 MRExpr(e) =>
                     @codemap::Spanned { node: StmtExpr(e, cx.next_id()),
                                     span: e.span},
diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs
index 732ae2ccb96..50eb03fc96e 100644
--- a/src/libsyntax/ext/tt/macro_rules.rs
+++ b/src/libsyntax/ext/tt/macro_rules.rs
@@ -149,6 +149,6 @@ pub fn add_new_extension(cx: @ExtCtxt,
 
     return MRDef(MacroDef{
         name: ident_to_str(&name),
-        ext: NormalTT(base::SyntaxExpanderTT{expander: exp, span: Some(sp)})
+        ext: NormalTT(exp, Some(sp))
     });
 }
-- 
cgit 1.4.1-3-g733a5


From 93337f0daab1958bd7f7614cf54d3f69f2ab64ce Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Tue, 28 May 2013 14:55:50 -0700
Subject: separate ItemDecorator from ItemDecorator

---
 src/libsyntax/ext/base.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/ext/base.rs b/src/libsyntax/ext/base.rs
index c71cff08ba8..ac3fc717500 100644
--- a/src/libsyntax/ext/base.rs
+++ b/src/libsyntax/ext/base.rs
@@ -33,7 +33,7 @@ pub struct MacroDef {
     ext: SyntaxExtension
 }
 
-pub type ItemDecorator = @fn(@ExtCtxt,
+pub type ItemDecoratorFun = @fn(@ExtCtxt,
                              Span,
                              @ast::MetaItem,
                              ~[@ast::item])
@@ -62,7 +62,7 @@ pub enum MacResult {
 pub enum SyntaxExtension {
 
     // #[auto_encode] and such
-    ItemDecorator(ItemDecorator),
+    ItemDecorator(ItemDecoratorFun),
 
     // Token-tree expanders
     NormalTT(SyntaxExpanderTTFun, Option<Span>),
-- 
cgit 1.4.1-3-g733a5


From b7c0512b27d287cf0657ae4b195d1ca41623c944 Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Tue, 28 May 2013 15:13:57 -0700
Subject: refactor so tt_fold only requires an ident->ident fn

---
 src/libsyntax/fold.rs | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/fold.rs b/src/libsyntax/fold.rs
index 7aa0f3abe87..750ec0be984 100644
--- a/src/libsyntax/fold.rs
+++ b/src/libsyntax/fold.rs
@@ -118,35 +118,37 @@ fn fold_mac_(m: &mac, fld: @ast_fold) -> mac {
         node: match m.node {
             mac_invoc_tt(ref p,ref tts) =>
             mac_invoc_tt(fld.fold_path(p),
-                         fold_tts(*tts,fld))
+                         fold_tts(*tts,|id|{fld.fold_ident(id)}))
         },
         span: fld.new_span(m.span)
     }
 }
 
-fn fold_tts(tts : &[token_tree], fld: @ast_fold) -> ~[token_tree] {
+// build a new vector of tts by appling the given function to
+// all of the identifiers in the token trees.
+pub fn fold_tts(tts : &[token_tree], f: @fn(ident)->ident) -> ~[token_tree] {
     do tts.map |tt| {
         match *tt {
             tt_tok(span, ref tok) =>
-            tt_tok(span,maybe_fold_ident(tok,fld)),
+            tt_tok(span,maybe_fold_ident(tok,f)),
             tt_delim(ref tts) =>
-            tt_delim(@mut fold_tts(**tts, fld)),
+            tt_delim(@mut fold_tts(**tts, f)),
             tt_seq(span, ref pattern, ref sep, is_optional) =>
             tt_seq(span,
-                   @mut fold_tts(**pattern, fld),
-                   sep.map(|tok|maybe_fold_ident(tok,fld)),
+                   @mut fold_tts(**pattern, f),
+                   sep.map(|tok|maybe_fold_ident(tok,f)),
                    is_optional),
             tt_nonterminal(sp,ref ident) =>
-            tt_nonterminal(sp,fld.fold_ident(*ident))
+            tt_nonterminal(sp,f(*ident))
         }
     }
 }
 
 // apply ident folder if it's an ident, otherwise leave it alone
-fn maybe_fold_ident(t: &token::Token, fld: @ast_fold) -> token::Token {
+fn maybe_fold_ident(t : &token::Token, f: @fn(ident)->ident) -> token::Token {
     match *t {
         token::IDENT(id,followed_by_colons) =>
-        token::IDENT(fld.fold_ident(id),followed_by_colons),
+        token::IDENT(f(id),followed_by_colons),
         _ => (*t).clone()
     }
 }
-- 
cgit 1.4.1-3-g733a5


From 91d3c364303c3a057feadd40adef0880531e08cc Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Tue, 25 Jun 2013 11:43:52 -0700
Subject: adding test case to check marking/unmarking

---
 src/libsyntax/ext/expand.rs  | 26 +++++++++++++++++---------
 src/libsyntax/fold.rs        |  4 ++--
 src/libsyntax/parse/token.rs |  7 ++++++-
 3 files changed, 25 insertions(+), 12 deletions(-)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index fbaeab03e8a..25edcf63faa 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -23,7 +23,7 @@ use opt_vec;
 use parse;
 use parse::{parse_item_from_source_str};
 use parse::token;
-use parse::token::{fresh_name, ident_to_str, intern};
+use parse::token::{fresh_mark, fresh_name, ident_to_str, intern};
 use visit;
 use visit::Visitor;
 
@@ -67,7 +67,9 @@ pub fn expand_expr(extsbox: @mut SyntaxEnv,
                                 },
                             });
 
-                            let expanded = match expandfun(cx, mac.span, *tts) {
+                            let fm = fresh_mark();
+                            let marked_tts = mark_tts(*tts,fm);
+                            let expanded = match expandfun(cx, mac.span, marked_tts) {
                                 MRExpr(e) => e,
                                 MRAny(expr_maker,_,_) => expr_maker(),
                                 _ => {
@@ -259,6 +261,12 @@ pub fn expand_expr(extsbox: @mut SyntaxEnv,
     }
 }
 
+// apply a fresh mark to the given token trees. Used prior to expansion of a macro.
+fn mark_tts(tts : &[token_tree], m : Mrk) -> ~[token_tree] {
+    fold_tts(tts,new_ident_marker(m))
+}
+
+
 // This is a secondary mechanism for invoking syntax extensions on items:
 // "decorator" attributes, such as #[auto_encode]. These are invoked by an
 // attribute prefixing an item, and are interpreted by feeding the item
@@ -1285,7 +1293,7 @@ pub fn new_ident_renamer(from: ast::Ident,
 
 
 // update the ctxts in a path to get a mark node
-pub fn new_ident_marker(mark: uint) ->
+pub fn new_ident_marker(mark: Mrk) ->
     @fn(ast::Ident)->ast::Ident {
     |id : ast::Ident|
     ast::Ident{
@@ -1461,18 +1469,18 @@ mod test {
 
     #[test]
     fn automatic_renaming () {
-        // "fn a() -> int { let b = 13; let c = b; b+c }"
-        //    --> b & c should get new names, in the expr too.
-        // "macro_rules! f (($x:ident) => ($x + b)) fn a() -> int { let b = 13; f!(b)}"
-        //    --> one should be renamed, one should not.
-
         let teststrs =
             ~[// b & c should get new names throughout, in the expr too:
                 @"fn a() -> int { let b = 13; let c = b; b+c }",
                 // the use of b before the + should be renamed, the other one not:
                 @"macro_rules! f (($x:ident) => ($x + b)) fn a() -> int { let b = 13; f!(b)}",
                 // the b before the plus should not be renamed (requires marks)
-                @"macro_rules! f (($x:ident) => ({let b=9; ($x + b)})) fn a() -> int { f!(b)}"];
+                @"macro_rules! f (($x:ident) => ({let b=9; ($x + b)})) fn a() -> int { f!(b)}",
+                // the z flows into and out of two macros (g & f) along one path, and one (just g) along the
+                // other, so the result of the whole thing should be "let z_123 = 3; z_123"
+                @"macro_rules! g (($x:ident) => ({macro_rules! f(($y:ident)=>({let $y=3;$x}));f!($x)}))
+                   fn a(){g!(z)}"
+            ];
         for s in teststrs.iter() {
             // we need regexps to test these!
             std::io::println(expand_and_resolve_and_pretty_print(*s));
diff --git a/src/libsyntax/fold.rs b/src/libsyntax/fold.rs
index 750ec0be984..52c148e7ba2 100644
--- a/src/libsyntax/fold.rs
+++ b/src/libsyntax/fold.rs
@@ -126,7 +126,7 @@ fn fold_mac_(m: &mac, fld: @ast_fold) -> mac {
 
 // build a new vector of tts by appling the given function to
 // all of the identifiers in the token trees.
-pub fn fold_tts(tts : &[token_tree], f: @fn(ident)->ident) -> ~[token_tree] {
+pub fn fold_tts(tts : &[token_tree], f: @fn(Ident)->Ident) -> ~[token_tree] {
     do tts.map |tt| {
         match *tt {
             tt_tok(span, ref tok) =>
@@ -145,7 +145,7 @@ pub fn fold_tts(tts : &[token_tree], f: @fn(ident)->ident) -> ~[token_tree] {
 }
 
 // apply ident folder if it's an ident, otherwise leave it alone
-fn maybe_fold_ident(t : &token::Token, f: @fn(ident)->ident) -> token::Token {
+fn maybe_fold_ident(t : &token::Token, f: @fn(Ident)->Ident) -> token::Token {
     match *t {
         token::IDENT(id,followed_by_colons) =>
         token::IDENT(f(id),followed_by_colons),
diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs
index 8de597733ae..66f121727af 100644
--- a/src/libsyntax/parse/token.rs
+++ b/src/libsyntax/parse/token.rs
@@ -9,7 +9,7 @@
 // except according to those terms.
 
 use ast;
-use ast::Name;
+use ast::{Name, Mrk};
 use ast_util;
 use parse::token;
 use util::interner::StrInterner;
@@ -557,6 +557,11 @@ pub fn fresh_name(src_name : &ast::Ident) -> Name {
     gensym(fmt!("%s_%u",ident_to_str(src_name),num))
 }
 
+// create a fresh mark.
+pub fn fresh_mark() -> Mrk {
+    gensym("mark")
+}
+
 /**
  * All the valid words that have meaning in the Rust language.
  *
-- 
cgit 1.4.1-3-g733a5


From 98a6cbdba320b106690c01a354f91019020a82d0 Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Wed, 29 May 2013 16:21:04 -0700
Subject: comments only

---
 src/libsyntax/ast.rs                | 1 +
 src/libsyntax/ast_util.rs           | 2 ++
 src/libsyntax/ext/expand.rs         | 2 ++
 src/libsyntax/ext/tt/macro_rules.rs | 6 ++++++
 4 files changed, 11 insertions(+)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs
index 8e974cae86e..cf8fe905d1a 100644
--- a/src/libsyntax/ast.rs
+++ b/src/libsyntax/ast.rs
@@ -642,6 +642,7 @@ pub enum matcher_ {
     // lo, hi position-in-match-array used:
     match_seq(~[matcher], Option<::parse::token::Token>, bool, uint, uint),
     // parse a Rust NT: name to bind, name of NT, position in match array:
+    // NOTE: 'name of NT' shouldnt really be represented as an ident, should it?
     match_nonterminal(Ident, Ident, uint)
 }
 
diff --git a/src/libsyntax/ast_util.rs b/src/libsyntax/ast_util.rs
index aeca145ea18..4c0ad816afb 100644
--- a/src/libsyntax/ast_util.rs
+++ b/src/libsyntax/ast_util.rs
@@ -28,6 +28,8 @@ pub fn path_name_i(idents: &[Ident]) -> ~str {
     idents.map(|i| token::interner_get(i.name)).connect("::")
 }
 
+// totally scary function: ignores all but the last element, should have
+// a different name
 pub fn path_to_ident(path: &Path) -> Ident {
     path.segments.last().identifier
 }
diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index 25edcf63faa..039ca36b556 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -1480,6 +1480,8 @@ mod test {
                 // other, so the result of the whole thing should be "let z_123 = 3; z_123"
                 @"macro_rules! g (($x:ident) => ({macro_rules! f(($y:ident)=>({let $y=3;$x}));f!($x)}))
                    fn a(){g!(z)}"
+                // create a really evil test case where a $x appears inside a binding of $x but *shouldnt*
+                // bind because it was inserted by a different macro....
             ];
         for s in teststrs.iter() {
             // we need regexps to test these!
diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs
index 50eb03fc96e..2145e4297e7 100644
--- a/src/libsyntax/ext/tt/macro_rules.rs
+++ b/src/libsyntax/ext/tt/macro_rules.rs
@@ -23,11 +23,15 @@ use parse::token::{get_ident_interner, special_idents, gensym_ident, ident_to_st
 use parse::token::{FAT_ARROW, SEMI, nt_matchers, nt_tt};
 use print;
 
+// this procedure performs the expansion of the
+// macro_rules! macro. It parses the RHS and adds
+// an extension to the current context.
 pub fn add_new_extension(cx: @ExtCtxt,
                          sp: Span,
                          name: Ident,
                          arg: ~[ast::token_tree])
                       -> base::MacResult {
+    // Wrap a matcher_ in a spanned to produce a matcher.
     // these spans won't matter, anyways
     fn ms(m: matcher_) -> matcher {
         Spanned {
@@ -39,11 +43,13 @@ pub fn add_new_extension(cx: @ExtCtxt,
     let lhs_nm =  gensym_ident("lhs");
     let rhs_nm =  gensym_ident("rhs");
 
+    // The pattern that macro_rules matches.
     // The grammar for macro_rules! is:
     // $( $lhs:mtcs => $rhs:tt );+
     // ...quasiquoting this would be nice.
     let argument_gram = ~[
         ms(match_seq(~[
+            // NOTE : probably just use an enum for the NT_name ?
             ms(match_nonterminal(lhs_nm, special_idents::matchers, 0u)),
             ms(match_tok(FAT_ARROW)),
             ms(match_nonterminal(rhs_nm, special_idents::tt, 1u)),
-- 
cgit 1.4.1-3-g733a5


From 9de40dfc870b355a737465d5f75f7efba8292728 Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Thu, 6 Jun 2013 18:09:31 -0700
Subject: remove FIXME #2888, now bug is fixed

---
 src/libsyntax/ext/expand.rs | 22 +++++++++++-----------
 src/libsyntax/fold.rs       |  5 ++---
 2 files changed, 13 insertions(+), 14 deletions(-)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index 039ca36b556..196e8644cd3 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -1456,16 +1456,14 @@ mod test {
         pprust::print_mod(s, &crate.module, crate.attrs);
     }
 
-    fn expand_and_resolve_and_pretty_print (crate_str : @str) -> ~str {
-        let resolver = new_ident_resolver();
-        let resolver_fold = fun_to_ident_folder(resolver);
-        let (crate_ast,ps) = string_to_crate_and_sess(crate_str);
+    //fn expand_and_resolve_and_pretty_print (crate_str : @str) -> ~str {
+        //let (crate_ast,ps) = string_to_crate_and_sess(crate_str);
         // the cfg argument actually does matter, here...
-        let expanded_ast = expand_crate(ps,~[],crate_ast);
+        //let expanded_ast = expand_crate(ps,~[],crate_ast);
         // std::io::println(fmt!("expanded: %?\n",expanded_ast));
-        let resolved_ast = resolver_fold.fold_crate(expanded_ast);
-        pprust::to_str(&resolved_ast,fake_print_crate,get_ident_interner())
-    }
+        //let resolved_ast = mtwt_resolve_crate(expanded_ast);
+        //pprust::to_str(&resolved_ast,fake_print_crate,get_ident_interner())
+    //}
 
     #[test]
     fn automatic_renaming () {
@@ -1476,16 +1474,18 @@ mod test {
                 @"macro_rules! f (($x:ident) => ($x + b)) fn a() -> int { let b = 13; f!(b)}",
                 // the b before the plus should not be renamed (requires marks)
                 @"macro_rules! f (($x:ident) => ({let b=9; ($x + b)})) fn a() -> int { f!(b)}",
+                // FIXME #6994: the next string exposes the bug referred to in issue 6994, so I'm
+                // commenting it out.
                 // the z flows into and out of two macros (g & f) along one path, and one (just g) along the
                 // other, so the result of the whole thing should be "let z_123 = 3; z_123"
-                @"macro_rules! g (($x:ident) => ({macro_rules! f(($y:ident)=>({let $y=3;$x}));f!($x)}))
-                   fn a(){g!(z)}"
+                //@"macro_rules! g (($x:ident) => ({macro_rules! f(($y:ident)=>({let $y=3;$x}));f!($x)}))
+                //   fn a(){g!(z)}"
                 // create a really evil test case where a $x appears inside a binding of $x but *shouldnt*
                 // bind because it was inserted by a different macro....
             ];
         for s in teststrs.iter() {
             // we need regexps to test these!
-            std::io::println(expand_and_resolve_and_pretty_print(*s));
+            //std::io::println(expand_and_resolve_and_pretty_print(*s));
         }
     }
 
diff --git a/src/libsyntax/fold.rs b/src/libsyntax/fold.rs
index 52c148e7ba2..5a099c96fa1 100644
--- a/src/libsyntax/fold.rs
+++ b/src/libsyntax/fold.rs
@@ -325,9 +325,8 @@ pub fn noop_fold_item_underscore(i: &item_, fld: @ast_fold) -> item_ {
             )
         }
         item_mac(ref m) => {
-            // FIXME #2888: we might actually want to do something here.
-            // ... okay, we're doing something. It would probably be nicer
-            // to add something to the ast_fold trait, but I'll defer
+            // It would probably be nicer
+            // to expose this in the ast_fold trait, but I'll defer
             // that work.
             item_mac(fold_mac_(m,fld))
         }
-- 
cgit 1.4.1-3-g733a5


From 7bf75adfd904642022a048c34440ec4b5dfd567d Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Fri, 7 Jun 2013 10:39:59 -0700
Subject: added FIXME comment

---
 src/libsyntax/ast.rs | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs
index cf8fe905d1a..c7e197ade09 100644
--- a/src/libsyntax/ast.rs
+++ b/src/libsyntax/ast.rs
@@ -20,6 +20,10 @@ use std::option::Option;
 use std::to_str::ToStr;
 use extra::serialize::{Encodable, Decodable, Encoder, Decoder};
 
+
+// FIXME #6993: in librustc, uses of "ident" should be replaced
+// by just "Name".
+
 // an identifier contains a Name (index into the interner
 // table) and a SyntaxContext to track renaming and
 // macro expansion per Flatt et al., "Macros
-- 
cgit 1.4.1-3-g733a5


From d1c01734f3a34958333fefb2fe91cea5e753ddd7 Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Fri, 7 Jun 2013 10:41:18 -0700
Subject: drop back to a simple gensym approach for fresh-name.

this is necessary so that the new idents are connected to the original strings.
this is important both for error messages, and so that top-level refs get connected
to the right things.
---
 src/libsyntax/parse/token.rs | 15 +++------------
 1 file changed, 3 insertions(+), 12 deletions(-)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs
index 66f121727af..6cdde3708a6 100644
--- a/src/libsyntax/parse/token.rs
+++ b/src/libsyntax/parse/token.rs
@@ -543,18 +543,9 @@ pub fn gensym_ident(str : &str) -> ast::Ident {
     ast::Ident::new(gensym(str))
 }
 
-
-// create a fresh name. In principle, this is just a
-// gensym, but for debugging purposes, you'd like the
-// resulting name to have a suggestive stringify, without
-// paying the cost of guaranteeing that the name is
-// truly unique.  I'm going to try to strike a balance
-// by using a gensym with a name that has a random number
-// at the end. So, the gensym guarantees the uniqueness,
-// and the int helps to avoid confusion.
-pub fn fresh_name(src_name : &ast::Ident) -> Name {
-    let num = rand::rng().gen_uint_range(0,0xffff);
-    gensym(fmt!("%s_%u",ident_to_str(src_name),num))
+// create a fresh name that maps to the same string as the old one.
+pub fn fresh_name(src : &ast::Ident) -> Name {
+    gensym(ident_to_str(src))
 }
 
 // create a fresh mark.
-- 
cgit 1.4.1-3-g733a5


From 3965725d51508e185f03bda0ce883f56c1d980e5 Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Fri, 7 Jun 2013 10:41:38 -0700
Subject: comments

---
 src/libsyntax/ast.rs        | 1 +
 src/libsyntax/ext/expand.rs | 3 +++
 2 files changed, 4 insertions(+)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs
index c7e197ade09..b3fb8859f65 100644
--- a/src/libsyntax/ast.rs
+++ b/src/libsyntax/ast.rs
@@ -98,6 +98,7 @@ pub enum SyntaxContext_ {
     // in the "from" slot. In essence, they're all
     // pointers to a single "rename" event node.
     Rename (Ident,Name,SyntaxContext),
+    // actually, IllegalCtxt may not be necessary.
     IllegalCtxt
 }
 
diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index 196e8644cd3..a75238d269b 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -553,6 +553,8 @@ fn expand_non_macro_stmt (exts: SyntaxEnv,
             let expanded_pat = fld.fold_pat(pat);
             // find the pat_idents in the pattern:
             // oh dear heaven... this is going to include the enum names, as well....
+            // ... but that should be okay, as long as the new names are gensyms
+            // for the old ones.
             let idents = @mut ~[];
             let name_finder = new_name_finder(idents);
             name_finder.visit_pat(expanded_pat,());
@@ -1467,6 +1469,7 @@ mod test {
 
     #[test]
     fn automatic_renaming () {
+        // need some other way to test these...
         let teststrs =
             ~[// b & c should get new names throughout, in the expr too:
                 @"fn a() -> int { let b = 13; let c = b; b+c }",
-- 
cgit 1.4.1-3-g733a5


From a666ddc135d472bcc6fcd82b71e6e393e77756cf Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Wed, 3 Jul 2013 11:34:01 -0700
Subject: make comparison of special_idents non-hygienic

---
 src/libsyntax/ext/expand.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index a75238d269b..dd5d0c815b5 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -385,7 +385,7 @@ pub fn expand_item_mac(extsbox: @mut SyntaxEnv,
                               fmt!("macro undefined: '%s!'", extnamestr)),
 
         Some(@SE(NormalTT(expander, span))) => {
-            if it.ident != parse::token::special_idents::invalid {
+            if it.ident.name != parse::token::special_idents::invalid.name {
                 cx.span_fatal(pth.span,
                               fmt!("macro %s! expects no ident argument, \
                                     given '%s'", extnamestr,
@@ -401,7 +401,7 @@ pub fn expand_item_mac(extsbox: @mut SyntaxEnv,
              expander(cx, it.span, tts)
         }
         Some(@SE(IdentTT(expander, span))) => {
-            if it.ident == parse::token::special_idents::invalid {
+            if it.ident.name == parse::token::special_idents::invalid.name {
                 cx.span_fatal(pth.span,
                               fmt!("macro %s! expects an ident argument",
                                    extnamestr));
-- 
cgit 1.4.1-3-g733a5


From f5ab8671198e37e15ab6c5512417ceadf6c1290c Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Fri, 7 Jun 2013 12:26:34 -0700
Subject: use empty_ctxt to simplify downstream

---
 src/libsyntax/ext/expand.rs | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index dd5d0c815b5..fa4b7f9ecd1 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -8,7 +8,8 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use ast::{Block, Crate, NodeId, DeclLocal, Expr_, ExprMac, Local, Ident, mac_invoc_tt};
+use ast::{Block, Crate, NodeId, DeclLocal, EMPTY_CTXT, Expr_, ExprMac};
+use ast::{Local, Ident, mac_invoc_tt};
 use ast::{item_mac, Mrk, Stmt_, StmtDecl, StmtMac, StmtExpr, StmtSemi};
 use ast::{ILLEGAL_CTXT, SCTable, token_tree};
 use ast;
@@ -1306,12 +1307,14 @@ pub fn new_ident_marker(mark: Mrk) ->
 
 // perform resolution (in the MTWT sense) on all of the
 // idents in the tree. This is the final step in expansion.
+// FIXME #6993: this function could go away, along with
+// the separate mtwt_resolution pass
 pub fn new_ident_resolver() ->
     @fn(ast::Ident)->ast::Ident {
     |id : ast::Ident|
     ast::Ident {
         name : mtwt_resolve(id),
-        ctxt : ILLEGAL_CTXT
+        ctxt : EMPTY_CTXT
     }
 }
 
-- 
cgit 1.4.1-3-g733a5


From 7a9af098f0a01cf0d0f320def7c495d1fa859e62 Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Fri, 7 Jun 2013 12:28:03 -0700
Subject: test case work

---
 src/libsyntax/ext/expand.rs | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index fa4b7f9ecd1..6d5811f1afb 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -1461,12 +1461,15 @@ mod test {
         pprust::print_mod(s, &crate.module, crate.attrs);
     }
 
-    //fn expand_and_resolve_and_pretty_print (crate_str : @str) -> ~str {
+    //fn expand_and_resolve(crate_str: @str) -> ast::crate {
         //let (crate_ast,ps) = string_to_crate_and_sess(crate_str);
         // the cfg argument actually does matter, here...
         //let expanded_ast = expand_crate(ps,~[],crate_ast);
         // std::io::println(fmt!("expanded: %?\n",expanded_ast));
-        //let resolved_ast = mtwt_resolve_crate(expanded_ast);
+        //mtwt_resolve_crate(expanded_ast)
+    //}
+    //fn expand_and_resolve_and_pretty_print (crate_str : @str) -> ~str {
+        //let resolved_ast = expand_and_resolve(crate_str);
         //pprust::to_str(&resolved_ast,fake_print_crate,get_ident_interner())
     //}
 
@@ -1503,4 +1506,9 @@ mod test {
         pat_idents.visit_pat(pat, ());
         assert_eq!(idents, @mut strs_to_idents(~["a","c","b","d"]));
     }
+
+/*    #[test]
+    fn debugging(){
+        io::println(fmt!("%?",expand_and_resolve(@~"fn main () {    let x : int = 13;}")))
+    }*/
 }
-- 
cgit 1.4.1-3-g733a5


From 9071ac60b621020c08bc74cd15c363a7780a2230 Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Fri, 7 Jun 2013 14:53:53 -0700
Subject: re-add debug version

---
 src/libsyntax/parse/token.rs | 5 +++++
 1 file changed, 5 insertions(+)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs
index 6cdde3708a6..132ad95e639 100644
--- a/src/libsyntax/parse/token.rs
+++ b/src/libsyntax/parse/token.rs
@@ -546,6 +546,11 @@ pub fn gensym_ident(str : &str) -> ast::Ident {
 // create a fresh name that maps to the same string as the old one.
 pub fn fresh_name(src : &ast::Ident) -> Name {
     gensym(ident_to_str(src))
+    // following: debug version. Could work in final except that it's incompatible with
+    // good error messages and uses of struct names in ambiguous could-be-binding
+    // locations.
+    /*let num = rand::rng().gen_uint_range(0,0xffff);
+    gensym(fmt!("%s_%u",ident_to_str(src),num))*/
 }
 
 // create a fresh mark.
-- 
cgit 1.4.1-3-g733a5


From 1f8e8564308917efe94663d3577bfe35948b624a Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Fri, 7 Jun 2013 14:54:48 -0700
Subject: re-add lost call to expand_block_elts

---
 src/libsyntax/ext/expand.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index 6d5811f1afb..4ffaf5df7b7 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -772,7 +772,8 @@ pub fn expand_block(extsbox: @mut SyntaxEnv,
                     orig: @fn(&Block, @ast_fold) -> Block)
                  -> Block {
     // see note below about treatment of exts table
-    with_exts_frame!(extsbox,false,orig(blk,fld))
+    with_exts_frame!(extsbox,false,
+                     expand_block_elts(*extsbox, blk, fld))
 }
 
 
-- 
cgit 1.4.1-3-g733a5


From 62fee04ae9f82c1d1fe03c68735ee2b9f69f9b8e Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Fri, 7 Jun 2013 15:01:28 -0700
Subject: test case work

---
 src/libsyntax/ext/expand.rs | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index 4ffaf5df7b7..ca82fd09a6b 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -1480,6 +1480,8 @@ mod test {
         let teststrs =
             ~[// b & c should get new names throughout, in the expr too:
                 @"fn a() -> int { let b = 13; let c = b; b+c }",
+                // both x's should be renamed (how is this causing a bug?)
+                @"fn main () {let x : int = 13;x;}",
                 // the use of b before the + should be renamed, the other one not:
                 @"macro_rules! f (($x:ident) => ($x + b)) fn a() -> int { let b = 13; f!(b)}",
                 // the b before the plus should not be renamed (requires marks)
@@ -1508,8 +1510,4 @@ mod test {
         assert_eq!(idents, @mut strs_to_idents(~["a","c","b","d"]));
     }
 
-/*    #[test]
-    fn debugging(){
-        io::println(fmt!("%?",expand_and_resolve(@~"fn main () {    let x : int = 13;}")))
-    }*/
 }
-- 
cgit 1.4.1-3-g733a5


From 015ba31c02b912220a9b7323e1a6ae3bad4ab927 Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Tue, 9 Jul 2013 16:00:41 -0700
Subject: test case support fns, remove debugging test case

---
 src/libsyntax/ext/expand.rs | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index ca82fd09a6b..baa4e616b4f 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -1462,10 +1462,14 @@ mod test {
         pprust::print_mod(s, &crate.module, crate.attrs);
     }
 
-    //fn expand_and_resolve(crate_str: @str) -> ast::crate {
-        //let (crate_ast,ps) = string_to_crate_and_sess(crate_str);
+    fn expand_crate_str(crate_str: @str) -> @ast::Crate {
+        let (crate_ast,ps) = string_to_crate_and_sess(crate_str);
         // the cfg argument actually does matter, here...
-        //let expanded_ast = expand_crate(ps,~[],crate_ast);
+        expand_crate(ps,~[],crate_ast)
+    }
+
+    //fn expand_and_resolve(crate_str: @str) -> ast::crate {
+        //let expanded_ast = expand_crate_str(crate_str);
         // std::io::println(fmt!("expanded: %?\n",expanded_ast));
         //mtwt_resolve_crate(expanded_ast)
     //}
-- 
cgit 1.4.1-3-g733a5


From d876a721d2efa7cbf7b0224c396a5da9079a8d5d Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Fri, 7 Jun 2013 17:06:57 -0700
Subject: new test that uncovers bug in fold

---
 src/libsyntax/fold.rs | 11 +++++++++++
 1 file changed, 11 insertions(+)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/fold.rs b/src/libsyntax/fold.rs
index 5a099c96fa1..f8671623992 100644
--- a/src/libsyntax/fold.rs
+++ b/src/libsyntax/fold.rs
@@ -1019,4 +1019,15 @@ mod test {
                                     token::get_ident_interner()),
                      ~"zz!zz((zz$zz:zz$(zz $zz:zz)zz+=>(zz$(zz$zz$zz)+)))");
     }
+
+    // and in cast expressions... this appears to be an existing bug.
+    #[test] fn ident_transformation_in_types () {
+        let zz_fold = fun_to_ident_folder(to_zz());
+        let ast = string_to_crate(@"fn a() {let z = 13 as int;}");
+        assert_pred!(matches_codepattern,
+                     "matches_codepattern",
+                     pprust::to_str(&zz_fold.fold_crate(ast),fake_print_crate,
+                                    token::get_ident_interner()),
+                     ~"fn zz(){let zz=13 as zz;}");
+    }
 }
-- 
cgit 1.4.1-3-g733a5


From cd4e637ae4113d4c49a2e5c533b5bf48fbf7cc6e Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Fri, 7 Jun 2013 17:14:06 -0700
Subject: fixed bug in fold's traversal of cast exprs

---
 src/libsyntax/fold.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/fold.rs b/src/libsyntax/fold.rs
index f8671623992..85519bd2671 100644
--- a/src/libsyntax/fold.rs
+++ b/src/libsyntax/fold.rs
@@ -545,7 +545,7 @@ pub fn noop_fold_expr(e: &Expr_, fld: @ast_fold) -> Expr_ {
         ExprDoBody(f) => ExprDoBody(fld.fold_expr(f)),
         ExprLit(_) => (*e).clone(),
         ExprCast(expr, ref ty) => {
-            ExprCast(fld.fold_expr(expr), (*ty).clone())
+            ExprCast(fld.fold_expr(expr), fld.fold_ty(ty))
         }
         ExprAddrOf(m, ohs) => ExprAddrOf(m, fld.fold_expr(ohs)),
         ExprIf(cond, ref tr, fl) => {
-- 
cgit 1.4.1-3-g733a5


From 9d33001a90319fc242dcf43ec3c7e1fa1c11d847 Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Tue, 25 Jun 2013 16:48:03 -0700
Subject: added test for ptr_eq on fresh_name-generated idents

---
 src/libsyntax/parse/token.rs | 33 ++++++++++++++++++++++++++++++++-
 1 file changed, 32 insertions(+), 1 deletion(-)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs
index 132ad95e639..29c460c5c3d 100644
--- a/src/libsyntax/parse/token.rs
+++ b/src/libsyntax/parse/token.rs
@@ -20,6 +20,7 @@ use std::cmp::Equiv;
 use std::local_data;
 use std::rand;
 use std::rand::RngUtil;
+use std::ptr::to_unsafe_ptr;
 
 #[deriving(Clone, Encodable, Decodable, Eq, IterBytes)]
 pub enum binop {
@@ -544,15 +545,31 @@ pub fn gensym_ident(str : &str) -> ast::Ident {
 }
 
 // create a fresh name that maps to the same string as the old one.
+// note that this guarantees that ptr_eq(ident_to_str(src),interner_get(fresh_name(src)));
+// that is, that the new name and the old one are connected to ptr_eq strings.
 pub fn fresh_name(src : &ast::Ident) -> Name {
     gensym(ident_to_str(src))
     // following: debug version. Could work in final except that it's incompatible with
     // good error messages and uses of struct names in ambiguous could-be-binding
-    // locations.
+    // locations. Also definitely destroys the guarantee given above about ptr_eq.
     /*let num = rand::rng().gen_uint_range(0,0xffff);
     gensym(fmt!("%s_%u",ident_to_str(src),num))*/
 }
 
+// it looks like there oughta be a str_ptr_eq fn, but no one bothered to implement it?
+pub fn str_ptr_eq<T>(a: @str, b: @str) -> bool {
+    // doesn't compile! ...because of rebase mangling. this should be fixed
+    // in the commit that follows this.
+    let (a_ptr, b_ptr): (*uint, *uint) = (to_unsafe_ptr(a), to_unsafe_ptr(b));
+    a_ptr == b_ptr
+}
+
+
+
+// return true when two identifiers refer (through the intern table) to the same ptr_eq
+// string. This is used to compare identifiers in places where hygienic comparison is
+// not wanted (i.e. not lexical vars).
+
 // create a fresh mark.
 pub fn fresh_mark() -> Mrk {
     gensym("mark")
@@ -698,5 +715,19 @@ pub fn is_reserved_keyword(tok: &Token) -> bool {
 #[cfg(test)]
 mod test {
     use super::*;
+    use std::io;
+    use std::managed;
+    use ast;
+    use ast_util;
+
+
+    #[test] fn t1() {
+        let ghi = str_to_ident("ghi");
+        assert_eq!(ident_to_str(&ghi),@"ghi");
+        let fresh = ast::Ident::new(fresh_name(&ghi));
+        assert_eq!(ident_to_str(&fresh),@"ghi");
+        assert!(str_ptr_eq(ident_to_str(&ghi),ident_to_str(&fresh)));
+        assert_eq!(3,4);
+    }
 
 }
-- 
cgit 1.4.1-3-g733a5


From 58e7598c2e1265a0f1292ed6f93bfb29abb93504 Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Wed, 26 Jun 2013 10:11:19 -0700
Subject: added gensym_copy mechanism to ensure sharing of pointers in the
 interner

this makes comparisons constant-time, and enables spelling-comparison
of identifiers, crucial in many parts of resolve.
---
 src/libsyntax/parse/token.rs   | 73 +++++++++++++++++++-----------------------
 src/libsyntax/util/interner.rs | 63 +++++++++++++++++++++++++++++++-----
 2 files changed, 88 insertions(+), 48 deletions(-)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs
index 29c460c5c3d..17928338f37 100644
--- a/src/libsyntax/parse/token.rs
+++ b/src/libsyntax/parse/token.rs
@@ -15,12 +15,12 @@ use parse::token;
 use util::interner::StrInterner;
 use util::interner;
 
+use std::cast;
 use std::char;
 use std::cmp::Equiv;
 use std::local_data;
 use std::rand;
 use std::rand::RngUtil;
-use std::ptr::to_unsafe_ptr;
 
 #[deriving(Clone, Encodable, Decodable, Eq, IterBytes)]
 pub enum binop {
@@ -382,30 +382,8 @@ pub fn token_to_binop(tok: &Token) -> Option<ast::BinOp> {
   }
 }
 
-pub struct ident_interner {
-    priv interner: StrInterner,
-}
-
-impl ident_interner {
-    pub fn intern(&self, val: &str) -> Name {
-        self.interner.intern(val)
-    }
-    pub fn gensym(&self, val: &str) -> Name {
-        self.interner.gensym(val)
-    }
-    pub fn get(&self, idx: Name) -> @str {
-        self.interner.get(idx)
-    }
-    // is this really something that should be exposed?
-    pub fn len(&self) -> uint {
-        self.interner.len()
-    }
-    pub fn find_equiv<Q:Hash + IterBytes + Equiv<@str>>(&self, val: &Q)
-                                                     -> Option<Name> {
-        self.interner.find_equiv(val)
-    }
-}
-
+// looks like we can get rid of this completely...
+pub type ident_interner = StrInterner;
 
 // return a fresh interner, preloaded with special identifiers.
 fn mk_fresh_ident_interner() -> @ident_interner {
@@ -486,9 +464,7 @@ fn mk_fresh_ident_interner() -> @ident_interner {
         "typeof",             // 67
     ];
 
-    @ident_interner {
-        interner: interner::StrInterner::prefill(init_vec)
-    }
+    @interner::StrInterner::prefill(init_vec)
 }
 
 // if an interner exists in TLS, return it. Otherwise, prepare a
@@ -509,7 +485,7 @@ pub fn get_ident_interner() -> @ident_interner {
 /* for when we don't care about the contents; doesn't interact with TLD or
    serialization */
 pub fn mk_fake_ident_interner() -> @ident_interner {
-    @ident_interner { interner: interner::StrInterner::new() }
+    @interner::StrInterner::new()
 }
 
 // maps a string to its interned representation
@@ -545,10 +521,11 @@ pub fn gensym_ident(str : &str) -> ast::Ident {
 }
 
 // create a fresh name that maps to the same string as the old one.
-// note that this guarantees that ptr_eq(ident_to_str(src),interner_get(fresh_name(src)));
+// note that this guarantees that str_ptr_eq(ident_to_str(src),interner_get(fresh_name(src)));
 // that is, that the new name and the old one are connected to ptr_eq strings.
 pub fn fresh_name(src : &ast::Ident) -> Name {
-    gensym(ident_to_str(src))
+    let interner = get_ident_interner();
+    interner.gensym_copy(src.name)
     // following: debug version. Could work in final except that it's incompatible with
     // good error messages and uses of struct names in ambiguous could-be-binding
     // locations. Also definitely destroys the guarantee given above about ptr_eq.
@@ -557,18 +534,26 @@ pub fn fresh_name(src : &ast::Ident) -> Name {
 }
 
 // it looks like there oughta be a str_ptr_eq fn, but no one bothered to implement it?
-pub fn str_ptr_eq<T>(a: @str, b: @str) -> bool {
-    // doesn't compile! ...because of rebase mangling. this should be fixed
-    // in the commit that follows this.
-    let (a_ptr, b_ptr): (*uint, *uint) = (to_unsafe_ptr(a), to_unsafe_ptr(b));
-    a_ptr == b_ptr
-}
-
 
+// determine whether two @str values are pointer-equal
+pub fn str_ptr_eq(a : @str, b : @str) -> bool {
+    unsafe {
+        let p : uint = cast::transmute(a);
+        let q : uint = cast::transmute(b);
+        let result = p == q;
+        // got to transmute them back, to make sure the ref count is correct:
+        let junk1 : @str = cast::transmute(p);
+        let junk2 : @str = cast::transmute(q);
+        result
+    }
+}
 
 // return true when two identifiers refer (through the intern table) to the same ptr_eq
 // string. This is used to compare identifiers in places where hygienic comparison is
 // not wanted (i.e. not lexical vars).
+pub fn ident_spelling_eq(a : &ast::Ident, b : &ast::Ident) -> bool {
+    str_ptr_eq(interner_get(a.name),interner_get(b.name))
+}
 
 // create a fresh mark.
 pub fn fresh_mark() -> Mrk {
@@ -721,13 +706,21 @@ mod test {
     use ast_util;
 
 
-    #[test] fn t1() {
+    #[test] fn str_ptr_eq_tests(){
+        let a = @"abc";
+        let b = @"abc";
+        let c = a;
+        assert!(str_ptr_eq(a,c));
+        assert!(!str_ptr_eq(a,b));
+    }
+
+    #[test] fn fresh_name_pointer_sharing() {
         let ghi = str_to_ident("ghi");
         assert_eq!(ident_to_str(&ghi),@"ghi");
+        assert!(str_ptr_eq(ident_to_str(&ghi),ident_to_str(&ghi)))
         let fresh = ast::Ident::new(fresh_name(&ghi));
         assert_eq!(ident_to_str(&fresh),@"ghi");
         assert!(str_ptr_eq(ident_to_str(&ghi),ident_to_str(&fresh)));
-        assert_eq!(3,4);
     }
 
 }
diff --git a/src/libsyntax/util/interner.rs b/src/libsyntax/util/interner.rs
index 46676ce1093..2b1e7eaa9b2 100644
--- a/src/libsyntax/util/interner.rs
+++ b/src/libsyntax/util/interner.rs
@@ -117,6 +117,23 @@ impl StrInterner {
         new_idx
     }
 
+    // I want these gensyms to share name pointers
+    // with existing entries. This would be automatic,
+    // except that the existing gensym creates its
+    // own managed ptr using to_managed. I think that
+    // adding this utility function is the most
+    // lightweight way to get what I want, though not
+    // necessarily the cleanest.
+
+    // create a gensym with the same name as an existing
+    // entry.
+    pub fn gensym_copy(&self, idx : uint) -> uint {
+        let new_idx = self.len();
+        // leave out of map to avoid colliding
+        self.vect.push(self.vect[idx]);
+        new_idx
+    }
+
     // this isn't "pure" in the traditional sense, because it can go from
     // failing to returning a value as items are interned. But for typestate,
     // where we first check a pred and then rely on it, ceasing to fail is ok.
@@ -144,23 +161,23 @@ mod tests {
     }
 
     #[test]
-    fn i2 () {
+    fn interner_tests () {
         let i : Interner<@str> = Interner::new();
         // first one is zero:
-        assert_eq!(i.intern (@"dog"), 0);
+        assert_eq!(i.intern(@"dog"), 0);
         // re-use gets the same entry:
-        assert_eq!(i.intern (@"dog"), 0);
+        assert_eq!(i.intern(@"dog"), 0);
         // different string gets a different #:
-        assert_eq!(i.intern (@"cat"), 1);
-        assert_eq!(i.intern (@"cat"), 1);
+        assert_eq!(i.intern(@"cat"), 1);
+        assert_eq!(i.intern(@"cat"), 1);
         // dog is still at zero
-        assert_eq!(i.intern (@"dog"), 0);
+        assert_eq!(i.intern(@"dog"), 0);
         // gensym gets 3
-        assert_eq!(i.gensym (@"zebra" ), 2);
+        assert_eq!(i.gensym(@"zebra" ), 2);
         // gensym of same string gets new number :
         assert_eq!(i.gensym (@"zebra" ), 3);
         // gensym of *existing* string gets new number:
-        assert_eq!(i.gensym (@"dog"), 4);
+        assert_eq!(i.gensym(@"dog"), 4);
         assert_eq!(i.get(0), @"dog");
         assert_eq!(i.get(1), @"cat");
         assert_eq!(i.get(2), @"zebra");
@@ -176,4 +193,34 @@ mod tests {
         assert_eq!(i.get(2), @"Carol");
         assert_eq!(i.intern(@"Bob"), 1);
     }
+
+    #[test]
+    fn string_interner_tests() {
+        let i : StrInterner = StrInterner::new();
+        // first one is zero:
+        assert_eq!(i.intern("dog"), 0);
+        // re-use gets the same entry:
+        assert_eq!(i.intern ("dog"), 0);
+        // different string gets a different #:
+        assert_eq!(i.intern("cat"), 1);
+        assert_eq!(i.intern("cat"), 1);
+        // dog is still at zero
+        assert_eq!(i.intern("dog"), 0);
+        // gensym gets 3
+        assert_eq!(i.gensym("zebra"), 2);
+        // gensym of same string gets new number :
+        assert_eq!(i.gensym("zebra"), 3);
+        // gensym of *existing* string gets new number:
+        assert_eq!(i.gensym("dog"), 4);
+        // gensym tests again with gensym_copy:
+        assert_eq!(i.gensym_copy(2), 5);
+        assert_eq!(i.get(5), @"zebra");
+        assert_eq!(i.gensym_copy(2), 6);
+        assert_eq!(i.get(6), @"zebra");
+        assert_eq!(i.get(0), @"dog");
+        assert_eq!(i.get(1), @"cat");
+        assert_eq!(i.get(2), @"zebra");
+        assert_eq!(i.get(3), @"zebra");
+        assert_eq!(i.get(4), @"dog");
+    }
 }
-- 
cgit 1.4.1-3-g733a5


From 963dab5dcb9c348bb2631998702695de62916b31 Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Sat, 29 Jun 2013 05:59:08 -0700
Subject: marking on both input and output from macros. nice shiny new test
 case framework

---
 src/libsyntax/ext/expand.rs | 304 +++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 275 insertions(+), 29 deletions(-)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index baa4e616b4f..63dd8231aa0 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -67,10 +67,10 @@ pub fn expand_expr(extsbox: @mut SyntaxEnv,
                                     span: exp_span,
                                 },
                             });
-
                             let fm = fresh_mark();
-                            let marked_tts = mark_tts(*tts,fm);
-                            let expanded = match expandfun(cx, mac.span, marked_tts) {
+                            // mark before:
+                            let marked_before = mark_tts(*tts,fm);
+                            let expanded = match expandfun(cx, mac.span, marked_before) {
                                 MRExpr(e) => e,
                                 MRAny(expr_maker,_,_) => expr_maker(),
                                 _ => {
@@ -83,10 +83,12 @@ pub fn expand_expr(extsbox: @mut SyntaxEnv,
                                     )
                                 }
                             };
+                            // mark after:
+                            let marked_after = mark_expr(expanded,fm);
 
                             //keep going, outside-in
                             let fully_expanded =
-                                fld.fold_expr(expanded).node.clone();
+                                fld.fold_expr(marked_after).node.clone();
                             cx.bt_pop();
 
                             (fully_expanded, s)
@@ -262,12 +264,6 @@ pub fn expand_expr(extsbox: @mut SyntaxEnv,
     }
 }
 
-// apply a fresh mark to the given token trees. Used prior to expansion of a macro.
-fn mark_tts(tts : &[token_tree], m : Mrk) -> ~[token_tree] {
-    fold_tts(tts,new_ident_marker(m))
-}
-
-
 // This is a secondary mechanism for invoking syntax extensions on items:
 // "decorator" attributes, such as #[auto_encode]. These are invoked by an
 // attribute prefixing an item, and are interpreted by feeding the item
@@ -315,7 +311,6 @@ pub fn expand_mod_items(extsbox: @mut SyntaxEnv,
     ast::_mod { items: new_items, ..module_ }
 }
 
-
 // eval $e with a new exts frame:
 macro_rules! with_exts_frame (
     ($extsboxexpr:expr,$macros_escape:expr,$e:expr) =>
@@ -381,6 +376,7 @@ pub fn expand_item_mac(extsbox: @mut SyntaxEnv,
 
     let extname = &pth.segments[0].identifier;
     let extnamestr = ident_to_str(extname);
+    let fm = fresh_mark();
     let expanded = match (*extsbox).find(&extname.name) {
         None => cx.span_fatal(pth.span,
                               fmt!("macro undefined: '%s!'", extnamestr)),
@@ -399,7 +395,11 @@ pub fn expand_item_mac(extsbox: @mut SyntaxEnv,
                     span: span
                 }
             });
-             expander(cx, it.span, tts)
+            // mark before expansion:
+            let marked_tts = mark_tts(tts,fm);
+            // mark after expansion:
+            // RIGHT HERE: can't apply mark_item to MacResult ... :(
+            expander(cx, it.span, marked_tts)
         }
         Some(@SE(IdentTT(expander, span))) => {
             if it.ident.name == parse::token::special_idents::invalid.name {
@@ -414,18 +414,24 @@ pub fn expand_item_mac(extsbox: @mut SyntaxEnv,
                     span: span
                 }
             });
-            expander(cx, it.span, it.ident, tts)
+            let fm = fresh_mark();
+            // mark before expansion:
+            let marked_tts = mark_tts(tts,fm);
+        expander(cx, it.span, it.ident, marked_tts)
         }
         _ => cx.span_fatal(
             it.span, fmt!("%s! is not legal in item position", extnamestr))
     };
 
     let maybe_it = match expanded {
-        MRItem(it) => fld.fold_item(it),
+        MRItem(it) => mark_item(it,fm).chain(|i| {fld.fold_item(i)}),
         MRExpr(_) => cx.span_fatal(pth.span,
                                    fmt!("expr macro in item position: %s", extnamestr)),
-        MRAny(_, item_maker, _) => item_maker().chain(|i| {fld.fold_item(i)}),
+        MRAny(_, item_maker, _) => item_maker().chain(|i| {mark_item(i,fm)})
+                                      .chain(|i| {fld.fold_item(i)}),
         MRDef(ref mdef) => {
+            // yikes... no idea how to apply the mark to this. I'm afraid
+            // we're going to have to wait-and-see on this one.
             insert_macro(*extsbox,intern(mdef.name), @SE((*mdef).ext));
             None
         }
@@ -460,6 +466,8 @@ pub fn expand_stmt(extsbox: @mut SyntaxEnv,
                    orig: @fn(&Stmt_, Span, @ast_fold)
                              -> (Option<Stmt_>, Span))
                 -> (Option<Stmt_>, Span) {
+    // why the copying here and not in expand_expr?
+    // looks like classic changed-in-only-one-place
     let (mac, pth, tts, semi) = match *s {
         StmtMac(ref mac, semi) => {
             match mac.node {
@@ -487,7 +495,10 @@ pub fn expand_stmt(extsbox: @mut SyntaxEnv,
                 call_site: sp,
                 callee: NameAndSpan { name: extnamestr, span: exp_span }
             });
-            let expanded = match expandfun(cx, mac.span, tts) {
+            let fm = fresh_mark();
+            // mark before expansion:
+            let marked_tts = mark_tts(tts,fm);
+            let expanded = match expandfun(cx, mac.span, marked_tts) {
                 MRExpr(e) =>
                     @codemap::Spanned { node: StmtExpr(e, cx.next_id()),
                                     span: e.span},
@@ -496,9 +507,10 @@ pub fn expand_stmt(extsbox: @mut SyntaxEnv,
                     pth.span,
                     fmt!("non-stmt macro in stmt pos: %s", extnamestr))
             };
+            let marked_after = mark_stmt(expanded,fm);
 
             //keep going, outside-in
-            let fully_expanded = match fld.fold_stmt(expanded) {
+            let fully_expanded = match fld.fold_stmt(marked_after) {
                 Some(stmt) => {
                     let fully_expanded = &stmt.node;
                     cx.bt_pop();
@@ -736,6 +748,132 @@ impl Visitor<()> for NewNameFinderContext {
     }
 }
 
+// a visitor that extracts the path exprs from
+// a crate/expression/whatever and puts them in a mutable
+// array (passed in to the traversal)
+#[deriving(Clone)]
+struct NewPathExprFinderContext {
+    path_accumulator: @mut ~[ast::Path],
+}
+
+
+// XXX : YIKES a lot of boilerplate again....
+impl Visitor<()> for NewPathExprFinderContext {
+
+    fn visit_expr(&mut self, expr: @ast::Expr, _: ()) {
+        match *expr {
+            ast::Expr{id:_,span:_,node:ast::ExprPath(ref p)} => {
+                self.path_accumulator.push(p.clone());
+                // not calling visit_path, should be fine.
+            }
+            _ => visit::walk_expr(self,expr,())
+        }
+    }
+
+
+    // XXX: Methods below can become default methods.
+
+    fn visit_pat(&mut self, pattern: @ast::Pat, _: ()) {
+        visit::walk_pat(self,pattern,())
+    }
+
+    fn visit_mod(&mut self, module: &ast::_mod, _: Span, _: NodeId, _: ()) {
+        visit::walk_mod(self, module, ())
+    }
+
+    fn visit_view_item(&mut self, view_item: &ast::view_item, _: ()) {
+        visit::walk_view_item(self, view_item, ())
+    }
+
+    fn visit_item(&mut self, item: @ast::item, _: ()) {
+        visit::walk_item(self, item, ())
+    }
+
+    fn visit_foreign_item(&mut self,
+                          foreign_item: @ast::foreign_item,
+                          _: ()) {
+        visit::walk_foreign_item(self, foreign_item, ())
+    }
+
+    fn visit_local(&mut self, local: @ast::Local, _: ()) {
+        visit::walk_local(self, local, ())
+    }
+
+    fn visit_block(&mut self, block: &ast::Block, _: ()) {
+        visit::walk_block(self, block, ())
+    }
+
+    fn visit_stmt(&mut self, stmt: @ast::Stmt, _: ()) {
+        visit::walk_stmt(self, stmt, ())
+    }
+
+    fn visit_arm(&mut self, arm: &ast::Arm, _: ()) {
+        visit::walk_arm(self, arm, ())
+    }
+
+    fn visit_decl(&mut self, decl: @ast::Decl, _: ()) {
+        visit::walk_decl(self, decl, ())
+    }
+
+    fn visit_expr_post(&mut self, _: @ast::Expr, _: ()) {
+        // Empty!
+    }
+
+    fn visit_ty(&mut self, typ: &ast::Ty, _: ()) {
+        visit::walk_ty(self, typ, ())
+    }
+
+    fn visit_generics(&mut self, generics: &ast::Generics, _: ()) {
+        visit::walk_generics(self, generics, ())
+    }
+
+    fn visit_fn(&mut self,
+                function_kind: &visit::fn_kind,
+                function_declaration: &ast::fn_decl,
+                block: &ast::Block,
+                span: Span,
+                node_id: NodeId,
+                _: ()) {
+        visit::walk_fn(self,
+                        function_kind,
+                        function_declaration,
+                        block,
+                        span,
+                        node_id,
+                        ())
+    }
+
+    fn visit_ty_method(&mut self, ty_method: &ast::TypeMethod, _: ()) {
+        visit::walk_ty_method(self, ty_method, ())
+    }
+
+    fn visit_trait_method(&mut self,
+                          trait_method: &ast::trait_method,
+                          _: ()) {
+        visit::walk_trait_method(self, trait_method, ())
+    }
+
+    fn visit_struct_def(&mut self,
+                        struct_def: @ast::struct_def,
+                        ident: Ident,
+                        generics: &ast::Generics,
+                        node_id: NodeId,
+                        _: ()) {
+        visit::walk_struct_def(self,
+                                struct_def,
+                                ident,
+                                generics,
+                                node_id,
+                                ())
+    }
+
+    fn visit_struct_field(&mut self,
+                          struct_field: @ast::struct_field,
+                          _: ()) {
+        visit::walk_struct_field(self, struct_field, ())
+    }
+}
+
 // return a visitor that extracts the pat_ident paths
 // from a given pattern and puts them in a mutable
 // array (passed in to the traversal)
@@ -746,6 +884,16 @@ pub fn new_name_finder(idents: @mut ~[ast::Ident]) -> @mut Visitor<()> {
     context as @mut Visitor<()>
 }
 
+// return a visitor that extracts the paths
+// from a given pattern and puts them in a mutable
+// array (passed in to the traversal)
+pub fn new_path_finder(paths: @mut ~[ast::Path]) -> @mut Visitor<()> {
+    let context = @mut NewPathExprFinderContext {
+        path_accumulator: paths,
+    };
+    context as @mut Visitor<()>
+}
+
 // given a mutable list of renames, return a tree-folder that applies those
 // renames.
 // FIXME #4536: currently pub to allow testing
@@ -1295,7 +1443,6 @@ pub fn new_ident_renamer(from: ast::Ident,
     }
 }
 
-
 // update the ctxts in a path to get a mark node
 pub fn new_ident_marker(mark: Mrk) ->
     @fn(ast::Ident)->ast::Ident {
@@ -1319,20 +1466,42 @@ pub fn new_ident_resolver() ->
     }
 }
 
+// apply a given mark to the given token trees. Used prior to expansion of a macro.
+fn mark_tts(tts : &[token_tree], m : Mrk) -> ~[token_tree] {
+    fold_tts(tts,new_ident_marker(m))
+}
+
+// apply a given mark to the given expr. Used following the expansion of a macro.
+fn mark_expr(expr : @ast::Expr, m : Mrk) -> @ast::Expr {
+    fun_to_ident_folder(new_ident_marker(m)).fold_expr(expr)
+}
+
+// apply a given mark to the given stmt. Used following the expansion of a macro.
+fn mark_stmt(expr : &ast::Stmt, m : Mrk) -> @ast::Stmt {
+    fun_to_ident_folder(new_ident_marker(m)).fold_stmt(expr).unwrap()
+}
+
+// apply a given mark to the given item. Used following the expansion of a macro.
+fn mark_item(expr : @ast::item, m : Mrk) -> Option<@ast::item> {
+    fun_to_ident_folder(new_ident_marker(m)).fold_item(expr)
+}
+
 #[cfg(test)]
 mod test {
     use super::*;
     use ast;
     use ast::{Attribute_, AttrOuter, MetaWord, EMPTY_CTXT};
-    use ast_util::{get_sctable, new_rename};
+    use ast_util::{get_sctable, mtwt_resolve, new_rename};
     use codemap;
     use codemap::Spanned;
     use parse;
     use parse::token::{gensym, intern, get_ident_interner};
     use print::pprust;
     use std;
+    use std::vec;
     use util::parser_testing::{string_to_crate_and_sess, string_to_item, string_to_pat};
     use util::parser_testing::{strs_to_idents};
+    use visit;
 
     // make sure that fail! is present
     #[test] fn fail_exists_test () {
@@ -1478,30 +1647,107 @@ mod test {
         //pprust::to_str(&resolved_ast,fake_print_crate,get_ident_interner())
     //}
 
+    // renaming tests expand a crate and then check that the bindings match
+    // the right varrefs. The specification of the test case includes the
+    // text of the crate, and also an array of arrays.  Each element in the
+    // outer array corresponds to a binding in the traversal of the AST
+    // induced by visit.  Each of these arrays contains a list of indexes,
+    // interpreted as the varrefs in the varref traversal that this binding
+    // should match.  So, for instance, in a program with two bindings and
+    // three varrefs, the array ~[~[1,2],~[0]] would indicate that the first
+    // binding should match the second two varrefs, and the second binding
+    // should match the first varref.
+    //
+    // The comparisons are done post-mtwt-resolve, so we're comparing renamed
+    // names; differences in marks don't matter any more.
+    type renaming_test = (&'static str, ~[~[uint]]);
+
     #[test]
     fn automatic_renaming () {
         // need some other way to test these...
-        let teststrs =
+        let tests : ~[renaming_test] =
             ~[// b & c should get new names throughout, in the expr too:
-                @"fn a() -> int { let b = 13; let c = b; b+c }",
+                ("fn a() -> int { let b = 13; let c = b; b+c }",
+                 ~[~[0,1],~[2]]),
                 // both x's should be renamed (how is this causing a bug?)
-                @"fn main () {let x : int = 13;x;}",
-                // the use of b before the + should be renamed, the other one not:
-                @"macro_rules! f (($x:ident) => ($x + b)) fn a() -> int { let b = 13; f!(b)}",
+                ("fn main () {let x : int = 13;x;}",
+                 ~[~[0]]),
+                // the use of b after the + should be renamed, the other one not:
+                ("macro_rules! f (($x:ident) => (b + $x)) fn a() -> int { let b = 13; f!(b)}",
+                 ~[~[1]]),
                 // the b before the plus should not be renamed (requires marks)
-                @"macro_rules! f (($x:ident) => ({let b=9; ($x + b)})) fn a() -> int { f!(b)}",
+                ("macro_rules! f (($x:ident) => ({let b=9; ($x + b)})) fn a() -> int { f!(b)}",
+                 ~[~[1]]),
+                // the marks going in and out of letty should cancel, allowing that $x to
+                // capture the one following the semicolon.
+                // this was an awesome test case, and caught a *lot* of bugs.
+                ("macro_rules! letty(($x:ident) => (let $x = 15;))
+                  macro_rules! user(($x:ident) => ({letty!($x); $x}))
+                  fn main() -> int {user!(z)}",
+                 ~[~[0]])
                 // FIXME #6994: the next string exposes the bug referred to in issue 6994, so I'm
                 // commenting it out.
                 // the z flows into and out of two macros (g & f) along one path, and one (just g) along the
                 // other, so the result of the whole thing should be "let z_123 = 3; z_123"
-                //@"macro_rules! g (($x:ident) => ({macro_rules! f(($y:ident)=>({let $y=3;$x}));f!($x)}))
+                //"macro_rules! g (($x:ident) => ({macro_rules! f(($y:ident)=>({let $y=3;$x}));f!($x)}))
                 //   fn a(){g!(z)}"
                 // create a really evil test case where a $x appears inside a binding of $x but *shouldnt*
                 // bind because it was inserted by a different macro....
             ];
-        for s in teststrs.iter() {
-            // we need regexps to test these!
-            //std::io::println(expand_and_resolve_and_pretty_print(*s));
+        for s in tests.iter() {
+            run_renaming_test(s);
+        }
+    }
+
+
+    fn run_renaming_test(t : &renaming_test) {
+        let (teststr, bound_connections) = match *t {
+            (ref str,ref conns) => (str.to_managed(), conns.clone())
+        };
+        let cr = expand_crate_str(teststr.to_managed());
+        // find the bindings:
+        let bindings = @mut ~[];
+        visit::walk_crate(&mut new_name_finder(bindings),cr,());
+        // find the varrefs:
+        let varrefs = @mut ~[];
+        visit::walk_crate(&mut new_path_finder(varrefs),cr,());
+        // must be one check clause for each binding:
+        assert_eq!(bindings.len(),bound_connections.len());
+        for (binding_idx,shouldmatch) in bound_connections.iter().enumerate() {
+            let binding_name = mtwt_resolve(bindings[binding_idx]);
+            // shouldmatch can't name varrefs that don't exist:
+            assert!((shouldmatch.len() == 0) ||
+                    (varrefs.len() > *shouldmatch.iter().max().unwrap()));
+            for (idx,varref) in varrefs.iter().enumerate() {
+                if shouldmatch.contains(&idx) {
+                    // it should be a path of length 1, and it should
+                    // be free-identifier=? to the given binding
+                    assert_eq!(varref.segments.len(),1);
+                    let varref_name = mtwt_resolve(varref.segments[0].identifier);
+                    if (!(varref_name==binding_name)){
+                        std::io::println("uh oh, should match but doesn't:");
+                        std::io::println(fmt!("varref: %?",varref));
+                        std::io::println(fmt!("binding: %?", bindings[binding_idx]));
+                        let table = get_sctable();
+                        std::io::println("SC table:");
+                        for (idx,val) in table.table.iter().enumerate() {
+                            std::io::println(fmt!("%4u : %?",idx,val));
+                        }
+                    }
+                    assert_eq!(varref_name,binding_name);
+                } else {
+                    let fail = (varref.segments.len() == 1)
+                        && (mtwt_resolve(varref.segments[0].identifier) == binding_name);
+                    // temp debugging:
+                    if (fail) {
+                        std::io::println("uh oh, matches but shouldn't:");
+                        std::io::println(fmt!("varref: %?",varref));
+                        std::io::println(fmt!("binding: %?", bindings[binding_idx]));
+                        std::io::println(fmt!("sc_table: %?",get_sctable()));
+                    }
+                    assert!(!fail);
+                }
+            }
         }
     }
 
-- 
cgit 1.4.1-3-g733a5


From 0c31c930ffc6a5c3b2cbaf45f7d5596840a8926d Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Tue, 9 Jul 2013 15:10:16 -0700
Subject: add test case for macro token comparison

---
 src/libsyntax/ext/expand.rs | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index 63dd8231aa0..d072f2a7812 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -1647,6 +1647,10 @@ mod test {
         //pprust::to_str(&resolved_ast,fake_print_crate,get_ident_interner())
     //}
 
+    #[test] fn macro_tokens_should_match(){
+        expand_crate_str(@"macro_rules! m((a)=>(13)) fn main(){m!(a);}");
+    }
+
     // renaming tests expand a crate and then check that the bindings match
     // the right varrefs. The specification of the test case includes the
     // text of the crate, and also an array of arrays.  Each element in the
-- 
cgit 1.4.1-3-g733a5


From 3621c674cc20e666cfa3fdef559f516b39a1189a Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Wed, 3 Jul 2013 15:15:45 -0700
Subject: comments

---
 src/libsyntax/parse/token.rs | 39 +++++++++++++++++++++++++++++++--------
 1 file changed, 31 insertions(+), 8 deletions(-)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs
index 17928338f37..311d498eec2 100644
--- a/src/libsyntax/parse/token.rs
+++ b/src/libsyntax/parse/token.rs
@@ -310,22 +310,23 @@ pub fn is_bar(t: &Token) -> bool {
     match *t { BINOP(OR) | OROR => true, _ => false }
 }
 
-
 pub mod special_idents {
     use ast::Ident;
 
-    pub static underscore : Ident = Ident { name: 0, ctxt: 0};
+    pub static underscore : Ident = Ident { name: 0, ctxt: 0}; // apparently unused?
     pub static anon : Ident = Ident { name: 1, ctxt: 0};
     pub static invalid : Ident = Ident { name: 2, ctxt: 0}; // ''
-    pub static unary : Ident = Ident { name: 3, ctxt: 0};
-    pub static not_fn : Ident = Ident { name: 4, ctxt: 0};
-    pub static idx_fn : Ident = Ident { name: 5, ctxt: 0};
-    pub static unary_minus_fn : Ident = Ident { name: 6, ctxt: 0};
+    pub static unary : Ident = Ident { name: 3, ctxt: 0}; // apparently unused?
+    pub static not_fn : Ident = Ident { name: 4, ctxt: 0}; // apparently unused?
+    pub static idx_fn : Ident = Ident { name: 5, ctxt: 0}; // apparently unused?
+    pub static unary_minus_fn : Ident = Ident { name: 6, ctxt: 0}; // apparently unused?
     pub static clownshoes_extensions : Ident = Ident { name: 7, ctxt: 0};
 
     pub static self_ : Ident = Ident { name: 8, ctxt: 0}; // 'self'
 
     /* for matcher NTs */
+    // none of these appear to be used, but perhaps references to
+    // these are artificially fabricated by the macro system....
     pub static item : Ident = Ident { name: 9, ctxt: 0};
     pub static block : Ident = Ident { name: 10, ctxt: 0};
     pub static stmt : Ident = Ident { name: 11, ctxt: 0};
@@ -337,7 +338,7 @@ pub mod special_idents {
     pub static tt : Ident = Ident { name: 17, ctxt: 0};
     pub static matchers : Ident = Ident { name: 18, ctxt: 0};
 
-    pub static str : Ident = Ident { name: 19, ctxt: 0}; // for the type
+    pub static str : Ident = Ident { name: 19, ctxt: 0}; // for the type // apparently unused?
 
     /* outside of libsyntax */
     pub static arg : Ident = Ident { name: 20, ctxt: 0};
@@ -350,10 +351,32 @@ pub mod special_idents {
     pub static statik : Ident = Ident { name: 27, ctxt: 0};
     pub static clownshoes_foreign_mod: Ident = Ident { name: 28, ctxt: 0};
     pub static unnamed_field: Ident = Ident { name: 29, ctxt: 0};
-    pub static c_abi: Ident = Ident { name: 30, ctxt: 0};
+    pub static c_abi: Ident = Ident { name: 30, ctxt: 0}; // apparently unused?
     pub static type_self: Ident = Ident { name: 31, ctxt: 0};    // `Self`
 }
 
+// here are the ones that actually occur in the source. Maybe the rest
+// should be removed?
+/*
+special_idents::anon
+special_idents::arg
+special_idents::blk
+special_idents::clownshoe_abi
+special_idents::clownshoe_stack_shim
+special_idents::clownshoes_extensions
+special_idents::clownshoes_foreign_mod
+special_idents::descrim
+special_idents::invalid
+special_idents::main
+special_idents::matchers
+special_idents::opaque
+special_idents::self_
+special_idents::statik
+special_idents::tt
+special_idents::type_self
+special_idents::unnamed_field
+*/
+
 /**
  * Maps a token to a record specifying the corresponding binary
  * operator
-- 
cgit 1.4.1-3-g733a5


From 09e6dda4f268e24c5d1f0804f5c1e57d1fcc158d Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Wed, 3 Jul 2013 15:16:04 -0700
Subject: add temporarily unused ctxt field to mac_invoc_tt

---
 src/libsyntax/ast.rs          |  2 +-
 src/libsyntax/ext/expand.rs   |  6 +++---
 src/libsyntax/fold.rs         |  5 +++--
 src/libsyntax/parse/parser.rs | 10 +++++-----
 src/libsyntax/print/pprust.rs |  4 ++--
 5 files changed, 14 insertions(+), 13 deletions(-)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs
index b3fb8859f65..c2f257643cd 100644
--- a/src/libsyntax/ast.rs
+++ b/src/libsyntax/ast.rs
@@ -659,7 +659,7 @@ pub type mac = Spanned<mac_>;
 // There's only one flavor, now, so this could presumably be simplified.
 #[deriving(Clone, Eq, Encodable, Decodable, IterBytes)]
 pub enum mac_ {
-    mac_invoc_tt(Path,~[token_tree]),   // new macro-invocation
+    mac_invoc_tt(Path,~[token_tree],SyntaxContext),   // new macro-invocation
 }
 
 pub type lit = Spanned<lit_>;
diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index d072f2a7812..5e68a75bffa 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -43,7 +43,7 @@ pub fn expand_expr(extsbox: @mut SyntaxEnv,
         ExprMac(ref mac) => {
             match (*mac).node {
                 // Token-tree macros:
-                mac_invoc_tt(ref pth, ref tts) => {
+                mac_invoc_tt(ref pth, ref tts, ctxt) => {
                     if (pth.segments.len() > 1u) {
                         cx.span_fatal(
                             pth.span,
@@ -368,7 +368,7 @@ pub fn expand_item_mac(extsbox: @mut SyntaxEnv,
                        fld: @ast_fold)
                     -> Option<@ast::item> {
     let (pth, tts) = match it.node {
-        item_mac(codemap::Spanned { node: mac_invoc_tt(ref pth, ref tts), _}) => {
+        item_mac(codemap::Spanned { node: mac_invoc_tt(ref pth, ref tts, ctxt), _}) => {
             (pth, (*tts).clone())
         }
         _ => cx.span_bug(it.span, "invalid item macro invocation")
@@ -471,7 +471,7 @@ pub fn expand_stmt(extsbox: @mut SyntaxEnv,
     let (mac, pth, tts, semi) = match *s {
         StmtMac(ref mac, semi) => {
             match mac.node {
-                mac_invoc_tt(ref pth, ref tts) => {
+                mac_invoc_tt(ref pth, ref tts, ctxt) => {
                     ((*mac).clone(), pth, (*tts).clone(), semi)
                 }
             }
diff --git a/src/libsyntax/fold.rs b/src/libsyntax/fold.rs
index 85519bd2671..799ff855cbe 100644
--- a/src/libsyntax/fold.rs
+++ b/src/libsyntax/fold.rs
@@ -116,9 +116,10 @@ fn fold_arg_(a: arg, fld: @ast_fold) -> arg {
 fn fold_mac_(m: &mac, fld: @ast_fold) -> mac {
     Spanned {
         node: match m.node {
-            mac_invoc_tt(ref p,ref tts) =>
+            mac_invoc_tt(ref p,ref tts,ctxt) =>
             mac_invoc_tt(fld.fold_path(p),
-                         fold_tts(*tts,|id|{fld.fold_ident(id)}))
+                         fold_tts(*tts,|id|{fld.fold_ident(id)}),
+                         ctxt)
         },
         span: fld.new_span(m.span)
     }
diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs
index 62abe3850c9..8b11a25f13c 100644
--- a/src/libsyntax/parse/parser.rs
+++ b/src/libsyntax/parse/parser.rs
@@ -21,7 +21,7 @@ use ast::{_mod, BiAdd, arg, Arm, Attribute, BindByRef, BindInfer};
 use ast::{BiBitAnd, BiBitOr, BiBitXor, Block};
 use ast::{BlockCheckMode, UnBox};
 use ast::{Crate, CrateConfig, Decl, DeclItem};
-use ast::{DeclLocal, DefaultBlock, UnDeref, BiDiv, enum_def, explicit_self};
+use ast::{DeclLocal, DefaultBlock, UnDeref, BiDiv, EMPTY_CTXT, enum_def, explicit_self};
 use ast::{Expr, Expr_, ExprAddrOf, ExprMatch, ExprAgain};
 use ast::{ExprAssign, ExprAssignOp, ExprBinary, ExprBlock};
 use ast::{ExprBreak, ExprCall, ExprCast, ExprDoBody};
@@ -1875,7 +1875,7 @@ impl Parser {
                                                 |p| p.parse_token_tree());
                 let hi = self.span.hi;
 
-                return self.mk_mac_expr(lo, hi, mac_invoc_tt(pth, tts));
+                return self.mk_mac_expr(lo, hi, mac_invoc_tt(pth, tts, EMPTY_CTXT));
             } else if *self.token == token::LBRACE {
                 // This might be a struct literal.
                 if self.looking_at_record_literal() {
@@ -3197,14 +3197,14 @@ impl Parser {
 
             if id == token::special_idents::invalid {
                 return @spanned(lo, hi, StmtMac(
-                    spanned(lo, hi, mac_invoc_tt(pth, tts)), false));
+                    spanned(lo, hi, mac_invoc_tt(pth, tts, EMPTY_CTXT)), false));
             } else {
                 // if it has a special ident, it's definitely an item
                 return @spanned(lo, hi, StmtDecl(
                     @spanned(lo, hi, DeclItem(
                         self.mk_item(
                             lo, hi, id /*id is good here*/,
-                            item_mac(spanned(lo, hi, mac_invoc_tt(pth, tts))),
+                            item_mac(spanned(lo, hi, mac_invoc_tt(pth, tts, EMPTY_CTXT))),
                             inherited, ~[/*no attrs*/]))),
                     self.get_id()));
             }
@@ -4809,7 +4809,7 @@ impl Parser {
                 _ => self.fatal("expected open delimiter")
             };
             // single-variant-enum... :
-            let m = ast::mac_invoc_tt(pth, tts);
+            let m = ast::mac_invoc_tt(pth, tts, EMPTY_CTXT);
             let m: ast::mac = codemap::Spanned { node: m,
                                              span: mk_sp(self.span.lo,
                                                          self.span.hi) };
diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs
index 32cf30fd3a0..4d464706d6f 100644
--- a/src/libsyntax/print/pprust.rs
+++ b/src/libsyntax/print/pprust.rs
@@ -619,7 +619,7 @@ pub fn print_item(s: @ps, item: &ast::item) {
         }
         bclose(s, item.span);
       }
-      ast::item_mac(codemap::Spanned { node: ast::mac_invoc_tt(ref pth, ref tts),
+      ast::item_mac(codemap::Spanned { node: ast::mac_invoc_tt(ref pth, ref tts, ctxt),
                                    _}) => {
         print_visibility(s, item.vis);
         print_path(s, pth, false);
@@ -1021,7 +1021,7 @@ pub fn print_if(s: @ps, test: &ast::Expr, blk: &ast::Block,
 
 pub fn print_mac(s: @ps, m: &ast::mac) {
     match m.node {
-      ast::mac_invoc_tt(ref pth, ref tts) => {
+      ast::mac_invoc_tt(ref pth, ref tts, ctxt) => {
         print_path(s, pth, false);
         word(s.s, "!");
         popen(s);
-- 
cgit 1.4.1-3-g733a5


From 34f31e296191f7c4cfcad8c81bf6bca00a32d9e2 Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Tue, 9 Jul 2013 15:56:21 -0700
Subject: rework fold so that fold_tts takes an ast_fold rather than a thunk,
 stop using closures in ident traversal

---
 src/libsyntax/ext/expand.rs | 106 +++++++++++++++++++++++++-------------------
 src/libsyntax/fold.rs       |  14 +++---
 2 files changed, 69 insertions(+), 51 deletions(-)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index 5e68a75bffa..c73f8b01b9c 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -1421,38 +1421,52 @@ pub fn expand_crate(parse_sess: @mut parse::ParseSess,
     return ret;
 }
 
-// given a function from idents to idents, produce
-// an ast_fold that applies that function:
-pub fn fun_to_ident_folder(f: @fn(ast::Ident)->ast::Ident) -> @ast_fold{
-    let afp = default_ast_fold();
-    let f_pre = @AstFoldFns{
-        fold_ident : |id, _| f(id),
-        .. *afp
-    };
-    make_fold(f_pre)
+// a function in SyntaxContext -> SyntaxContext
+pub trait CtxtFn{
+    fn f(&self, ast::SyntaxContext) -> ast::SyntaxContext;
 }
 
-// update the ctxts in a path to get a rename node
-pub fn new_ident_renamer(from: ast::Ident,
-                      to: ast::Name) ->
-    @fn(ast::Ident)->ast::Ident {
-    |id : ast::Ident|
-    ast::Ident{
-        name: id.name,
-        ctxt: new_rename(from,to,id.ctxt)
+pub struct Renamer {
+    from : ast::Ident,
+    to : ast::Name
+}
+
+impl CtxtFn for Renamer {
+    fn f(&self, ctxt : ast::SyntaxContext) -> ast::SyntaxContext {
+        new_rename(self.from,self.to,ctxt)
     }
 }
 
-// update the ctxts in a path to get a mark node
-pub fn new_ident_marker(mark: Mrk) ->
-    @fn(ast::Ident)->ast::Ident {
-    |id : ast::Ident|
-    ast::Ident{
-        name: id.name,
-        ctxt: new_mark(mark,id.ctxt)
+pub struct Marker { mark : Mrk }
+
+impl CtxtFn for Marker {
+    fn f(&self, ctxt : ast::SyntaxContext) -> ast::SyntaxContext {
+        new_mark(self.mark,ctxt)
+    }
+}
+
+// given a function from ctxts to ctxts, produce
+// an ast_fold that applies that function to all ctxts:
+pub fn fun_to_ctxt_folder<T : 'static + CtxtFn>(cf: @T) -> @AstFoldFns {
+    let afp = default_ast_fold();
+    let fi : @fn(ast::Ident, @ast_fold) -> ast::Ident =
+        |ast::Ident{name, ctxt}, _| {
+        ast::Ident{name:name,ctxt:cf.f(ctxt)}
+    };
+    @AstFoldFns{
+        fold_ident : fi,
+        // check that it works, then add the fold_expr clause....
+        .. *afp
     }
 }
 
+// just a convenience:
+pub fn new_mark_folder(m : Mrk) -> @AstFoldFns { fun_to_ctxt_folder(@Marker{mark:m}) }
+pub fn new_rename_folder(from : ast::Ident, to : ast::Name) -> @AstFoldFns {
+    fun_to_ctxt_folder(@Renamer{from:from,to:to})
+}
+
+/*
 // perform resolution (in the MTWT sense) on all of the
 // idents in the tree. This is the final step in expansion.
 // FIXME #6993: this function could go away, along with
@@ -1465,25 +1479,26 @@ pub fn new_ident_resolver() ->
         ctxt : EMPTY_CTXT
     }
 }
+*/
 
 // apply a given mark to the given token trees. Used prior to expansion of a macro.
 fn mark_tts(tts : &[token_tree], m : Mrk) -> ~[token_tree] {
-    fold_tts(tts,new_ident_marker(m))
+    fold_tts(tts,new_mark_folder(m) as @ast_fold)
 }
 
 // apply a given mark to the given expr. Used following the expansion of a macro.
 fn mark_expr(expr : @ast::Expr, m : Mrk) -> @ast::Expr {
-    fun_to_ident_folder(new_ident_marker(m)).fold_expr(expr)
+    new_mark_folder(m).fold_expr(expr)
 }
 
 // apply a given mark to the given stmt. Used following the expansion of a macro.
 fn mark_stmt(expr : &ast::Stmt, m : Mrk) -> @ast::Stmt {
-    fun_to_ident_folder(new_ident_marker(m)).fold_stmt(expr).unwrap()
+    new_mark_folder(m).fold_stmt(expr).unwrap()
 }
 
 // apply a given mark to the given item. Used following the expansion of a macro.
 fn mark_item(expr : @ast::item, m : Mrk) -> Option<@ast::item> {
-    fun_to_ident_folder(new_ident_marker(m)).fold_item(expr)
+    new_mark_folder(m).fold_item(expr)
 }
 
 #[cfg(test)]
@@ -1499,8 +1514,8 @@ mod test {
     use print::pprust;
     use std;
     use std::vec;
-    use util::parser_testing::{string_to_crate_and_sess, string_to_item, string_to_pat};
-    use util::parser_testing::{strs_to_idents};
+    use util::parser_testing::{string_to_crate, string_to_crate_and_sess, string_to_item};
+    use util::parser_testing::{string_to_pat, strs_to_idents};
     use visit;
 
     // make sure that fail! is present
@@ -1601,30 +1616,31 @@ mod test {
 
     #[test]
     fn renaming () {
-        let item_ast = string_to_item(@"fn a() -> int { let b = 13; b }").unwrap();
+        let item_ast = string_to_crate(@"fn f() -> int { a }");
         let a_name = intern("a");
         let a2_name = gensym("a2");
-        let renamer = new_ident_renamer(ast::Ident{name:a_name,ctxt:EMPTY_CTXT},
+        let renamer = new_rename_folder(ast::Ident{name:a_name,ctxt:EMPTY_CTXT},
                                         a2_name);
-        let renamed_ast = fun_to_ident_folder(renamer).fold_item(item_ast).unwrap();
-        let resolver = new_ident_resolver();
-        let resolver_fold = fun_to_ident_folder(resolver);
-        let resolved_ast = resolver_fold.fold_item(renamed_ast).unwrap();
-        let resolved_as_str = pprust::item_to_str(resolved_ast,
-                                                  get_ident_interner());
-        assert_eq!(resolved_as_str,~"fn a2() -> int { let b = 13; b }");
+        let renamed_ast = renamer.fold_crate(item_ast);
+        let varrefs = @mut ~[];
+        visit::walk_crate(&mut new_path_finder(varrefs), &renamed_ast, ());
+        match varrefs {
+            @[Path{segments:[ref seg],_}] => assert_eq!(mtwt_resolve(seg.identifier),a2_name),
+            _ => assert_eq!(0,1)
+        }
 
         // try a double-rename, with pending_renames.
         let a3_name = gensym("a3");
         let ctxt2 = new_rename(ast::Ident::new(a_name),a2_name,EMPTY_CTXT);
         let pending_renames = @mut ~[(ast::Ident::new(a_name),a2_name),
                                      (ast::Ident{name:a_name,ctxt:ctxt2},a3_name)];
-        let double_renamed = renames_to_fold(pending_renames).fold_item(item_ast).unwrap();
-        let resolved_again = resolver_fold.fold_item(double_renamed).unwrap();
-        let double_renamed_as_str = pprust::item_to_str(resolved_again,
-                                                        get_ident_interner());
-        assert_eq!(double_renamed_as_str,~"fn a3() -> int { let b = 13; b }");
-
+        let double_renamed = renames_to_fold(pending_renames).fold_crate(item_ast);
+        let varrefs = @mut ~[];
+        visit::walk_crate(&mut new_path_finder(varrefs), &double_renamed, ());
+        match varrefs {
+            @[Path{segments:[ref seg],_}] => assert_eq!(mtwt_resolve(seg.identifier),a2_name),
+            _ => assert_eq!(0,1)
+        }
     }
 
     fn fake_print_crate(s: @pprust::ps, crate: &ast::Crate) {
diff --git a/src/libsyntax/fold.rs b/src/libsyntax/fold.rs
index 799ff855cbe..9aacc18eabf 100644
--- a/src/libsyntax/fold.rs
+++ b/src/libsyntax/fold.rs
@@ -118,16 +118,16 @@ fn fold_mac_(m: &mac, fld: @ast_fold) -> mac {
         node: match m.node {
             mac_invoc_tt(ref p,ref tts,ctxt) =>
             mac_invoc_tt(fld.fold_path(p),
-                         fold_tts(*tts,|id|{fld.fold_ident(id)}),
+                         fold_tts(*tts,fld),
                          ctxt)
         },
         span: fld.new_span(m.span)
     }
 }
 
-// build a new vector of tts by appling the given function to
+// build a new vector of tts by appling the ast_fold's fold_ident to
 // all of the identifiers in the token trees.
-pub fn fold_tts(tts : &[token_tree], f: @fn(Ident)->Ident) -> ~[token_tree] {
+pub fn fold_tts(tts : &[token_tree], f : @ast_fold) -> ~[token_tree] {
     do tts.map |tt| {
         match *tt {
             tt_tok(span, ref tok) =>
@@ -140,16 +140,16 @@ pub fn fold_tts(tts : &[token_tree], f: @fn(Ident)->Ident) -> ~[token_tree] {
                    sep.map(|tok|maybe_fold_ident(tok,f)),
                    is_optional),
             tt_nonterminal(sp,ref ident) =>
-            tt_nonterminal(sp,f(*ident))
+            tt_nonterminal(sp,f.fold_ident(*ident))
         }
     }
 }
 
 // apply ident folder if it's an ident, otherwise leave it alone
-fn maybe_fold_ident(t : &token::Token, f: @fn(Ident)->Ident) -> token::Token {
+fn maybe_fold_ident(t : &token::Token, f: @ast_fold) -> token::Token {
     match *t {
         token::IDENT(id,followed_by_colons) =>
-        token::IDENT(f(id),followed_by_colons),
+        token::IDENT(f.fold_ident(id),followed_by_colons),
         _ => (*t).clone()
     }
 }
@@ -948,6 +948,8 @@ impl AstFoldExtensions for @ast_fold {
     }
 }
 
+// brson agrees with me that this function's existence is probably
+// not a good or useful thing.
 pub fn make_fold(afp: ast_fold_fns) -> @ast_fold {
     afp as @ast_fold
 }
-- 
cgit 1.4.1-3-g733a5


From 3261b6eece84161355bac8dad8427af5221cbf0d Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Fri, 5 Jul 2013 13:57:53 -0700
Subject: WIP: adding context to macros

---
 src/libsyntax/ast.rs        |  2 +-
 src/libsyntax/ext/expand.rs | 24 +++++++++++++++---------
 2 files changed, 16 insertions(+), 10 deletions(-)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs
index c2f257643cd..31729493815 100644
--- a/src/libsyntax/ast.rs
+++ b/src/libsyntax/ast.rs
@@ -472,7 +472,7 @@ pub enum BlockCheckMode {
     UnsafeBlock,
 }
 
-#[deriving(Eq, Encodable, Decodable,IterBytes)]
+#[deriving(Clone, Eq, Encodable, Decodable,IterBytes)]
 pub struct Expr {
     id: NodeId,
     node: Expr_,
diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index c73f8b01b9c..228a9ba03a2 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -603,7 +603,7 @@ fn expand_non_macro_stmt (exts: SyntaxEnv,
 }
 
 // a visitor that extracts the pat_ident paths
-// from a given pattern and puts them in a mutable
+// from a given thingy and puts them in a mutable
 // array (passed in to the traversal)
 #[deriving(Clone)]
 struct NewNameFinderContext {
@@ -748,15 +748,14 @@ impl Visitor<()> for NewNameFinderContext {
     }
 }
 
-// a visitor that extracts the path exprs from
-// a crate/expression/whatever and puts them in a mutable
+// a visitor that extracts the paths
+// from a given thingy and puts them in a mutable
 // array (passed in to the traversal)
 #[deriving(Clone)]
 struct NewPathExprFinderContext {
     path_accumulator: @mut ~[ast::Path],
 }
 
-
 // XXX : YIKES a lot of boilerplate again....
 impl Visitor<()> for NewPathExprFinderContext {
 
@@ -875,7 +874,7 @@ impl Visitor<()> for NewPathExprFinderContext {
 }
 
 // return a visitor that extracts the pat_ident paths
-// from a given pattern and puts them in a mutable
+// from a given thingy and puts them in a mutable
 // array (passed in to the traversal)
 pub fn new_name_finder(idents: @mut ~[ast::Ident]) -> @mut Visitor<()> {
     let context = @mut NewNameFinderContext {
@@ -913,6 +912,7 @@ pub fn renames_to_fold(renames : @mut ~[(ast::Ident,ast::Name)]) -> @ast_fold {
     make_fold(f_pre)
 }
 
+// expand a block. pushes a new exts_frame, then calls expand_block_elts
 pub fn expand_block(extsbox: @mut SyntaxEnv,
                     _cx: @ExtCtxt,
                     blk: &Block,
@@ -924,7 +924,7 @@ pub fn expand_block(extsbox: @mut SyntaxEnv,
                      expand_block_elts(*extsbox, blk, fld))
 }
 
-
+// expand the elements of a block.
 pub fn expand_block_elts(exts: SyntaxEnv, b: &Block, fld: @ast_fold) -> Block {
     let block_info = get_block_info(exts);
     let pending_renames = block_info.pending_renames;
@@ -949,6 +949,7 @@ pub fn expand_block_elts(exts: SyntaxEnv, b: &Block, fld: @ast_fold) -> Block {
 }
 
 // rename_fold should never return "None".
+// (basically, just .get() with a better message...)
 fn mustbesome<T>(val : Option<T>) -> T {
     match val {
         Some(v) => v,
@@ -971,8 +972,8 @@ pub fn new_span(cx: @ExtCtxt, sp: Span) -> Span {
 }
 
 // FIXME (#2247): this is a moderately bad kludge to inject some macros into
-// the default compilation environment. It would be much nicer to use
-// a mechanism like syntax_quote to ensure hygiene.
+// the default compilation environment in that it injects strings, rather than
+// syntax elements.
 
 pub fn std_macros() -> @str {
     return
@@ -1453,6 +1454,10 @@ pub fn fun_to_ctxt_folder<T : 'static + CtxtFn>(cf: @T) -> @AstFoldFns {
         |ast::Ident{name, ctxt}, _| {
         ast::Ident{name:name,ctxt:cf.f(ctxt)}
     };
+    // we've also got to pick up macro invocations; they can
+    // appear as exprs, stmts, items, and types. urg, it's going
+    // to be easier just to add a fold_mac, I think.
+    //let fold_ex : @
     @AstFoldFns{
         fold_ident : fi,
         // check that it works, then add the fold_expr clause....
@@ -1631,6 +1636,7 @@ mod test {
 
         // try a double-rename, with pending_renames.
         let a3_name = gensym("a3");
+        // a context that renames from ("a",empty) to "a2" :
         let ctxt2 = new_rename(ast::Ident::new(a_name),a2_name,EMPTY_CTXT);
         let pending_renames = @mut ~[(ast::Ident::new(a_name),a2_name),
                                      (ast::Ident{name:a_name,ctxt:ctxt2},a3_name)];
@@ -1638,7 +1644,7 @@ mod test {
         let varrefs = @mut ~[];
         visit::walk_crate(&mut new_path_finder(varrefs), &double_renamed, ());
         match varrefs {
-            @[Path{segments:[ref seg],_}] => assert_eq!(mtwt_resolve(seg.identifier),a2_name),
+            @[Path{segments:[ref seg],_}] => assert_eq!(mtwt_resolve(seg.identifier),a3_name),
             _ => assert_eq!(0,1)
         }
     }
-- 
cgit 1.4.1-3-g733a5


From dfa04cecca1d7128cc4d8e3868f0b237f6266911 Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Fri, 5 Jul 2013 14:16:39 -0700
Subject: add fold_mac field to fold.rs

---
 src/libsyntax/fold.rs | 62 +++++++++++++++++++++++++++++++--------------------
 1 file changed, 38 insertions(+), 24 deletions(-)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/fold.rs b/src/libsyntax/fold.rs
index 9aacc18eabf..823e018a718 100644
--- a/src/libsyntax/fold.rs
+++ b/src/libsyntax/fold.rs
@@ -14,6 +14,14 @@ use codemap::{Span, Spanned};
 use parse::token;
 use opt_vec::OptVec;
 
+// this file defines an ast_fold trait for objects that can perform
+// a "fold" on Rust ASTs. It also contains a structure that implements
+// that trait, and a "default_fold" whose fields contain closures
+// that perform "default traversals", visiting all of the sub-elements
+// and re-assembling the result. The "fun_to_ident_folder" in the
+// test module provides a simple example of creating a very simple
+// fold that only looks at identifiers.
+
 pub trait ast_fold {
     fn fold_crate(@self, &Crate) -> Crate;
     fn fold_view_item(@self, &view_item) -> view_item;
@@ -35,6 +43,7 @@ pub trait ast_fold {
     fn fold_ident(@self, Ident) -> Ident;
     fn fold_path(@self, &Path) -> Path;
     fn fold_local(@self, @Local) -> @Local;
+    fn fold_mac(@self, &mac) -> mac;
     fn map_exprs(@self, @fn(@Expr) -> @Expr, &[@Expr]) -> ~[@Expr];
     fn new_id(@self, NodeId) -> NodeId;
     fn new_span(@self, Span) -> Span;
@@ -64,6 +73,7 @@ pub struct AstFoldFns {
     fold_ident: @fn(Ident, @ast_fold) -> Ident,
     fold_path: @fn(&Path, @ast_fold) -> Path,
     fold_local: @fn(@Local, @ast_fold) -> @Local,
+    fold_mac: @fn(&mac_, Span, @ast_fold) -> (mac_, Span),
     map_exprs: @fn(@fn(@Expr) -> @Expr, &[@Expr]) -> ~[@Expr],
     new_id: @fn(NodeId) -> NodeId,
     new_span: @fn(Span) -> Span
@@ -112,19 +122,6 @@ fn fold_arg_(a: arg, fld: @ast_fold) -> arg {
     }
 }
 
-//used in noop_fold_expr, and possibly elsewhere in the future
-fn fold_mac_(m: &mac, fld: @ast_fold) -> mac {
-    Spanned {
-        node: match m.node {
-            mac_invoc_tt(ref p,ref tts,ctxt) =>
-            mac_invoc_tt(fld.fold_path(p),
-                         fold_tts(*tts,fld),
-                         ctxt)
-        },
-        span: fld.new_span(m.span)
-    }
-}
-
 // build a new vector of tts by appling the ast_fold's fold_ident to
 // all of the identifiers in the token trees.
 pub fn fold_tts(tts : &[token_tree], f : @ast_fold) -> ~[token_tree] {
@@ -326,10 +323,7 @@ pub fn noop_fold_item_underscore(i: &item_, fld: @ast_fold) -> item_ {
             )
         }
         item_mac(ref m) => {
-            // It would probably be nicer
-            // to expose this in the ast_fold trait, but I'll defer
-            // that work.
-            item_mac(fold_mac_(m,fld))
+            item_mac(fld.fold_mac(m))
         }
     }
 }
@@ -398,7 +392,6 @@ pub fn noop_fold_block(b: &Block, fld: @ast_fold) -> Block {
 }
 
 fn noop_fold_stmt(s: &Stmt_, fld: @ast_fold) -> Option<Stmt_> {
-    let fold_mac = |x| fold_mac_(x, fld);
     match *s {
         StmtDecl(d, nid) => {
             match fld.fold_decl(d) {
@@ -412,7 +405,7 @@ fn noop_fold_stmt(s: &Stmt_, fld: @ast_fold) -> Option<Stmt_> {
         StmtSemi(e, nid) => {
             Some(StmtSemi(fld.fold_expr(e), fld.new_id(nid)))
         }
-        StmtMac(ref mac, semi) => Some(StmtMac(fold_mac(mac), semi))
+        StmtMac(ref mac, semi) => Some(StmtMac(fld.fold_mac(mac), semi))
     }
 }
 
@@ -480,6 +473,12 @@ fn noop_fold_decl(d: &Decl_, fld: @ast_fold) -> Option<Decl_> {
     }
 }
 
+// lift a function in ast-thingy X fold -> ast-thingy to a function
+// in (ast-thingy X span X fold) -> (ast-thingy X fold). Basically,
+// carries the span around.
+// It seems strange to me that the call to new_fold doesn't happen
+// here but instead in the impl down below.... probably just an
+// accident?
 pub fn wrap<T>(f: @fn(&T, @ast_fold) -> T)
             -> @fn(&T, Span, @ast_fold) -> (T, Span) {
     let result: @fn(&T, Span, @ast_fold) -> (T, Span) = |x, s, fld| {
@@ -498,8 +497,6 @@ pub fn noop_fold_expr(e: &Expr_, fld: @ast_fold) -> Expr_ {
     }
     let fold_field = |x| fold_field_(x, fld);
 
-    let fold_mac = |x| fold_mac_(x, fld);
-
     match *e {
         ExprVstore(e, v) => {
             ExprVstore(fld.fold_expr(e), v)
@@ -631,7 +628,7 @@ pub fn noop_fold_expr(e: &Expr_, fld: @ast_fold) -> Expr_ {
                 .. (*a).clone()
             })
         }
-        ExprMac(ref mac) => ExprMac(fold_mac(mac)),
+        ExprMac(ref mac) => ExprMac(fld.fold_mac(mac)),
         ExprStruct(ref path, ref fields, maybe_expr) => {
             ExprStruct(
                 fld.fold_path(path),
@@ -644,7 +641,6 @@ pub fn noop_fold_expr(e: &Expr_, fld: @ast_fold) -> Expr_ {
 }
 
 pub fn noop_fold_ty(t: &ty_, fld: @ast_fold) -> ty_ {
-    let fold_mac = |x| fold_mac_(x, fld);
     fn fold_mt(mt: &mt, fld: @ast_fold) -> mt {
         mt {
             ty: ~fld.fold_ty(mt.ty),
@@ -700,7 +696,7 @@ pub fn noop_fold_ty(t: &ty_, fld: @ast_fold) -> ty_ {
             )
         }
         ty_typeof(e) => ty_typeof(fld.fold_expr(e)),
-        ty_mac(ref mac) => ty_mac(fold_mac(mac))
+        ty_mac(ref mac) => ty_mac(fld.fold_mac(mac))
     }
 }
 
@@ -787,6 +783,19 @@ fn noop_fold_local(l: @Local, fld: @ast_fold) -> @Local {
     }
 }
 
+// the default macro traversal. visit the path
+// using fold_path, and the tts using fold_tts,
+// and the span using new_span
+fn noop_fold_mac(m: &mac_, fld: @ast_fold) -> mac_ {
+    match *m {
+        mac_invoc_tt(ref p,ref tts,ctxt) =>
+        mac_invoc_tt(fld.fold_path(p),
+                     fold_tts(*tts,fld),
+                     ctxt)
+    }
+}
+
+
 /* temporarily eta-expand because of a compiler bug with using `fn<T>` as a
    value */
 fn noop_map_exprs(f: @fn(@Expr) -> @Expr, es: &[@Expr]) -> ~[@Expr] {
@@ -819,6 +828,7 @@ pub fn default_ast_fold() -> ast_fold_fns {
         fold_ident: noop_fold_ident,
         fold_path: noop_fold_path,
         fold_local: noop_fold_local,
+        fold_mac: wrap(noop_fold_mac),
         map_exprs: noop_map_exprs,
         new_id: noop_id,
         new_span: noop_span,
@@ -924,6 +934,10 @@ impl ast_fold for AstFoldFns {
     fn fold_local(@self, x: @Local) -> @Local {
         (self.fold_local)(x, self as @ast_fold)
     }
+    fn fold_mac(@self, x: &mac) -> mac {
+        let (n, s) = (self.fold_mac)(&x.node, x.span, self as @ast_fold);
+        Spanned { node: n, span: (self.new_span)(s) }
+    }
     fn map_exprs(@self,
                  f: @fn(@Expr) -> @Expr,
                  e: &[@Expr])
-- 
cgit 1.4.1-3-g733a5


From 2c51e262f367f0f079135a5a176706ea5ef78f78 Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Mon, 8 Jul 2013 10:37:07 -0700
Subject: add fold_mac clause to fun_to_ctxt_folder

---
 src/libsyntax/ext/expand.rs | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index 228a9ba03a2..a62b015c6cb 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -1454,13 +1454,20 @@ pub fn fun_to_ctxt_folder<T : 'static + CtxtFn>(cf: @T) -> @AstFoldFns {
         |ast::Ident{name, ctxt}, _| {
         ast::Ident{name:name,ctxt:cf.f(ctxt)}
     };
-    // we've also got to pick up macro invocations; they can
-    // appear as exprs, stmts, items, and types. urg, it's going
-    // to be easier just to add a fold_mac, I think.
-    //let fold_ex : @
+    let fm : @fn(&ast::mac_, span, @ast_fold) -> (ast::mac_,span) =
+        |m, sp, fld| {
+        match *m {
+            mac_invoc_tt(ref path, ref tts, ctxt) =>
+            (mac_invoc_tt(fld.fold_path(path),
+                         fold_tts(*tts,fld),
+                         cf.f(ctxt)),
+            sp)
+        }
+
+    };
     @AstFoldFns{
         fold_ident : fi,
-        // check that it works, then add the fold_expr clause....
+        fold_mac : fm,
         .. *afp
     }
 }
-- 
cgit 1.4.1-3-g733a5


From b9bb4abcb66804e4b78704068703f0dab5c72887 Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Mon, 8 Jul 2013 15:55:14 -0700
Subject: capturing macros now implemented

---
 src/libsyntax/ext/base.rs           |  92 ++++++++++++++--------
 src/libsyntax/ext/expand.rs         | 151 +++++++++++++++++++++++++++---------
 src/libsyntax/ext/quote.rs          |  59 ++++++++++----
 src/libsyntax/ext/tt/macro_rules.rs |   4 +-
 src/libsyntax/fold.rs               |   1 +
 5 files changed, 222 insertions(+), 85 deletions(-)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/ext/base.rs b/src/libsyntax/ext/base.rs
index ac3fc717500..24f70bfeaa8 100644
--- a/src/libsyntax/ext/base.rs
+++ b/src/libsyntax/ext/base.rs
@@ -33,6 +33,10 @@ pub struct MacroDef {
     ext: SyntaxExtension
 }
 
+// No context arg for an Item Decorator macro, simply because
+// adding it would require adding a ctxt field to all items.
+// we could do this if it turns out to be useful.
+
 pub type ItemDecoratorFun = @fn(@ExtCtxt,
                              Span,
                              @ast::MetaItem,
@@ -41,15 +45,34 @@ pub type ItemDecoratorFun = @fn(@ExtCtxt,
 
 pub type SyntaxExpanderTTFun = @fn(@ExtCtxt,
                                    Span,
-                                   &[ast::token_tree])
+                                   &[ast::token_tree],
+                                   ast::SyntaxContext)
                                 -> MacResult;
 
 pub type SyntaxExpanderTTItemFun = @fn(@ExtCtxt,
+                                       Span,
+                                       ast::Ident,
+                                       ~[ast::token_tree],
+                                       ast::SyntaxContext)
+                                    -> MacResult;
+
+// oog... in order to make the presentation of builtin_normal_tt_no_ctxt
+// and builtin_ident_tt_no_ctxt palatable, we need one-off types for
+// functions that don't consume a ctxt:
+
+pub type SyntaxExpanderTTFunNoCtxt = @fn(@ExtCtxt,
+                                   Span,
+                                   &[ast::token_tree])
+                                -> MacResult;
+
+pub type SyntaxExpanderTTItemFunNoCtxt = @fn(@ExtCtxt,
                                        Span,
                                        ast::Ident,
                                        ~[ast::token_tree])
                                     -> MacResult;
 
+
+
 pub enum MacResult {
     MRExpr(@ast::Expr),
     MRItem(@ast::item),
@@ -78,6 +101,7 @@ pub enum SyntaxExtension {
     IdentTT(SyntaxExpanderTTItemFun, Option<Span>),
 }
 
+
 // The SyntaxEnv is the environment that's threaded through the expansion
 // of macros. It contains bindings for macros, and also a special binding
 // for " block" (not a legal identifier) that maps to a BlockInfo
@@ -109,12 +133,16 @@ type RenameList = ~[(ast::Ident,Name)];
 // AST nodes into full ASTs
 pub fn syntax_expander_table() -> SyntaxEnv {
     // utility function to simplify creating NormalTT syntax extensions
-    fn builtin_normal_tt(f: SyntaxExpanderTTFun) -> @Transformer {
-        @SE(NormalTT(f, None))
+    // that ignore their contexts
+    fn builtin_normal_tt_no_ctxt(f: SyntaxExpanderTTFunNoCtxt) -> @Transformer {
+        let wrapped_expander : SyntaxExpanderTTFun = |a,b,c,d|{f(a,b,c)};
+        @SE(NormalTT(wrapped_expander, None))
     }
     // utility function to simplify creating IdentTT syntax extensions
-    fn builtin_item_tt(f: SyntaxExpanderTTItemFun) -> @Transformer {
-        @SE(IdentTT(f, None))
+    // that ignore their contexts
+    fn builtin_item_tt_no_ctxt(f: SyntaxExpanderTTItemFunNoCtxt) -> @Transformer {
+        let wrapped_expander : SyntaxExpanderTTItemFun = |a,b,c,d,e|{f(a,b,c,d)};
+        @SE(IdentTT(wrapped_expander, None))
     }
     let mut syntax_expanders = HashMap::new();
     // NB identifier starts with space, and can't conflict with legal idents
@@ -124,16 +152,16 @@ pub fn syntax_expander_table() -> SyntaxEnv {
                                 pending_renames : @mut ~[]
                             }));
     syntax_expanders.insert(intern(&"macro_rules"),
-                            builtin_item_tt(
+                            builtin_item_tt_no_ctxt(
                                 ext::tt::macro_rules::add_new_extension));
     syntax_expanders.insert(intern(&"fmt"),
-                            builtin_normal_tt(ext::fmt::expand_syntax_ext));
+                            builtin_normal_tt_no_ctxt(ext::fmt::expand_syntax_ext));
     syntax_expanders.insert(intern(&"format"),
-                            builtin_normal_tt(ext::ifmt::expand_format));
+                            builtin_normal_tt_no_ctxt(ext::ifmt::expand_format));
     syntax_expanders.insert(intern(&"write"),
-                            builtin_normal_tt(ext::ifmt::expand_write));
+                            builtin_normal_tt_no_ctxt(ext::ifmt::expand_write));
     syntax_expanders.insert(intern(&"writeln"),
-                            builtin_normal_tt(ext::ifmt::expand_writeln));
+                            builtin_normal_tt_no_ctxt(ext::ifmt::expand_writeln));
     syntax_expanders.insert(
         intern(&"auto_encode"),
         @SE(ItemDecorator(ext::auto_encode::expand_auto_encode)));
@@ -141,16 +169,16 @@ pub fn syntax_expander_table() -> SyntaxEnv {
         intern(&"auto_decode"),
         @SE(ItemDecorator(ext::auto_encode::expand_auto_decode)));
     syntax_expanders.insert(intern(&"env"),
-                            builtin_normal_tt(ext::env::expand_env));
+                            builtin_normal_tt_no_ctxt(ext::env::expand_env));
     syntax_expanders.insert(intern(&"option_env"),
-                            builtin_normal_tt(ext::env::expand_option_env));
+                            builtin_normal_tt_no_ctxt(ext::env::expand_option_env));
     syntax_expanders.insert(intern("bytes"),
-                            builtin_normal_tt(ext::bytes::expand_syntax_ext));
+                            builtin_normal_tt_no_ctxt(ext::bytes::expand_syntax_ext));
     syntax_expanders.insert(intern("concat_idents"),
-                            builtin_normal_tt(
+                            builtin_normal_tt_no_ctxt(
                                 ext::concat_idents::expand_syntax_ext));
     syntax_expanders.insert(intern(&"log_syntax"),
-                            builtin_normal_tt(
+                            builtin_normal_tt_no_ctxt(
                                 ext::log_syntax::expand_syntax_ext));
     syntax_expanders.insert(intern(&"deriving"),
                             @SE(ItemDecorator(
@@ -158,49 +186,49 @@ pub fn syntax_expander_table() -> SyntaxEnv {
 
     // Quasi-quoting expanders
     syntax_expanders.insert(intern(&"quote_tokens"),
-                       builtin_normal_tt(ext::quote::expand_quote_tokens));
+        @SE(NormalTT(ext::quote::expand_quote_tokens, None)));
     syntax_expanders.insert(intern(&"quote_expr"),
-                       builtin_normal_tt(ext::quote::expand_quote_expr));
+        @SE(NormalTT(ext::quote::expand_quote_expr, None)));
     syntax_expanders.insert(intern(&"quote_ty"),
-                       builtin_normal_tt(ext::quote::expand_quote_ty));
+        @SE(NormalTT(ext::quote::expand_quote_ty, None)));
     syntax_expanders.insert(intern(&"quote_item"),
-                       builtin_normal_tt(ext::quote::expand_quote_item));
+        @SE(NormalTT(ext::quote::expand_quote_item, None)));
     syntax_expanders.insert(intern(&"quote_pat"),
-                       builtin_normal_tt(ext::quote::expand_quote_pat));
+        @SE(NormalTT(ext::quote::expand_quote_pat, None)));
     syntax_expanders.insert(intern(&"quote_stmt"),
-                       builtin_normal_tt(ext::quote::expand_quote_stmt));
+        @SE(NormalTT(ext::quote::expand_quote_stmt, None)));
 
     syntax_expanders.insert(intern(&"line"),
-                            builtin_normal_tt(
+                            builtin_normal_tt_no_ctxt(
                                 ext::source_util::expand_line));
     syntax_expanders.insert(intern(&"col"),
-                            builtin_normal_tt(
+                            builtin_normal_tt_no_ctxt(
                                 ext::source_util::expand_col));
     syntax_expanders.insert(intern(&"file"),
-                            builtin_normal_tt(
+                            builtin_normal_tt_no_ctxt(
                                 ext::source_util::expand_file));
     syntax_expanders.insert(intern(&"stringify"),
-                            builtin_normal_tt(
+                            builtin_normal_tt_no_ctxt(
                                 ext::source_util::expand_stringify));
     syntax_expanders.insert(intern(&"include"),
-                            builtin_normal_tt(
+                            builtin_normal_tt_no_ctxt(
                                 ext::source_util::expand_include));
     syntax_expanders.insert(intern(&"include_str"),
-                            builtin_normal_tt(
+                            builtin_normal_tt_no_ctxt(
                                 ext::source_util::expand_include_str));
     syntax_expanders.insert(intern(&"include_bin"),
-                            builtin_normal_tt(
+                            builtin_normal_tt_no_ctxt(
                                 ext::source_util::expand_include_bin));
     syntax_expanders.insert(intern(&"module_path"),
-                            builtin_normal_tt(
+                            builtin_normal_tt_no_ctxt(
                                 ext::source_util::expand_mod));
     syntax_expanders.insert(intern(&"asm"),
-                            builtin_normal_tt(ext::asm::expand_asm));
+                            builtin_normal_tt_no_ctxt(ext::asm::expand_asm));
     syntax_expanders.insert(intern(&"cfg"),
-                            builtin_normal_tt(ext::cfg::expand_cfg));
+                            builtin_normal_tt_no_ctxt(ext::cfg::expand_cfg));
     syntax_expanders.insert(
         intern(&"trace_macros"),
-        builtin_normal_tt(ext::trace_macros::expand_trace_macros));
+        builtin_normal_tt_no_ctxt(ext::trace_macros::expand_trace_macros));
     MapChain::new(~syntax_expanders)
 }
 
diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index a62b015c6cb..91a56590435 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -8,7 +8,7 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use ast::{Block, Crate, NodeId, DeclLocal, EMPTY_CTXT, Expr_, ExprMac};
+use ast::{Block, Crate, NodeId, DeclLocal, EMPTY_CTXT, Expr_, ExprMac, SyntaxContext};
 use ast::{Local, Ident, mac_invoc_tt};
 use ast::{item_mac, Mrk, Stmt_, StmtDecl, StmtMac, StmtExpr, StmtSemi};
 use ast::{ILLEGAL_CTXT, SCTable, token_tree};
@@ -42,6 +42,11 @@ pub fn expand_expr(extsbox: @mut SyntaxEnv,
         // entry-point for all syntax extensions.
         ExprMac(ref mac) => {
             match (*mac).node {
+                // it would almost certainly be cleaner to pass the whole
+                // macro invocation in, rather than pulling it apart and
+                // marking the tts and the ctxt separately. This also goes
+                // for the other three macro invocation chunks of code
+                // in this file.
                 // Token-tree macros:
                 mac_invoc_tt(ref pth, ref tts, ctxt) => {
                     if (pth.segments.len() > 1u) {
@@ -70,7 +75,8 @@ pub fn expand_expr(extsbox: @mut SyntaxEnv,
                             let fm = fresh_mark();
                             // mark before:
                             let marked_before = mark_tts(*tts,fm);
-                            let expanded = match expandfun(cx, mac.span, marked_before) {
+                            let marked_ctxt = new_mark(fm, ctxt);
+                            let expanded = match expandfun(cx, mac.span, marked_before, marked_ctxt) {
                                 MRExpr(e) => e,
                                 MRAny(expr_maker,_,_) => expr_maker(),
                                 _ => {
@@ -367,9 +373,9 @@ pub fn expand_item_mac(extsbox: @mut SyntaxEnv,
                        cx: @ExtCtxt, it: @ast::item,
                        fld: @ast_fold)
                     -> Option<@ast::item> {
-    let (pth, tts) = match it.node {
+    let (pth, tts, ctxt) = match it.node {
         item_mac(codemap::Spanned { node: mac_invoc_tt(ref pth, ref tts, ctxt), _}) => {
-            (pth, (*tts).clone())
+            (pth, (*tts).clone(), ctxt)
         }
         _ => cx.span_bug(it.span, "invalid item macro invocation")
     };
@@ -396,10 +402,9 @@ pub fn expand_item_mac(extsbox: @mut SyntaxEnv,
                 }
             });
             // mark before expansion:
-            let marked_tts = mark_tts(tts,fm);
-            // mark after expansion:
-            // RIGHT HERE: can't apply mark_item to MacResult ... :(
-            expander(cx, it.span, marked_tts)
+            let marked_before = mark_tts(tts,fm);
+            let marked_ctxt = new_mark(fm,ctxt);
+            expander(cx, it.span, marked_before, marked_ctxt)
         }
         Some(@SE(IdentTT(expander, span))) => {
             if it.ident.name == parse::token::special_idents::invalid.name {
@@ -417,7 +422,8 @@ pub fn expand_item_mac(extsbox: @mut SyntaxEnv,
             let fm = fresh_mark();
             // mark before expansion:
             let marked_tts = mark_tts(tts,fm);
-        expander(cx, it.span, it.ident, marked_tts)
+            let marked_ctxt = new_mark(fm,ctxt);
+            expander(cx, it.span, it.ident, marked_tts, marked_ctxt)
         }
         _ => cx.span_fatal(
             it.span, fmt!("%s! is not legal in item position", extnamestr))
@@ -468,11 +474,11 @@ pub fn expand_stmt(extsbox: @mut SyntaxEnv,
                 -> (Option<Stmt_>, Span) {
     // why the copying here and not in expand_expr?
     // looks like classic changed-in-only-one-place
-    let (mac, pth, tts, semi) = match *s {
+    let (mac, pth, tts, semi, ctxt) = match *s {
         StmtMac(ref mac, semi) => {
             match mac.node {
                 mac_invoc_tt(ref pth, ref tts, ctxt) => {
-                    ((*mac).clone(), pth, (*tts).clone(), semi)
+                    ((*mac).clone(), pth, (*tts).clone(), semi, ctxt)
                 }
             }
         }
@@ -498,7 +504,8 @@ pub fn expand_stmt(extsbox: @mut SyntaxEnv,
             let fm = fresh_mark();
             // mark before expansion:
             let marked_tts = mark_tts(tts,fm);
-            let expanded = match expandfun(cx, mac.span, marked_tts) {
+            let marked_ctxt = new_mark(fm,ctxt);
+            let expanded = match expandfun(cx, mac.span, marked_tts, marked_ctxt) {
                 MRExpr(e) =>
                     @codemap::Spanned { node: StmtExpr(e, cx.next_id()),
                                     span: e.span},
@@ -893,25 +900,6 @@ pub fn new_path_finder(paths: @mut ~[ast::Path]) -> @mut Visitor<()> {
     context as @mut Visitor<()>
 }
 
-// given a mutable list of renames, return a tree-folder that applies those
-// renames.
-// FIXME #4536: currently pub to allow testing
-pub fn renames_to_fold(renames : @mut ~[(ast::Ident,ast::Name)]) -> @ast_fold {
-    let afp = default_ast_fold();
-    let f_pre = @AstFoldFns {
-        fold_ident: |id,_| {
-            // the individual elements are memoized... it would
-            // also be possible to memoize on the whole list at once.
-            let new_ctxt = renames.iter().fold(id.ctxt,|ctxt,&(from,to)| {
-                new_rename(from,to,ctxt)
-            });
-            ast::Ident{name:id.name,ctxt:new_ctxt}
-        },
-        .. *afp
-    };
-    make_fold(f_pre)
-}
-
 // expand a block. pushes a new exts_frame, then calls expand_block_elts
 pub fn expand_block(extsbox: @mut SyntaxEnv,
                     _cx: @ExtCtxt,
@@ -1427,6 +1415,7 @@ pub trait CtxtFn{
     fn f(&self, ast::SyntaxContext) -> ast::SyntaxContext;
 }
 
+// a renamer adds a rename to the syntax context
 pub struct Renamer {
     from : ast::Ident,
     to : ast::Name
@@ -1438,6 +1427,22 @@ impl CtxtFn for Renamer {
     }
 }
 
+// a renamer that performs a whole bunch of renames
+pub struct MultiRenamer {
+    renames : @mut ~[(ast::Ident,ast::Name)]
+}
+
+impl CtxtFn for MultiRenamer {
+    fn f(&self, starting_ctxt : ast::SyntaxContext) -> ast::SyntaxContext {
+        // the individual elements are memoized... it would
+        // also be possible to memoize on the whole list at once.
+        self.renames.iter().fold(starting_ctxt,|ctxt,&(from,to)| {
+            new_rename(from,to,ctxt)
+        })
+    }
+}
+
+// a marker adds the given mark to the syntax context
 pub struct Marker { mark : Mrk }
 
 impl CtxtFn for Marker {
@@ -1446,6 +1451,15 @@ impl CtxtFn for Marker {
     }
 }
 
+// a repainter just replaces the given context with the one it's closed over
+pub struct Repainter { ctxt : SyntaxContext }
+
+impl CtxtFn for Repainter {
+    fn f(&self, ctxt : ast::SyntaxContext) -> ast::SyntaxContext {
+        self.ctxt
+    }
+}
+
 // given a function from ctxts to ctxts, produce
 // an ast_fold that applies that function to all ctxts:
 pub fn fun_to_ctxt_folder<T : 'static + CtxtFn>(cf: @T) -> @AstFoldFns {
@@ -1454,7 +1468,7 @@ pub fn fun_to_ctxt_folder<T : 'static + CtxtFn>(cf: @T) -> @AstFoldFns {
         |ast::Ident{name, ctxt}, _| {
         ast::Ident{name:name,ctxt:cf.f(ctxt)}
     };
-    let fm : @fn(&ast::mac_, span, @ast_fold) -> (ast::mac_,span) =
+    let fm : @fn(&ast::mac_, Span, @ast_fold) -> (ast::mac_,Span) =
         |m, sp, fld| {
         match *m {
             mac_invoc_tt(ref path, ref tts, ctxt) =>
@@ -1472,6 +1486,15 @@ pub fn fun_to_ctxt_folder<T : 'static + CtxtFn>(cf: @T) -> @AstFoldFns {
     }
 }
 
+
+
+// given a mutable list of renames, return a tree-folder that applies those
+// renames.
+// FIXME #4536: currently pub to allow testing
+pub fn renames_to_fold(renames : @mut ~[(ast::Ident,ast::Name)]) -> @AstFoldFns {
+    fun_to_ctxt_folder(@MultiRenamer{renames : renames})
+}
+
 // just a convenience:
 pub fn new_mark_folder(m : Mrk) -> @AstFoldFns { fun_to_ctxt_folder(@Marker{mark:m}) }
 pub fn new_rename_folder(from : ast::Ident, to : ast::Name) -> @AstFoldFns {
@@ -1513,6 +1536,12 @@ fn mark_item(expr : @ast::item, m : Mrk) -> Option<@ast::item> {
     new_mark_folder(m).fold_item(expr)
 }
 
+// replace all contexts in a given expr with the given mark. Used
+// for capturing macros
+pub fn replace_ctxts(expr : @ast::Expr, ctxt : SyntaxContext) -> @ast::Expr {
+    fun_to_ctxt_folder(@Repainter{ctxt:ctxt}).fold_expr(expr)
+}
+
 #[cfg(test)]
 mod test {
     use super::*;
@@ -1522,7 +1551,7 @@ mod test {
     use codemap;
     use codemap::Spanned;
     use parse;
-    use parse::token::{gensym, intern, get_ident_interner};
+    use parse::token::{gensym, intern, get_ident_interner, ident_to_str};
     use print::pprust;
     use std;
     use std::vec;
@@ -1656,8 +1685,9 @@ mod test {
         }
     }
 
-    fn fake_print_crate(s: @pprust::ps, crate: &ast::Crate) {
-        pprust::print_mod(s, &crate.module, crate.attrs);
+    fn fake_print_crate(crate: @ast::Crate) {
+        let s = pprust::rust_printer(std::io::stderr(),get_ident_interner());
+        pprust::print_crate_(s, crate);
     }
 
     fn expand_crate_str(crate_str: @str) -> @ast::Crate {
@@ -1784,6 +1814,57 @@ mod test {
         }
     }
 
+    #[test] fn quote_expr_test() {
+        quote_ext_cx_test(@"fn main(){let ext_cx = 13; quote_expr!(dontcare);}");
+    }
+    #[test] fn quote_item_test() {
+        quote_ext_cx_test(@"fn main(){let ext_cx = 13; quote_item!(dontcare);}");
+    }
+    #[test] fn quote_pat_test() {
+        quote_ext_cx_test(@"fn main(){let ext_cx = 13; quote_pat!(dontcare);}");
+    }
+    #[test] fn quote_ty_test() {
+        quote_ext_cx_test(@"fn main(){let ext_cx = 13; quote_ty!(dontcare);}");
+    }
+    #[test] fn quote_tokens_test() {
+        quote_ext_cx_test(@"fn main(){let ext_cx = 13; quote_tokens!(dontcare);}");
+    }
+
+    fn quote_ext_cx_test(crate_str : @str) {
+        let crate = expand_crate_str(crate_str);
+        // find the ext_cx binding
+        let bindings = @mut ~[];
+        visit::walk_crate(&mut new_name_finder(bindings), crate, ());
+        let cxbinds : ~[&ast::Ident] = bindings.iter().filter(|b|{@"ext_cx" == (ident_to_str(*b))}).collect();
+        let cxbind = match cxbinds {
+            [b] => b,
+            _ => fail!("expected just one binding for ext_cx")
+        };
+        let resolved_binding = mtwt_resolve(*cxbind);
+        // find all the ext_cx varrefs:
+        let varrefs = @mut ~[];
+        visit::walk_crate(&mut new_path_finder(varrefs), crate, ());
+        // the ext_cx binding should bind all of the ext_cx varrefs:
+        for (idx,v) in varrefs.iter().filter(|p|{ p.segments.len() == 1
+                    && (@"ext_cx" == (ident_to_str(&p.segments[0].identifier)))
+            }).enumerate() {
+            if (mtwt_resolve(v.segments[0].identifier) != resolved_binding) {
+                std::io::println("uh oh, ext_cx binding didn't match ext_cx varref:");
+                std::io::println(fmt!("this is varref # %?",idx));
+                std::io::println(fmt!("binding: %?",cxbind));
+                std::io::println(fmt!("resolves to: %?",resolved_binding));
+                std::io::println(fmt!("varref: %?",v.segments[0]));
+                std::io::println(fmt!("resolves to: %?",mtwt_resolve(v.segments[0].identifier)));
+                let table = get_sctable();
+                std::io::println("SC table:");
+                for (idx,val) in table.table.iter().enumerate() {
+                    std::io::println(fmt!("%4u : %?",idx,val));
+                }
+            }
+            assert_eq!(mtwt_resolve(v.segments[0].identifier),resolved_binding);
+        };
+    }
+
     #[test]
     fn pat_idents(){
         let pat = string_to_pat(@"(a,Foo{x:c @ (b,9),y:Bar(4,d)})");
diff --git a/src/libsyntax/ext/quote.rs b/src/libsyntax/ext/quote.rs
index 4e8b7467c5c..d1299cf56cb 100644
--- a/src/libsyntax/ext/quote.rs
+++ b/src/libsyntax/ext/quote.rs
@@ -12,6 +12,7 @@ use ast;
 use codemap::{BytePos, Pos, Span};
 use ext::base::ExtCtxt;
 use ext::base;
+use ext::expand;
 use ext::build::AstBuilder;
 use parse::token::*;
 use parse::token;
@@ -291,47 +292,73 @@ pub mod rt {
 
 pub fn expand_quote_tokens(cx: @ExtCtxt,
                            sp: Span,
-                           tts: &[ast::token_tree]) -> base::MacResult {
+                           tts: &[ast::token_tree],
+                           ctxt: ast::SyntaxContext) -> base::MacResult {
     let (cx_expr, expr) = expand_tts(cx, sp, tts);
-    base::MRExpr(expand_wrapper(cx, sp, cx_expr, expr))
+    let expanded = expand_wrapper(cx, sp, cx_expr, expr);
+    // repaint the expanded code so it's as though it was the original text.
+    let repainted = expand::replace_ctxts(expanded,ctxt);
+    base::MRExpr(repainted)
 }
 
 pub fn expand_quote_expr(cx: @ExtCtxt,
                          sp: Span,
-                         tts: &[ast::token_tree]) -> base::MacResult {
-    base::MRExpr(expand_parse_call(cx, sp, "parse_expr", ~[], tts))
+                         tts: &[ast::token_tree],
+                         ctxt: ast::SyntaxContext) -> base::MacResult {
+    let expanded = expand_parse_call(cx, sp, "parse_expr", ~[], tts);
+    // repaint the expanded code so it's as though it was the original text.
+    let repainted = expand::replace_ctxts(expanded,ctxt);
+    base::MRExpr(repainted)
 }
 
+// these probably need to be capturing, too...
+
 pub fn expand_quote_item(cx: @ExtCtxt,
                          sp: Span,
-                         tts: &[ast::token_tree]) -> base::MacResult {
+                         tts: &[ast::token_tree],
+                         ctxt: ast::SyntaxContext) -> base::MacResult {
     let e_attrs = cx.expr_vec_uniq(sp, ~[]);
-    base::MRExpr(expand_parse_call(cx, sp, "parse_item",
-                                    ~[e_attrs], tts))
+    let expanded = expand_parse_call(cx, sp, "parse_item",
+                                    ~[e_attrs], tts);
+    // repaint the expanded code so it's as though it was the original text.
+    let repainted = expand::replace_ctxts(expanded,ctxt);
+    base::MRExpr(repainted)
 }
 
 pub fn expand_quote_pat(cx: @ExtCtxt,
                         sp: Span,
-                        tts: &[ast::token_tree]) -> base::MacResult {
+                        tts: &[ast::token_tree],
+                        ctxt: ast::SyntaxContext) -> base::MacResult {
     let e_refutable = cx.expr_lit(sp, ast::lit_bool(true));
-    base::MRExpr(expand_parse_call(cx, sp, "parse_pat",
-                                    ~[e_refutable], tts))
+    let expanded = expand_parse_call(cx, sp, "parse_pat",
+                                    ~[e_refutable], tts);
+    // repaint the expanded code so it's as though it was the original text.
+    let repainted = expand::replace_ctxts(expanded,ctxt);
+    base::MRExpr(repainted)
 }
 
 pub fn expand_quote_ty(cx: @ExtCtxt,
                        sp: Span,
-                       tts: &[ast::token_tree]) -> base::MacResult {
+                       tts: &[ast::token_tree],
+                       ctxt: ast::SyntaxContext) -> base::MacResult {
     let e_param_colons = cx.expr_lit(sp, ast::lit_bool(false));
-    base::MRExpr(expand_parse_call(cx, sp, "parse_ty",
-                                    ~[e_param_colons], tts))
+    let expanded = expand_parse_call(cx, sp, "parse_ty",
+                                     ~[e_param_colons], tts);
+    // repaint the expanded code so it's as though it was the original text.
+    let repainted = expand::replace_ctxts(expanded,ctxt);
+    base::MRExpr(repainted)
 }
 
 pub fn expand_quote_stmt(cx: @ExtCtxt,
                          sp: Span,
-                         tts: &[ast::token_tree]) -> base::MacResult {
+                         tts: &[ast::token_tree],
+                         ctxt: ast::SyntaxContext) -> base::MacResult {
     let e_attrs = cx.expr_vec_uniq(sp, ~[]);
-    base::MRExpr(expand_parse_call(cx, sp, "parse_stmt",
-                                    ~[e_attrs], tts))
+    let expanded = expand_parse_call(cx, sp, "parse_stmt",
+                                    ~[e_attrs], tts);
+    // repaint the expanded code so it's as though it was the original text.
+    let repainted = expand::replace_ctxts(expanded,ctxt);
+    base::MRExpr(repainted)
 }
 
 fn ids_ext(strs: ~[~str]) -> ~[ast::Ident] {
diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs
index 2145e4297e7..074ec3cd195 100644
--- a/src/libsyntax/ext/tt/macro_rules.rs
+++ b/src/libsyntax/ext/tt/macro_rules.rs
@@ -150,8 +150,8 @@ pub fn add_new_extension(cx: @ExtCtxt,
         cx.span_fatal(best_fail_spot, best_fail_msg);
     }
 
-    let exp: @fn(@ExtCtxt, Span, &[ast::token_tree]) -> MacResult =
-        |cx, sp, arg| generic_extension(cx, sp, name, arg, *lhses, *rhses);
+    let exp: @fn(@ExtCtxt, Span, &[ast::token_tree], ctxt: ast::SyntaxContext) -> MacResult =
+        |cx, sp, arg, _ctxt| generic_extension(cx, sp, name, arg, *lhses, *rhses);
 
     return MRDef(MacroDef{
         name: ident_to_str(&name),
diff --git a/src/libsyntax/fold.rs b/src/libsyntax/fold.rs
index 823e018a718..b805d55830f 100644
--- a/src/libsyntax/fold.rs
+++ b/src/libsyntax/fold.rs
@@ -209,6 +209,7 @@ pub fn noop_fold_crate(c: &Crate, fld: @ast_fold) -> Crate {
 }
 
 fn noop_fold_view_item(vi: &view_item_, _fld: @ast_fold) -> view_item_ {
+    // FIXME #7654: doesn't iterate over idents in a view_item_use
     return /* FIXME (#2543) */ (*vi).clone();
 }
 
-- 
cgit 1.4.1-3-g733a5


From e29d25338d7b0374f3dd5976b57463e6aafab0ac Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Tue, 9 Jul 2013 14:24:03 -0700
Subject: remove dead code

---
 src/libsyntax/ext/expand.rs | 15 ---------------
 1 file changed, 15 deletions(-)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index 91a56590435..dfeb938c709 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -1501,21 +1501,6 @@ pub fn new_rename_folder(from : ast::Ident, to : ast::Name) -> @AstFoldFns {
     fun_to_ctxt_folder(@Renamer{from:from,to:to})
 }
 
-/*
-// perform resolution (in the MTWT sense) on all of the
-// idents in the tree. This is the final step in expansion.
-// FIXME #6993: this function could go away, along with
-// the separate mtwt_resolution pass
-pub fn new_ident_resolver() ->
-    @fn(ast::Ident)->ast::Ident {
-    |id : ast::Ident|
-    ast::Ident {
-        name : mtwt_resolve(id),
-        ctxt : EMPTY_CTXT
-    }
-}
-*/
-
 // apply a given mark to the given token trees. Used prior to expansion of a macro.
 fn mark_tts(tts : &[token_tree], m : Mrk) -> ~[token_tree] {
     fold_tts(tts,new_mark_folder(m) as @ast_fold)
-- 
cgit 1.4.1-3-g733a5


From bc2a44daf1cf348da98de9553fccc23806e84f71 Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Wed, 10 Jul 2013 11:52:39 -0700
Subject: fix one remaining token comparison, refactor token comparison to
 avoid == check

---
 src/libsyntax/ext/tt/macro_parser.rs | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs
index f859a656d1f..ee7750bfd57 100644
--- a/src/libsyntax/ext/tt/macro_parser.rs
+++ b/src/libsyntax/ext/tt/macro_parser.rs
@@ -234,16 +234,12 @@ pub fn parse_or_else(
     }
 }
 
-// temporary for testing
+// perform a token equality check, ignoring syntax context (that is, an unhygienic comparison)
 pub fn token_name_eq(t1 : &Token, t2 : &Token) -> bool {
-    if (*t1 == *t2) {
-        true
-    } else {
-        match (t1,t2) {
-            (&token::IDENT(id1,_),&token::IDENT(id2,_)) =>
-            id1.name == id2.name,
-            _ => false
-        }
+    match (t1,t2) {
+        (&token::IDENT(id1,_),&token::IDENT(id2,_)) =>
+        id1.name == id2.name,
+        _ => *t1 == *t2
     }
 }
 
@@ -310,7 +306,10 @@ pub fn parse(
                     // the *_t vars are workarounds for the lack of unary move
                     match ei.sep {
                       Some(ref t) if idx == len => { // we need a separator
-                        if tok == (*t) { //pass the separator
+                        // i'm conflicted about whether this should be hygienic....
+                        // though in this case, if the separators are never legal
+                        // idents, it shouldn't matter.
+                        if token_name_eq(&tok, t) { //pass the separator
                             let mut ei_t = ei.clone();
                             ei_t.idx += 1;
                             next_eis.push(ei_t);
@@ -367,7 +366,7 @@ pub fn parse(
         }
 
         /* error messages here could be improved with links to orig. rules */
-        if tok == EOF {
+        if token_name_eq(&tok, &EOF) {
             if eof_eis.len() == 1u {
                 let mut v = ~[];
                 for dv in eof_eis[0u].matches.mut_iter() {
-- 
cgit 1.4.1-3-g733a5


From 9ab2cfdae610374760b86a8b6069baf365970f2f Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Wed, 10 Jul 2013 11:52:59 -0700
Subject: added utility function

---
 src/libsyntax/ast_util.rs | 37 +++++++++++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/ast_util.rs b/src/libsyntax/ast_util.rs
index 4c0ad816afb..dfac782929d 100644
--- a/src/libsyntax/ast_util.rs
+++ b/src/libsyntax/ast_util.rs
@@ -985,6 +985,32 @@ pub fn getLast(arr: &~[Mrk]) -> uint {
     *arr.last()
 }
 
+// are two paths equal when compared unhygienically?
+// since I'm using this to replace ==, it seems appropriate
+// to compare the span, global, etc. fields as well.
+pub fn path_name_eq(a : &ast::Path, b : &ast::Path) -> bool {
+    (a.span == b.span)
+    && (a.global == b.global)
+    // NOTE: ident->name in lifetimes!
+    && (a.rp == b.rp)
+    // NOTE: can a type contain an ident?
+    && (a.types == b.types)
+    && (idents_name_eq(a.idents, b.idents))
+}
+
+// are two arrays of idents equal when compared unhygienically?
+pub fn idents_name_eq(a : &[ast::ident], b : &[ast::ident]) -> bool {
+    if (a.len() != b.len()) {
+        false
+    } else {
+        for a.iter().enumerate().advance |(idx,id)|{
+            if (id.name != b[idx].name) {
+                return false;
+            }
+        }
+        true
+    }
+}
 
 #[cfg(test)]
 mod test {
@@ -992,6 +1018,17 @@ mod test {
     use super::*;
     use std::io;
 
+    #[test] fn idents_name_eq_test() {
+        assert!(idents_name_eq(~[ident{name:3,ctxt:4},
+                                 ident{name:78,ctxt:82}],
+                               ~[ident{name:3,ctxt:104},
+                                 ident{name:78,ctxt:182}]));
+        assert!(!idents_name_eq(~[ident{name:3,ctxt:4},
+                                  ident{name:78,ctxt:82}],
+                                ~[ident{name:3,ctxt:104},
+                                  ident{name:77,ctxt:182}]));
+    }
+
     #[test] fn xorpush_test () {
         let mut s = ~[];
         xorPush(&mut s,14);
-- 
cgit 1.4.1-3-g733a5


From 0954e66442a169be40f1e65de68a85d7e3dacf3a Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Thu, 5 Sep 2013 14:15:00 -0700
Subject: uncomment mtwt_resolve calls

---
 src/librustc/middle/resolve.rs       | 14 +++++++++----
 src/libsyntax/ast_util.rs            | 38 ++++++++++++++++++++----------------
 src/libsyntax/ext/tt/macro_parser.rs |  4 ++--
 src/libsyntax/parse/token.rs         | 20 +++++++++++++++++++
 4 files changed, 53 insertions(+), 23 deletions(-)

(limited to 'src/libsyntax')

diff --git a/src/librustc/middle/resolve.rs b/src/librustc/middle/resolve.rs
index 8c8dedeef32..292047d885d 100644
--- a/src/librustc/middle/resolve.rs
+++ b/src/librustc/middle/resolve.rs
@@ -23,7 +23,7 @@ use middle::pat_util::pat_bindings;
 
 use syntax::ast::*;
 use syntax::ast;
-use syntax::ast_util::{def_id_of_def, local_def}; // mtwt_resolve
+use syntax::ast_util::{def_id_of_def, local_def, mtwt_resolve};
 use syntax::ast_util::{path_to_ident, walk_pat, trait_method_to_ty_method};
 use syntax::ast_util::{Privacy, Public, Private};
 use syntax::ast_util::{variant_visibility_to_privacy, visibility_to_privacy};
@@ -4067,10 +4067,14 @@ impl Resolver {
                              None, visitor);
     }
 
+    // build a map from pattern identifiers to binding-info's.
+    // this is done hygienically. This could arise for a macro
+    // that expands into an or-pattern where one 'x' was from the
+    // user and one 'x' came from the macro.
     pub fn binding_mode_map(@mut self, pat: @Pat) -> BindingMap {
         let mut result = HashMap::new();
         do pat_bindings(self.def_map, pat) |binding_mode, _id, sp, path| {
-            let name = path_to_ident(path).name; // mtwt_resolve(path_to_ident(path));
+            let name = mtwt_resolve(path_to_ident(path));
             result.insert(name,
                           binding_info {span: sp,
                                         binding_mode: binding_mode});
@@ -4078,6 +4082,8 @@ impl Resolver {
         return result;
     }
 
+    // check that all of the arms in an or-pattern have exactly the
+    // same set of bindings, with the same binding modes for each.
     pub fn check_consistent_bindings(@mut self, arm: &Arm) {
         if arm.pats.len() == 0 { return; }
         let map_0 = self.binding_mode_map(arm.pats[0]);
@@ -4293,7 +4299,7 @@ impl Resolver {
                     // what you want).
 
                     let ident = path.segments[0].identifier;
-                    let renamed = ident.name; // mtwt_resolve(ident);
+                    let renamed = mtwt_resolve(ident);
 
                     match self.resolve_bare_identifier_pattern(ident) {
                         FoundStructOrEnumVariant(def)
@@ -4833,7 +4839,7 @@ impl Resolver {
         let search_result;
         match namespace {
             ValueNS => {
-                let renamed = ident.name; // mtwt_resolve(ident);
+                let renamed = mtwt_resolve(ident);
                 search_result = self.search_ribs(self.value_ribs, renamed,
                                                  span,
                                                  DontAllowCapturingSelf);
diff --git a/src/libsyntax/ast_util.rs b/src/libsyntax/ast_util.rs
index dfac782929d..321ac9428ea 100644
--- a/src/libsyntax/ast_util.rs
+++ b/src/libsyntax/ast_util.rs
@@ -991,20 +991,19 @@ pub fn getLast(arr: &~[Mrk]) -> uint {
 pub fn path_name_eq(a : &ast::Path, b : &ast::Path) -> bool {
     (a.span == b.span)
     && (a.global == b.global)
-    // NOTE: ident->name in lifetimes!
-    && (a.rp == b.rp)
-    // NOTE: can a type contain an ident?
-    && (a.types == b.types)
-    && (idents_name_eq(a.idents, b.idents))
+    && (segments_name_eq(a.segments, b.segments))
 }
 
-// are two arrays of idents equal when compared unhygienically?
-pub fn idents_name_eq(a : &[ast::ident], b : &[ast::ident]) -> bool {
+// are two arrays of segments equal when compared unhygienically?
+pub fn segments_name_eq(a : &[ast::PathSegment], b : &[ast::PathSegment]) -> bool {
     if (a.len() != b.len()) {
         false
     } else {
-        for a.iter().enumerate().advance |(idx,id)|{
-            if (id.name != b[idx].name) {
+        for (idx,seg) in a.iter().enumerate() {
+            if (seg.identifier.name != b[idx].identifier.name)
+                // ident -> name problems in lifetime comparison?
+                || (seg.lifetime != b[idx].lifetime)
+                || (seg.types != b[idx].types) {
                 return false;
             }
         }
@@ -1017,16 +1016,21 @@ mod test {
     use ast::*;
     use super::*;
     use std::io;
+    use opt_vec;
+
+    fn ident_to_segment(id : &ident) -> PathSegment {
+        PathSegment{identifier:id.clone(), lifetime: None, types: opt_vec::Empty}
+    }
 
     #[test] fn idents_name_eq_test() {
-        assert!(idents_name_eq(~[ident{name:3,ctxt:4},
-                                 ident{name:78,ctxt:82}],
-                               ~[ident{name:3,ctxt:104},
-                                 ident{name:78,ctxt:182}]));
-        assert!(!idents_name_eq(~[ident{name:3,ctxt:4},
-                                  ident{name:78,ctxt:82}],
-                                ~[ident{name:3,ctxt:104},
-                                  ident{name:77,ctxt:182}]));
+        assert!(segments_name_eq([ident{name:3,ctxt:4},
+                                   ident{name:78,ctxt:82}].map(ident_to_segment),
+                                 [ident{name:3,ctxt:104},
+                                   ident{name:78,ctxt:182}].map(ident_to_segment)));
+        assert!(!segments_name_eq([ident{name:3,ctxt:4},
+                                    ident{name:78,ctxt:82}].map(ident_to_segment),
+                                  [ident{name:3,ctxt:104},
+                                    ident{name:77,ctxt:182}].map(ident_to_segment)));
     }
 
     #[test] fn xorpush_test () {
diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs
index ee7750bfd57..aa4183837e3 100644
--- a/src/libsyntax/ext/tt/macro_parser.rs
+++ b/src/libsyntax/ext/tt/macro_parser.rs
@@ -355,8 +355,8 @@ pub fn parse(
                   match_nonterminal(_,_,_) => { bb_eis.push(ei) }
                   match_tok(ref t) => {
                     let mut ei_t = ei.clone();
-                    if (token_name_eq(t,&tok)) {
-                    //if (token::mtwt_token_eq(t,&tok)) {
+                    //if (token_name_eq(t,&tok)) {
+                    if (token::mtwt_token_eq(t,&tok)) {
                         ei_t.idx += 1;
                         next_eis.push(ei_t);
                     }
diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs
index 311d498eec2..39e0f85920c 100644
--- a/src/libsyntax/parse/token.rs
+++ b/src/libsyntax/parse/token.rs
@@ -720,6 +720,15 @@ pub fn is_reserved_keyword(tok: &Token) -> bool {
     }
 }
 
+pub fn mtwt_token_eq(t1 : &Token, t2 : &Token) -> bool {
+    match (t1,t2) {
+        (&IDENT(id1,_),&IDENT(id2,_)) =>
+        ast_util::mtwt_resolve(id1) == ast_util::mtwt_resolve(id2),
+        _ => *t1 == *t2
+    }
+}
+
+
 #[cfg(test)]
 mod test {
     use super::*;
@@ -728,6 +737,17 @@ mod test {
     use ast;
     use ast_util;
 
+    fn mark_ident(id : ast::ident, m : ast::Mrk) -> ast::ident {
+        ast::ident{name:id.name,ctxt:ast_util::new_mark(m,id.ctxt)}
+    }
+
+    #[test] fn mtwt_token_eq_test() {
+        assert!(mtwt_token_eq(&GT,&GT));
+        let a = str_to_ident("bac");
+        let a1 = mark_ident(a,92);
+        assert!(mtwt_token_eq(&IDENT(a,true),&IDENT(a1,false)));
+    }
+
 
     #[test] fn str_ptr_eq_tests(){
         let a = @"abc";
-- 
cgit 1.4.1-3-g733a5


From ec0a64def54e0b381ede187bfe199b7620b56c45 Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Wed, 10 Jul 2013 16:40:09 -0700
Subject: memoization for resolve

---
 src/libsyntax/ast_util.rs    | 129 ++++++++++++++++++++++++++++++-------------
 src/libsyntax/parse/token.rs |   4 +-
 2 files changed, 93 insertions(+), 40 deletions(-)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/ast_util.rs b/src/libsyntax/ast_util.rs
index 321ac9428ea..e0d58c14a9a 100644
--- a/src/libsyntax/ast_util.rs
+++ b/src/libsyntax/ast_util.rs
@@ -915,30 +915,63 @@ fn idx_push<T>(vec: &mut ~[T], val: T) -> uint {
 
 /// Resolve a syntax object to a name, per MTWT.
 pub fn mtwt_resolve(id : Ident) -> Name {
-    resolve_internal(id, get_sctable())
+    resolve_internal(id, get_sctable(), get_resolve_table())
+}
+
+// FIXME #4536: must be pub for testing
+pub type ResolveTable = HashMap<(Name,SyntaxContext),Name>;
+
+// okay, I admit, putting this in TLS is not so nice:
+// fetch the SCTable from TLS, create one if it doesn't yet exist.
+pub fn get_resolve_table() -> @mut ResolveTable {
+    static resolve_table_key: local_data::Key<@@mut ResolveTable> = &local_data::Key;
+    match local_data::get(resolve_table_key, |k| k.map(|&k| *k)) {
+        None => {
+            let new_table = @@mut HashMap::new();
+            local_data::set(resolve_table_key,new_table);
+            *new_table
+        },
+        Some(intr) => *intr
+    }
 }
 
 // Resolve a syntax object to a name, per MTWT.
+// adding memoization to possibly resolve 500+ seconds in resolve for librustc (!)
 // FIXME #4536 : currently pub to allow testing
-pub fn resolve_internal(id : Ident, table : &mut SCTable) -> Name {
-    match table.table[id.ctxt] {
-        EmptyCtxt => id.name,
-        // ignore marks here:
-        Mark(_,subctxt) => resolve_internal(Ident{name:id.name, ctxt: subctxt},table),
-        // do the rename if necessary:
-        Rename(Ident{name,ctxt},toname,subctxt) => {
-            // this could be cached or computed eagerly:
-            let resolvedfrom = resolve_internal(Ident{name:name,ctxt:ctxt},table);
-            let resolvedthis = resolve_internal(Ident{name:id.name,ctxt:subctxt},table);
-            if ((resolvedthis == resolvedfrom)
-                && (marksof(ctxt,resolvedthis,table)
-                    == marksof(subctxt,resolvedthis,table))) {
-                toname
-            } else {
-                resolvedthis
-            }
+pub fn resolve_internal(id : Ident,
+                        table : &mut SCTable,
+                        resolve_table : &mut ResolveTable) -> Name {
+    let key = (id.name,id.ctxt);
+    match resolve_table.contains_key(&key) {
+        false => {
+            let resolved = {
+                match table.table[id.ctxt] {
+                    EmptyCtxt => id.name,
+                    // ignore marks here:
+                    Mark(_,subctxt) => resolve_internal(Ident{name:id.name, ctxt: subctxt},table,resolve_table),
+                    // do the rename if necessary:
+                    Rename(Ident{name,ctxt},toname,subctxt) => {
+                        let resolvedfrom = resolve_internal(Ident{name:name,ctxt:ctxt},table,resolve_table);
+                        let resolvedthis = resolve_internal(Ident{name:id.name,ctxt:subctxt},table,resolve_table);
+                        if ((resolvedthis == resolvedfrom)
+                            && (marksof(ctxt,resolvedthis,table)
+                                == marksof(subctxt,resolvedthis,table))) {
+                            toname
+                        } else {
+                            resolvedthis
+                        }
+                    }
+                    IllegalCtxt() => fail!(~"expected resolvable context, got IllegalCtxt")
+                }
+            };
+            resolve_table.insert(key,resolved);
+            resolved
+        }
+        true => {
+            // it's guaranteed to be there, because we just checked that it was
+            // there and we never remove anything from the table:
+            *(resolve_table.find(&key).unwrap())
         }
-        IllegalCtxt() => fail!(~"expected resolvable context, got IllegalCtxt")
     }
 }
 
@@ -1017,20 +1050,21 @@ mod test {
     use super::*;
     use std::io;
     use opt_vec;
+    use std::hash::HashMap;
 
-    fn ident_to_segment(id : &ident) -> PathSegment {
+    fn ident_to_segment(id : &Ident) -> PathSegment {
         PathSegment{identifier:id.clone(), lifetime: None, types: opt_vec::Empty}
     }
 
     #[test] fn idents_name_eq_test() {
-        assert!(segments_name_eq([ident{name:3,ctxt:4},
-                                   ident{name:78,ctxt:82}].map(ident_to_segment),
-                                 [ident{name:3,ctxt:104},
-                                   ident{name:78,ctxt:182}].map(ident_to_segment)));
-        assert!(!segments_name_eq([ident{name:3,ctxt:4},
-                                    ident{name:78,ctxt:82}].map(ident_to_segment),
-                                  [ident{name:3,ctxt:104},
-                                    ident{name:77,ctxt:182}].map(ident_to_segment)));
+        assert!(segments_name_eq([Ident{name:3,ctxt:4},
+                                   Ident{name:78,ctxt:82}].map(ident_to_segment),
+                                 [Ident{name:3,ctxt:104},
+                                   Ident{name:78,ctxt:182}].map(ident_to_segment)));
+        assert!(!segments_name_eq([Ident{name:3,ctxt:4},
+                                    Ident{name:78,ctxt:82}].map(ident_to_segment),
+                                  [Ident{name:3,ctxt:104},
+                                    Ident{name:77,ctxt:182}].map(ident_to_segment)));
     }
 
     #[test] fn xorpush_test () {
@@ -1162,29 +1196,30 @@ mod test {
     #[test] fn resolve_tests () {
         let a = 40;
         let mut t = new_sctable_internal();
+        let mut rt = HashMap::new();
         // - ctxt is MT
-        assert_eq!(resolve_internal(id(a,EMPTY_CTXT),&mut t),a);
+        assert_eq!(resolve_internal(id(a,EMPTY_CTXT),&mut t, &mut rt),a);
         // - simple ignored marks
         { let sc = unfold_marks(~[1,2,3],EMPTY_CTXT,&mut t);
-         assert_eq!(resolve_internal(id(a,sc),&mut t),a);}
+         assert_eq!(resolve_internal(id(a,sc),&mut t, &mut rt),a);}
         // - orthogonal rename where names don't match
         { let sc = unfold_test_sc(~[R(id(50,EMPTY_CTXT),51),M(12)],EMPTY_CTXT,&mut t);
-         assert_eq!(resolve_internal(id(a,sc),&mut t),a);}
+         assert_eq!(resolve_internal(id(a,sc),&mut t, &mut rt),a);}
         // - rename where names do match, but marks don't
         { let sc1 = new_mark_internal(1,EMPTY_CTXT,&mut t);
          let sc = unfold_test_sc(~[R(id(a,sc1),50),
                                    M(1),
                                    M(2)],
                                  EMPTY_CTXT,&mut t);
-        assert_eq!(resolve_internal(id(a,sc),&mut t), a);}
+        assert_eq!(resolve_internal(id(a,sc),&mut t, &mut rt), a);}
         // - rename where names and marks match
         { let sc1 = unfold_test_sc(~[M(1),M(2)],EMPTY_CTXT,&mut t);
          let sc = unfold_test_sc(~[R(id(a,sc1),50),M(1),M(2)],EMPTY_CTXT,&mut t);
-         assert_eq!(resolve_internal(id(a,sc),&mut t), 50); }
+         assert_eq!(resolve_internal(id(a,sc),&mut t, &mut rt), 50); }
         // - rename where names and marks match by literal sharing
         { let sc1 = unfold_test_sc(~[M(1),M(2)],EMPTY_CTXT,&mut t);
          let sc = unfold_test_sc(~[R(id(a,sc1),50)],sc1,&mut t);
-         assert_eq!(resolve_internal(id(a,sc),&mut t), 50); }
+         assert_eq!(resolve_internal(id(a,sc),&mut t, &mut rt), 50); }
         // - two renames of the same var.. can only happen if you use
         // local-expand to prevent the inner binding from being renamed
         // during the rename-pass caused by the first:
@@ -1192,22 +1227,28 @@ mod test {
         { let sc = unfold_test_sc(~[R(id(a,EMPTY_CTXT),50),
                                     R(id(a,EMPTY_CTXT),51)],
                                   EMPTY_CTXT,&mut t);
-         assert_eq!(resolve_internal(id(a,sc),&mut t), 51); }
+         assert_eq!(resolve_internal(id(a,sc),&mut t, &mut rt), 51); }
         // the simplest double-rename:
         { let a_to_a50 = new_rename_internal(id(a,EMPTY_CTXT),50,EMPTY_CTXT,&mut t);
          let a50_to_a51 = new_rename_internal(id(a,a_to_a50),51,a_to_a50,&mut t);
-         assert_eq!(resolve_internal(id(a,a50_to_a51),&mut t),51);
+         assert_eq!(resolve_internal(id(a,a50_to_a51),&mut t, &mut rt),51);
          // mark on the outside doesn't stop rename:
          let sc = new_mark_internal(9,a50_to_a51,&mut t);
-         assert_eq!(resolve_internal(id(a,sc),&mut t),51);
+         assert_eq!(resolve_internal(id(a,sc),&mut t, &mut rt),51);
          // but mark on the inside does:
          let a50_to_a51_b = unfold_test_sc(~[R(id(a,a_to_a50),51),
                                               M(9)],
                                            a_to_a50,
                                            &mut t);
-         assert_eq!(resolve_internal(id(a,a50_to_a51_b),&mut t),50);}
+         assert_eq!(resolve_internal(id(a,a50_to_a51_b),&mut t, &mut rt),50);}
+    }
+
+    #[test] fn mtwt_resolve_test(){
+        let a = 40;
+        assert_eq!(mtwt_resolve(id(a,EMPTY_CTXT)),a);
     }
 
+
     #[test] fn hashing_tests () {
         let mut t = new_sctable_internal();
         assert_eq!(new_mark_internal(12,EMPTY_CTXT,&mut t),2);
@@ -1217,4 +1258,16 @@ mod test {
         // I'm assuming that the rename table will behave the same....
     }
 
+    #[test] fn resolve_table_hashing_tests() {
+        let mut t = new_sctable_internal();
+        let mut rt = HashMap::new();
+        assert_eq!(rt.len(),0);
+        resolve_internal(id(30,EMPTY_CTXT),&mut t, &mut rt);
+        assert_eq!(rt.len(),1);
+        resolve_internal(id(39,EMPTY_CTXT),&mut t, &mut rt);
+        assert_eq!(rt.len(),2);
+        resolve_internal(id(30,EMPTY_CTXT),&mut t, &mut rt);
+        assert_eq!(rt.len(),2);
+    }
+
 }
diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs
index 39e0f85920c..aa26feec28c 100644
--- a/src/libsyntax/parse/token.rs
+++ b/src/libsyntax/parse/token.rs
@@ -737,8 +737,8 @@ mod test {
     use ast;
     use ast_util;
 
-    fn mark_ident(id : ast::ident, m : ast::Mrk) -> ast::ident {
-        ast::ident{name:id.name,ctxt:ast_util::new_mark(m,id.ctxt)}
+    fn mark_ident(id : ast::Ident, m : ast::Mrk) -> ast::Ident {
+        ast::Ident{name:id.name,ctxt:ast_util::new_mark(m,id.ctxt)}
     }
 
     #[test] fn mtwt_token_eq_test() {
-- 
cgit 1.4.1-3-g733a5


From 60562ac9f8c68c1cf3b30efcd6d6954a9b82acbe Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Thu, 11 Jul 2013 22:58:14 -0700
Subject: whitespace, reindentation, and comments only

---
 src/librustc/middle/trans/base.rs   |  3 ++-
 src/librustc/middle/trans/expr.rs   |  5 +++--
 src/libsyntax/ast.rs                |  8 +++++++-
 src/libsyntax/ast_util.rs           | 12 +++++++----
 src/libsyntax/ext/expand.rs         | 40 ++++++++++++++++++++-----------------
 src/libsyntax/ext/tt/macro_rules.rs |  1 -
 6 files changed, 42 insertions(+), 27 deletions(-)

(limited to 'src/libsyntax')

diff --git a/src/librustc/middle/trans/base.rs b/src/librustc/middle/trans/base.rs
index 8d2b9d7a0ee..9e281172b26 100644
--- a/src/librustc/middle/trans/base.rs
+++ b/src/librustc/middle/trans/base.rs
@@ -2787,7 +2787,8 @@ pub fn declare_intrinsics(llmod: ModuleRef) -> HashMap<&'static str, ValueRef> {
 
 pub fn declare_dbg_intrinsics(llmod: ModuleRef, intrinsics: &mut HashMap<&'static str, ValueRef>) {
     ifn!(intrinsics, "llvm.dbg.declare", [Type::metadata(), Type::metadata()], Type::void());
-    ifn!(intrinsics, "llvm.dbg.value",   [Type::metadata(), Type::i64(), Type::metadata()], Type::void());
+    ifn!(intrinsics,
+         "llvm.dbg.value",   [Type::metadata(), Type::i64(), Type::metadata()], Type::void());
 }
 
 pub fn trap(bcx: @mut Block) {
diff --git a/src/librustc/middle/trans/expr.rs b/src/librustc/middle/trans/expr.rs
index 3ecb319e468..819d390ac28 100644
--- a/src/librustc/middle/trans/expr.rs
+++ b/src/librustc/middle/trans/expr.rs
@@ -1176,8 +1176,9 @@ fn trans_rec_or_struct(bcx: @mut Block,
         let mut need_base = vec::from_elem(field_tys.len(), true);
 
         let numbered_fields = do fields.map |field| {
-            let opt_pos = field_tys.iter().position(|field_ty|
-                                                    field_ty.ident.name == field.ident.name);
+            let opt_pos =
+                field_tys.iter().position(|field_ty|
+                                          field_ty.ident.name == field.ident.name);
             match opt_pos {
                 Some(i) => {
                     need_base[i] = false;
diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs
index 31729493815..6a4b2ade931 100644
--- a/src/libsyntax/ast.rs
+++ b/src/libsyntax/ast.rs
@@ -51,6 +51,12 @@ impl Ident {
         if (self.ctxt == other.ctxt) {
             self.name == other.name
         } else {
+            // IF YOU SEE ONE OF THESE FAILS: it means that you're comparing
+            // idents that have different contexts. You can't fix this without
+            // knowing whether the comparison should be hygienic or non-hygienic.
+            // if it should be non-hygienic (most things are), just compare the
+            // 'name' fields of the idents. Or, even better, replace the idents
+            // with Name's.
             fail!(fmt!("not allowed to compare these idents: %?, %?", self, other));
         }
     }
@@ -128,6 +134,7 @@ pub type FnIdent = Option<Ident>;
 pub struct Lifetime {
     id: NodeId,
     span: Span,
+    // FIXME #7743 : change this to Name!
     ident: Ident
 }
 
@@ -647,7 +654,6 @@ pub enum matcher_ {
     // lo, hi position-in-match-array used:
     match_seq(~[matcher], Option<::parse::token::Token>, bool, uint, uint),
     // parse a Rust NT: name to bind, name of NT, position in match array:
-    // NOTE: 'name of NT' shouldnt really be represented as an ident, should it?
     match_nonterminal(Ident, Ident, uint)
 }
 
diff --git a/src/libsyntax/ast_util.rs b/src/libsyntax/ast_util.rs
index e0d58c14a9a..57a0e12ec2f 100644
--- a/src/libsyntax/ast_util.rs
+++ b/src/libsyntax/ast_util.rs
@@ -948,11 +948,14 @@ pub fn resolve_internal(id : Ident,
                 match table.table[id.ctxt] {
                     EmptyCtxt => id.name,
                     // ignore marks here:
-                    Mark(_,subctxt) => resolve_internal(Ident{name:id.name, ctxt: subctxt},table,resolve_table),
+                    Mark(_,subctxt) =>
+                        resolve_internal(Ident{name:id.name, ctxt: subctxt},table,resolve_table),
                     // do the rename if necessary:
                     Rename(Ident{name,ctxt},toname,subctxt) => {
-                        let resolvedfrom = resolve_internal(Ident{name:name,ctxt:ctxt},table,resolve_table);
-                        let resolvedthis = resolve_internal(Ident{name:id.name,ctxt:subctxt},table,resolve_table);
+                        let resolvedfrom =
+                            resolve_internal(Ident{name:name,ctxt:ctxt},table,resolve_table);
+                        let resolvedthis =
+                            resolve_internal(Ident{name:id.name,ctxt:subctxt},table,resolve_table);
                         if ((resolvedthis == resolvedfrom)
                             && (marksof(ctxt,resolvedthis,table)
                                 == marksof(subctxt,resolvedthis,table))) {
@@ -1034,8 +1037,9 @@ pub fn segments_name_eq(a : &[ast::PathSegment], b : &[ast::PathSegment]) -> boo
     } else {
         for (idx,seg) in a.iter().enumerate() {
             if (seg.identifier.name != b[idx].identifier.name)
-                // ident -> name problems in lifetime comparison?
+                // FIXME #7743: ident -> name problems in lifetime comparison?
                 || (seg.lifetime != b[idx].lifetime)
+                // can types contain idents?
                 || (seg.types != b[idx].types) {
                 return false;
             }
diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index dfeb938c709..8ee045ba8c1 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -76,19 +76,20 @@ pub fn expand_expr(extsbox: @mut SyntaxEnv,
                             // mark before:
                             let marked_before = mark_tts(*tts,fm);
                             let marked_ctxt = new_mark(fm, ctxt);
-                            let expanded = match expandfun(cx, mac.span, marked_before, marked_ctxt) {
-                                MRExpr(e) => e,
-                                MRAny(expr_maker,_,_) => expr_maker(),
-                                _ => {
-                                    cx.span_fatal(
-                                        pth.span,
-                                        fmt!(
-                                            "non-expr macro in expr pos: %s",
-                                            extnamestr
+                            let expanded =
+                                match expandfun(cx, mac.span, marked_before, marked_ctxt) {
+                                    MRExpr(e) => e,
+                                    MRAny(expr_maker,_,_) => expr_maker(),
+                                    _ => {
+                                        cx.span_fatal(
+                                            pth.span,
+                                            fmt!(
+                                                "non-expr macro in expr pos: %s",
+                                                extnamestr
+                                            )
                                         )
-                                    )
-                                }
-                            };
+                                    }
+                                };
                             // mark after:
                             let marked_after = mark_expr(expanded,fm);
 
@@ -1735,12 +1736,14 @@ mod test {
                  ~[~[0]])
                 // FIXME #6994: the next string exposes the bug referred to in issue 6994, so I'm
                 // commenting it out.
-                // the z flows into and out of two macros (g & f) along one path, and one (just g) along the
-                // other, so the result of the whole thing should be "let z_123 = 3; z_123"
-                //"macro_rules! g (($x:ident) => ({macro_rules! f(($y:ident)=>({let $y=3;$x}));f!($x)}))
+                // the z flows into and out of two macros (g & f) along one path, and one
+                // (just g) along the other, so the result of the whole thing should
+                // be "let z_123 = 3; z_123"
+                //"macro_rules! g (($x:ident) =>
+                //   ({macro_rules! f(($y:ident)=>({let $y=3;$x}));f!($x)}))
                 //   fn a(){g!(z)}"
-                // create a really evil test case where a $x appears inside a binding of $x but *shouldnt*
-                // bind because it was inserted by a different macro....
+                // create a really evil test case where a $x appears inside a binding of $x
+                // but *shouldnt* bind because it was inserted by a different macro....
             ];
         for s in tests.iter() {
             run_renaming_test(s);
@@ -1820,7 +1823,8 @@ mod test {
         // find the ext_cx binding
         let bindings = @mut ~[];
         visit::walk_crate(&mut new_name_finder(bindings), crate, ());
-        let cxbinds : ~[&ast::Ident] = bindings.iter().filter(|b|{@"ext_cx" == (ident_to_str(*b))}).collect();
+        let cxbinds : ~[&ast::Ident] =
+            bindings.iter().filter(|b|{@"ext_cx" == (ident_to_str(*b))}).collect();
         let cxbind = match cxbinds {
             [b] => b,
             _ => fail!("expected just one binding for ext_cx")
diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs
index 074ec3cd195..4b0974b70bf 100644
--- a/src/libsyntax/ext/tt/macro_rules.rs
+++ b/src/libsyntax/ext/tt/macro_rules.rs
@@ -49,7 +49,6 @@ pub fn add_new_extension(cx: @ExtCtxt,
     // ...quasiquoting this would be nice.
     let argument_gram = ~[
         ms(match_seq(~[
-            // NOTE : probably just use an enum for the NT_name ?
             ms(match_nonterminal(lhs_nm, special_idents::matchers, 0u)),
             ms(match_tok(FAT_ARROW)),
             ms(match_nonterminal(rhs_nm, special_idents::tt, 1u)),
-- 
cgit 1.4.1-3-g733a5


From dbf4e19ea5946723acc1495ac82ed73f784821dd Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Thu, 11 Jul 2013 23:07:34 -0700
Subject: remove unneeded imports, clean up unused var warnings

---
 src/libsyntax/ext/base.rs     |  4 ++--
 src/libsyntax/ext/expand.rs   | 14 +++++++-------
 src/libsyntax/parse/token.rs  |  7 ++-----
 src/libsyntax/print/pprust.rs |  6 ++++--
 4 files changed, 15 insertions(+), 16 deletions(-)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/ext/base.rs b/src/libsyntax/ext/base.rs
index 24f70bfeaa8..afb8802968c 100644
--- a/src/libsyntax/ext/base.rs
+++ b/src/libsyntax/ext/base.rs
@@ -135,13 +135,13 @@ pub fn syntax_expander_table() -> SyntaxEnv {
     // utility function to simplify creating NormalTT syntax extensions
     // that ignore their contexts
     fn builtin_normal_tt_no_ctxt(f: SyntaxExpanderTTFunNoCtxt) -> @Transformer {
-        let wrapped_expander : SyntaxExpanderTTFun = |a,b,c,d|{f(a,b,c)};
+        let wrapped_expander : SyntaxExpanderTTFun = |a,b,c,_d|{f(a,b,c)};
         @SE(NormalTT(wrapped_expander, None))
     }
     // utility function to simplify creating IdentTT syntax extensions
     // that ignore their contexts
     fn builtin_item_tt_no_ctxt(f: SyntaxExpanderTTItemFunNoCtxt) -> @Transformer {
-        let wrapped_expander : SyntaxExpanderTTItemFun = |a,b,c,d,e|{f(a,b,c,d)};
+        let wrapped_expander : SyntaxExpanderTTItemFun = |a,b,c,d,_e|{f(a,b,c,d)};
         @SE(IdentTT(wrapped_expander, None))
     }
     let mut syntax_expanders = HashMap::new();
diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index 8ee045ba8c1..120f75406c7 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -8,12 +8,12 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use ast::{Block, Crate, NodeId, DeclLocal, EMPTY_CTXT, Expr_, ExprMac, SyntaxContext};
+use ast::{Block, Crate, NodeId, DeclLocal, Expr_, ExprMac, SyntaxContext};
 use ast::{Local, Ident, mac_invoc_tt};
 use ast::{item_mac, Mrk, Stmt_, StmtDecl, StmtMac, StmtExpr, StmtSemi};
-use ast::{ILLEGAL_CTXT, SCTable, token_tree};
+use ast::{token_tree};
 use ast;
-use ast_util::{new_rename, new_mark, mtwt_resolve};
+use ast_util::{new_rename, new_mark};
 use attr;
 use attr::AttrMetaMethods;
 use codemap;
@@ -585,7 +585,7 @@ fn expand_non_macro_stmt (exts: SyntaxEnv,
                 let new_name = fresh_name(ident);
                 new_pending_renames.push((*ident,new_name));
             }
-            let mut rename_fld = renames_to_fold(new_pending_renames);
+            let rename_fld = renames_to_fold(new_pending_renames);
             // rewrite the pattern using the new names (the old ones
             // have already been applied):
             let rewritten_pat = rename_fld.fold_pat(expanded_pat);
@@ -906,7 +906,7 @@ pub fn expand_block(extsbox: @mut SyntaxEnv,
                     _cx: @ExtCtxt,
                     blk: &Block,
                     fld: @ast_fold,
-                    orig: @fn(&Block, @ast_fold) -> Block)
+                    _orig: @fn(&Block, @ast_fold) -> Block)
                  -> Block {
     // see note below about treatment of exts table
     with_exts_frame!(extsbox,false,
@@ -917,7 +917,7 @@ pub fn expand_block(extsbox: @mut SyntaxEnv,
 pub fn expand_block_elts(exts: SyntaxEnv, b: &Block, fld: @ast_fold) -> Block {
     let block_info = get_block_info(exts);
     let pending_renames = block_info.pending_renames;
-    let mut rename_fld = renames_to_fold(pending_renames);
+    let rename_fld = renames_to_fold(pending_renames);
     let new_view_items = b.view_items.map(|x| fld.fold_view_item(x));
     let mut new_stmts = ~[];
     for x in b.stmts.iter() {
@@ -1456,7 +1456,7 @@ impl CtxtFn for Marker {
 pub struct Repainter { ctxt : SyntaxContext }
 
 impl CtxtFn for Repainter {
-    fn f(&self, ctxt : ast::SyntaxContext) -> ast::SyntaxContext {
+    fn f(&self, _ctxt : ast::SyntaxContext) -> ast::SyntaxContext {
         self.ctxt
     }
 }
diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs
index aa26feec28c..d0372191084 100644
--- a/src/libsyntax/parse/token.rs
+++ b/src/libsyntax/parse/token.rs
@@ -17,10 +17,7 @@ use util::interner;
 
 use std::cast;
 use std::char;
-use std::cmp::Equiv;
 use std::local_data;
-use std::rand;
-use std::rand::RngUtil;
 
 #[deriving(Clone, Encodable, Decodable, Eq, IterBytes)]
 pub enum binop {
@@ -565,8 +562,8 @@ pub fn str_ptr_eq(a : @str, b : @str) -> bool {
         let q : uint = cast::transmute(b);
         let result = p == q;
         // got to transmute them back, to make sure the ref count is correct:
-        let junk1 : @str = cast::transmute(p);
-        let junk2 : @str = cast::transmute(q);
+        let _junk1 : @str = cast::transmute(p);
+        let _junk2 : @str = cast::transmute(q);
         result
     }
 }
diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs
index 4d464706d6f..8871413a46d 100644
--- a/src/libsyntax/print/pprust.rs
+++ b/src/libsyntax/print/pprust.rs
@@ -619,7 +619,8 @@ pub fn print_item(s: @ps, item: &ast::item) {
         }
         bclose(s, item.span);
       }
-      ast::item_mac(codemap::Spanned { node: ast::mac_invoc_tt(ref pth, ref tts, ctxt),
+      // I think it's reasonable to hide the context here:
+      ast::item_mac(codemap::Spanned { node: ast::mac_invoc_tt(ref pth, ref tts, _),
                                    _}) => {
         print_visibility(s, item.vis);
         print_path(s, pth, false);
@@ -1021,7 +1022,8 @@ pub fn print_if(s: @ps, test: &ast::Expr, blk: &ast::Block,
 
 pub fn print_mac(s: @ps, m: &ast::mac) {
     match m.node {
-      ast::mac_invoc_tt(ref pth, ref tts, ctxt) => {
+      // I think it's reasonable to hide the ctxt here:
+      ast::mac_invoc_tt(ref pth, ref tts, _) => {
         print_path(s, pth, false);
         word(s.s, "!");
         popen(s);
-- 
cgit 1.4.1-3-g733a5


From 7dca301cf4ed4d5199d060352f6bcec9a3be3829 Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Fri, 12 Jul 2013 08:55:12 -0700
Subject: ident->name

---
 src/libsyntax/print/pprust.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs
index 8871413a46d..f440e0a1771 100644
--- a/src/libsyntax/print/pprust.rs
+++ b/src/libsyntax/print/pprust.rs
@@ -1926,8 +1926,8 @@ pub fn print_arg(s: @ps, input: &ast::arg) {
         match input.pat.node {
             ast::PatIdent(_, ref path, _) if
                 path.segments.len() == 1 &&
-                path.segments[0].identifier ==
-                    parse::token::special_idents::invalid => {
+                path.segments[0].identifier.name ==
+                    parse::token::special_idents::invalid.name => {
                 // Do nothing.
             }
             _ => {
-- 
cgit 1.4.1-3-g733a5


From 7b548e71806b735a215f7dba26a20771c4f924bd Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Fri, 12 Jul 2013 18:35:05 -0700
Subject: expose mtwt_marksof for testing

---
 src/libsyntax/ast_util.rs | 5 +++++
 1 file changed, 5 insertions(+)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/ast_util.rs b/src/libsyntax/ast_util.rs
index 57a0e12ec2f..3043d087a15 100644
--- a/src/libsyntax/ast_util.rs
+++ b/src/libsyntax/ast_util.rs
@@ -979,6 +979,11 @@ pub fn resolve_internal(id : Ident,
 }
 
 /// Compute the marks associated with a syntax context.
+pub fn mtwt_marksof(ctxt: SyntaxContext, stopname: Name) -> ~[Mrk] {
+    marksof(ctxt, stopname, get_sctable())
+}
+
+// the internal function for computing marks
 // it's not clear to me whether it's better to use a [] mutable
 // vector or a cons-list for this.
 pub fn marksof(ctxt: SyntaxContext, stopname: Name, table: &SCTable) -> ~[Mrk] {
-- 
cgit 1.4.1-3-g733a5


From dc7f3df27fd17b41b3448f461138947db4c9bab2 Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Fri, 12 Jul 2013 18:35:47 -0700
Subject: awesome new bug! added test case

---
 src/libsyntax/ext/expand.rs | 81 +++++++++++++++++++++++++++++++++++++--------
 1 file changed, 68 insertions(+), 13 deletions(-)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index 120f75406c7..3a5b9f2f7c6 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -420,7 +420,6 @@ pub fn expand_item_mac(extsbox: @mut SyntaxEnv,
                     span: span
                 }
             });
-            let fm = fresh_mark();
             // mark before expansion:
             let marked_tts = mark_tts(tts,fm);
             let marked_ctxt = new_mark(fm,ctxt);
@@ -1533,7 +1532,7 @@ mod test {
     use super::*;
     use ast;
     use ast::{Attribute_, AttrOuter, MetaWord, EMPTY_CTXT};
-    use ast_util::{get_sctable, mtwt_resolve, new_rename};
+    use ast_util::{get_sctable, mtwt_marksof, mtwt_resolve, new_rename};
     use codemap;
     use codemap::Spanned;
     use parse;
@@ -1709,31 +1708,40 @@ mod test {
     //
     // The comparisons are done post-mtwt-resolve, so we're comparing renamed
     // names; differences in marks don't matter any more.
-    type renaming_test = (&'static str, ~[~[uint]]);
+    //
+    // oog... I also want tests that check "binding-identifier-=?". That is,
+    // not just "do these have the same name", but "do they have the same
+    // name *and* the same marks"? Understanding this is really pretty painful.
+    // in principle, you might want to control this boolean on a per-varref basis,
+    // but that would make things even harder to understand, and might not be
+    // necessary for thorough testing.
+    type renaming_test = (&'static str, ~[~[uint]], bool);
 
     #[test]
     fn automatic_renaming () {
-        // need some other way to test these...
         let tests : ~[renaming_test] =
             ~[// b & c should get new names throughout, in the expr too:
                 ("fn a() -> int { let b = 13; let c = b; b+c }",
-                 ~[~[0,1],~[2]]),
+                 ~[~[0,1],~[2]], false),
                 // both x's should be renamed (how is this causing a bug?)
                 ("fn main () {let x : int = 13;x;}",
-                 ~[~[0]]),
+                 ~[~[0]], false),
                 // the use of b after the + should be renamed, the other one not:
                 ("macro_rules! f (($x:ident) => (b + $x)) fn a() -> int { let b = 13; f!(b)}",
-                 ~[~[1]]),
+                 ~[~[1]], false),
                 // the b before the plus should not be renamed (requires marks)
                 ("macro_rules! f (($x:ident) => ({let b=9; ($x + b)})) fn a() -> int { f!(b)}",
-                 ~[~[1]]),
+                 ~[~[1]], false),
                 // the marks going in and out of letty should cancel, allowing that $x to
                 // capture the one following the semicolon.
                 // this was an awesome test case, and caught a *lot* of bugs.
                 ("macro_rules! letty(($x:ident) => (let $x = 15;))
                   macro_rules! user(($x:ident) => ({letty!($x); $x}))
                   fn main() -> int {user!(z)}",
-                 ~[~[0]])
+                 ~[~[0]], false),
+                // can't believe I missed this one : a macro def that refers to a local var:
+                ("fn main() {let x = 19; macro_rules! getx(()=>(x)); getx!();}",
+                ~[~[0]], true)
                 // FIXME #6994: the next string exposes the bug referred to in issue 6994, so I'm
                 // commenting it out.
                 // the z flows into and out of two macros (g & f) along one path, and one
@@ -1750,10 +1758,10 @@ mod test {
         }
     }
 
-
+    // run one of the renaming tests
     fn run_renaming_test(t : &renaming_test) {
-        let (teststr, bound_connections) = match *t {
-            (ref str,ref conns) => (str.to_managed(), conns.clone())
+        let (teststr, bound_connections, bound_ident_check) = match *t {
+            (ref str,ref conns, bic) => (str.to_managed(), conns.clone(), bic)
         };
         let cr = expand_crate_str(teststr.to_managed());
         // find the bindings:
@@ -1766,15 +1774,18 @@ mod test {
         assert_eq!(bindings.len(),bound_connections.len());
         for (binding_idx,shouldmatch) in bound_connections.iter().enumerate() {
             let binding_name = mtwt_resolve(bindings[binding_idx]);
+            let binding_marks = mtwt_marksof(bindings[binding_idx].ctxt,binding_name);
             // shouldmatch can't name varrefs that don't exist:
             assert!((shouldmatch.len() == 0) ||
                     (varrefs.len() > *shouldmatch.iter().max().unwrap()));
             for (idx,varref) in varrefs.iter().enumerate() {
                 if shouldmatch.contains(&idx) {
                     // it should be a path of length 1, and it should
-                    // be free-identifier=? to the given binding
+                    // be free-identifier=? or bound-identifier=? to the given binding
                     assert_eq!(varref.segments.len(),1);
                     let varref_name = mtwt_resolve(varref.segments[0].identifier);
+                    let varref_marks = mtwt_marksof(varref.segments[0].identifier.ctxt,
+                                                    binding_name);
                     if (!(varref_name==binding_name)){
                         std::io::println("uh oh, should match but doesn't:");
                         std::io::println(fmt!("varref: %?",varref));
@@ -1786,6 +1797,10 @@ mod test {
                         }
                     }
                     assert_eq!(varref_name,binding_name);
+                    if (bound_ident_check) {
+                        // we need to check the marks, too:
+                        assert_eq!(varref_marks,binding_marks.clone());
+                    }
                 } else {
                     let fail = (varref.segments.len() == 1)
                         && (mtwt_resolve(varref.segments[0].identifier) == binding_name);
@@ -1854,6 +1869,46 @@ mod test {
         };
     }
 
+    #[test] fn fmt_in_macro_used_inside_module_macro() {
+        let crate_str = @"macro_rules! fmt_wrap(($b:expr)=>(fmt!(\"left: %?\", $b)))
+macro_rules! foo_module (() => (mod generated { fn a() { let xx = 147; fmt_wrap!(xx);}}))
+foo_module!()
+";
+        let cr = expand_crate_str(crate_str);
+        // find the xx binding
+        let bindings = @mut ~[];
+        visit::walk_crate(&mut new_name_finder(bindings), cr, ());
+        let cxbinds : ~[&ast::Ident] =
+            bindings.iter().filter(|b|{@"xx" == (ident_to_str(*b))}).collect();
+        let cxbind = match cxbinds {
+            [b] => b,
+            _ => fail!("expected just one binding for ext_cx")
+        };
+        let resolved_binding = mtwt_resolve(*cxbind);
+        // find all the xx varrefs:
+        let varrefs = @mut ~[];
+        visit::walk_crate(&mut new_path_finder(varrefs), cr, ());
+        // the xx binding should bind all of the xx varrefs:
+        for (idx,v) in varrefs.iter().filter(|p|{ p.segments.len() == 1
+                                          && (@"xx" == (ident_to_str(&p.segments[0].identifier)))
+                                     }).enumerate() {
+            if (mtwt_resolve(v.segments[0].identifier) != resolved_binding) {
+                std::io::println("uh oh, xx binding didn't match xx varref:");
+                std::io::println(fmt!("this is xx varref # %?",idx));
+                std::io::println(fmt!("binding: %?",cxbind));
+                std::io::println(fmt!("resolves to: %?",resolved_binding));
+                std::io::println(fmt!("varref: %?",v.segments[0].identifier));
+                std::io::println(fmt!("resolves to: %?",mtwt_resolve(v.segments[0].identifier)));
+                let table = get_sctable();
+                std::io::println("SC table:");
+                for (idx,val) in table.table.iter().enumerate() {
+                    std::io::println(fmt!("%4u : %?",idx,val));
+                }
+            }
+            assert_eq!(mtwt_resolve(v.segments[0].identifier),resolved_binding);
+        };
+    }
+
     #[test]
     fn pat_idents(){
         let pat = string_to_pat(@"(a,Foo{x:c @ (b,9),y:Bar(4,d)})");
-- 
cgit 1.4.1-3-g733a5


From 1a065841172a0824046b1f041bdb1134ba820db8 Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Sat, 13 Jul 2013 10:18:37 -0700
Subject: comment on hygienic context extension train fns

---
 src/libsyntax/ext/expand.rs | 11 +++++++++++
 1 file changed, 11 insertions(+)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index 3a5b9f2f7c6..0e2fc4b59b5 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -1410,6 +1410,17 @@ pub fn expand_crate(parse_sess: @mut parse::ParseSess,
     return ret;
 }
 
+// HYGIENIC CONTEXT EXTENSION:
+// all of these functions are for walking over
+// ASTs and making some change to the context of every
+// element that has one. a CtxtFn is a trait-ified
+// version of a closure in (SyntaxContext -> SyntaxContext).
+// the ones defined here include:
+// Renamer - add a rename to a context
+// MultiRenamer - add a set of renames to a context
+// Marker - add a mark to a context
+// Repainter - replace a context (maybe Replacer would be a better name?)
+
 // a function in SyntaxContext -> SyntaxContext
 pub trait CtxtFn{
     fn f(&self, ast::SyntaxContext) -> ast::SyntaxContext;
-- 
cgit 1.4.1-3-g733a5


From 985a92ec123ff05a361bde6af4602282e3851d77 Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Sat, 13 Jul 2013 19:12:25 -0700
Subject: comments

---
 src/libsyntax/ast.rs | 9 +++++++++
 1 file changed, 9 insertions(+)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs
index 6a4b2ade931..b993f98ec82 100644
--- a/src/libsyntax/ast.rs
+++ b/src/libsyntax/ast.rs
@@ -81,6 +81,15 @@ impl Ident {
 // storage.
 pub type SyntaxContext = uint;
 
+// the SCTable contains a table of SyntaxContext_'s. It
+// represents a flattened tree structure, to avoid having
+// managed pointers everywhere (that caused an ICE).
+// the mark_memo and rename_memo fields are side-tables
+// that ensure that adding the same mark to the same context
+// gives you back the same context as before. This shouldn't
+// change the semantics--everything here is immutable--but
+// it should cut down on memory use *a lot*; applying a mark
+// to a tree containing 50 identifiers would otherwise generate
 pub struct SCTable {
     table : ~[SyntaxContext_],
     mark_memo : HashMap<(SyntaxContext,Mrk),SyntaxContext>,
-- 
cgit 1.4.1-3-g733a5


From e681e7843e6da4348648dcbdf11a633828c1b1a9 Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Sun, 14 Jul 2013 15:23:56 -0400
Subject: added string_to_tts

---
 src/libsyntax/util/parser_testing.rs | 6 ++++++
 1 file changed, 6 insertions(+)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/util/parser_testing.rs b/src/libsyntax/util/parser_testing.rs
index 51fd5be71ab..23396c06a9a 100644
--- a/src/libsyntax/util/parser_testing.rs
+++ b/src/libsyntax/util/parser_testing.rs
@@ -22,6 +22,12 @@ pub fn string_to_tts_and_sess (source_str : @str) -> (~[ast::token_tree],@mut Pa
     (filemap_to_tts(ps,string_to_filemap(ps,source_str,@"bogofile")),ps)
 }
 
+// map a string to tts, using a made-up filename:
+pub fn string_to_tts(source_str : @str) -> ~[ast::token_tree] {
+    let (tts,_) = string_to_tts_and_sess(source_str);
+    tts
+}
+
 pub fn string_to_parser_and_sess(source_str: @str) -> (Parser,@mut ParseSess) {
     let ps = new_parse_sess(None);
     (new_parser_from_source_str(ps,~[],@"bogofile",source_str),ps)
-- 
cgit 1.4.1-3-g733a5


From fddc815adaabb87c23e74c2bce41cddb5d433326 Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Sun, 14 Jul 2013 15:25:04 -0400
Subject: WIP: adding mark-cancelling for macro_rules

---
 src/libsyntax/ast_util.rs           | 10 ++++++++
 src/libsyntax/ext/base.rs           |  3 +--
 src/libsyntax/ext/expand.rs         | 46 +++++++++++++++++++++++++++++++++----
 src/libsyntax/ext/tt/macro_rules.rs |  5 +++-
 4 files changed, 57 insertions(+), 7 deletions(-)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/ast_util.rs b/src/libsyntax/ast_util.rs
index 3043d087a15..253bed4e3b8 100644
--- a/src/libsyntax/ast_util.rs
+++ b/src/libsyntax/ast_util.rs
@@ -1010,6 +1010,16 @@ pub fn marksof(ctxt: SyntaxContext, stopname: Name, table: &SCTable) -> ~[Mrk] {
     }
 }
 
+/// Return the outer mark for a context with a mark at the outside.
+/// FAILS when outside is not a mark.
+pub fn mtwt_outer_mark(ctxt: SyntaxContext) -> Mrk {
+    let sctable = get_sctable();
+    match sctable.table[ctxt] {
+        ast::Mark(mrk,_) => mrk,
+        _ => fail!("can't retrieve outer mark when outside is not a mark")
+    }
+}
+
 /// Push a name... unless it matches the one on top, in which
 /// case pop and discard (so two of the same marks cancel)
 pub fn xorPush(marks: &mut ~[uint], mark: uint) {
diff --git a/src/libsyntax/ext/base.rs b/src/libsyntax/ext/base.rs
index afb8802968c..7590125a189 100644
--- a/src/libsyntax/ext/base.rs
+++ b/src/libsyntax/ext/base.rs
@@ -152,8 +152,7 @@ pub fn syntax_expander_table() -> SyntaxEnv {
                                 pending_renames : @mut ~[]
                             }));
     syntax_expanders.insert(intern(&"macro_rules"),
-                            builtin_item_tt_no_ctxt(
-                                ext::tt::macro_rules::add_new_extension));
+                            @SE(IdentTT(ext::tt::macro_rules::add_new_extension, None)));
     syntax_expanders.insert(intern(&"fmt"),
                             builtin_normal_tt_no_ctxt(ext::fmt::expand_syntax_ext));
     syntax_expanders.insert(intern(&"format"),
diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index 0e2fc4b59b5..ad07d166f45 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -13,7 +13,8 @@ use ast::{Local, Ident, mac_invoc_tt};
 use ast::{item_mac, Mrk, Stmt_, StmtDecl, StmtMac, StmtExpr, StmtSemi};
 use ast::{token_tree};
 use ast;
-use ast_util::{new_rename, new_mark};
+use ast_util::{mtwt_outer_mark, new_rename, new_mark};
+use ast_util;
 use attr;
 use attr::AttrMetaMethods;
 use codemap;
@@ -1507,7 +1508,10 @@ pub fn renames_to_fold(renames : @mut ~[(ast::Ident,ast::Name)]) -> @AstFoldFns
 }
 
 // just a convenience:
-pub fn new_mark_folder(m : Mrk) -> @AstFoldFns { fun_to_ctxt_folder(@Marker{mark:m}) }
+pub fn new_mark_folder(m : Mrk) -> @AstFoldFns {
+    fun_to_ctxt_folder(@Marker{mark:m})
+}
+
 pub fn new_rename_folder(from : ast::Ident, to : ast::Name) -> @AstFoldFns {
     fun_to_ctxt_folder(@Renamer{from:from,to:to})
 }
@@ -1538,6 +1542,16 @@ pub fn replace_ctxts(expr : @ast::Expr, ctxt : SyntaxContext) -> @ast::Expr {
     fun_to_ctxt_folder(@Repainter{ctxt:ctxt}).fold_expr(expr)
 }
 
+// take the mark from the given ctxt (that has a mark at the outside),
+// and apply it to everything in the token trees, thereby cancelling
+// that mark.
+pub fn mtwt_cancel_outer_mark(tts: &[ast::token_tree], ctxt: ast::SyntaxContext)
+    -> ~[ast::token_tree] {
+    let outer_mark = mtwt_outer_mark(ctxt);
+    mark_tts(tts,outer_mark)
+}
+
+
 #[cfg(test)]
 mod test {
     use super::*;
@@ -1546,13 +1560,15 @@ mod test {
     use ast_util::{get_sctable, mtwt_marksof, mtwt_resolve, new_rename};
     use codemap;
     use codemap::Spanned;
+    use fold;
     use parse;
-    use parse::token::{gensym, intern, get_ident_interner, ident_to_str};
+    use parse::token::{fresh_mark, gensym, intern, get_ident_interner, ident_to_str};
+    use parse::token;
     use print::pprust;
     use std;
     use std::vec;
     use util::parser_testing::{string_to_crate, string_to_crate_and_sess, string_to_item};
-    use util::parser_testing::{string_to_pat, strs_to_idents};
+    use util::parser_testing::{string_to_pat, string_to_tts, strs_to_idents};
     use visit;
 
     // make sure that fail! is present
@@ -1651,6 +1667,28 @@ mod test {
         }
     }
 
+    #[test] fn cancel_outer_mark_test(){
+        let invalid_name = token::special_idents::invalid.name;
+        let ident_str = @"x";
+        let tts = string_to_tts(ident_str);
+        let fm = fresh_mark();
+        let marked_once = fold::fold_tts(tts,new_mark_folder(fm) as @fold::ast_fold);
+        assert_eq!(marked_once.len(),1);
+        let marked_once_ctxt =
+            match marked_once[0] {
+                ast::tt_tok(_,token::IDENT(id,_)) => id.ctxt,
+                _ => fail!(fmt!("unexpected shape for marked tts: %?",marked_once[0]))
+            };
+        assert_eq!(mtwt_marksof(marked_once_ctxt,invalid_name),~[fm]);
+        let remarked = mtwt_cancel_outer_mark(marked_once,marked_once_ctxt);
+        assert_eq!(remarked.len(),1);
+        match remarked[0] {
+            ast::tt_tok(_,token::IDENT(id,_)) =>
+            assert_eq!(mtwt_marksof(id.ctxt,invalid_name),~[]),
+            _ => fail!(fmt!("unexpected shape for marked tts: %?",remarked[0]))
+        }
+    }
+
     #[test]
     fn renaming () {
         let item_ast = string_to_crate(@"fn f() -> int { a }");
diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs
index 4b0974b70bf..74de8eaa09e 100644
--- a/src/libsyntax/ext/tt/macro_rules.rs
+++ b/src/libsyntax/ext/tt/macro_rules.rs
@@ -14,6 +14,7 @@ use ast;
 use codemap::{Span, Spanned, dummy_sp};
 use ext::base::{ExtCtxt, MacResult, MRAny, MRDef, MacroDef, NormalTT};
 use ext::base;
+use ext::expand;
 use ext::tt::macro_parser::{error};
 use ext::tt::macro_parser::{named_match, matched_seq, matched_nonterminal};
 use ext::tt::macro_parser::{parse, parse_or_else, success, failure};
@@ -29,8 +30,10 @@ use print;
 pub fn add_new_extension(cx: @ExtCtxt,
                          sp: Span,
                          name: Ident,
-                         arg: ~[ast::token_tree])
+                         arg: ~[ast::token_tree],
+                         stx_ctxt: ast::SyntaxContext)
                       -> base::MacResult {
+    let arg = expand::mtwt_cancel_outer_mark(arg,stx_ctxt);
     // Wrap a matcher_ in a spanned to produce a matcher.
     // these spans won't matter, anyways
     fn ms(m: matcher_) -> matcher {
-- 
cgit 1.4.1-3-g733a5


From 8330411688b038b95695f9823d3937ad2ebf6152 Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Fri, 26 Jul 2013 12:57:30 -0400
Subject: fixed a bug that caused double-expand-traversal of macros that expand
 into modules.

---
 src/libsyntax/ext/expand.rs | 29 ++++++++++-------------------
 1 file changed, 10 insertions(+), 19 deletions(-)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index ad07d166f45..ad1420f35cd 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -342,25 +342,16 @@ pub fn expand_item(extsbox: @mut SyntaxEnv,
                    fld: @ast_fold,
                    orig: @fn(@ast::item, @ast_fold) -> Option<@ast::item>)
                 -> Option<@ast::item> {
-    // need to do expansion first... it might turn out to be a module.
-    let maybe_it = match it.node {
-      ast::item_mac(*) => expand_item_mac(extsbox, cx, it, fld),
-      _ => Some(it)
-    };
-    match maybe_it {
-      Some(it) => {
-          match it.node {
-              ast::item_mod(_) | ast::item_foreign_mod(_) => {
-                  cx.mod_push(it.ident);
-                  let macro_escape = contains_macro_escape(it.attrs);
-                  let result = with_exts_frame!(extsbox,macro_escape,orig(it,fld));
-                  cx.mod_pop();
-                  result
-              }
-              _ => orig(it,fld)
-          }
-      }
-      None => None
+    match it.node {
+        ast::item_mac(*) => expand_item_mac(extsbox, cx, it, fld),
+        ast::item_mod(_) | ast::item_foreign_mod(_) => {
+            cx.mod_push(it.ident);
+            let macro_escape = contains_macro_escape(it.attrs);
+            let result = with_exts_frame!(extsbox,macro_escape,orig(it,fld));
+            cx.mod_pop();
+            result
+        },
+        _ => orig(it,fld)
     }
 }
 
-- 
cgit 1.4.1-3-g733a5


From fe1d21caec3a7d11f847a07769062804442338de Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Fri, 26 Jul 2013 13:27:38 -0400
Subject: add display_sctable fn to ast_util

---
 src/libsyntax/ast_util.rs | 9 +++++++++
 1 file changed, 9 insertions(+)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/ast_util.rs b/src/libsyntax/ast_util.rs
index 253bed4e3b8..ee898c182e0 100644
--- a/src/libsyntax/ast_util.rs
+++ b/src/libsyntax/ast_util.rs
@@ -907,6 +907,15 @@ pub fn get_sctable() -> @mut SCTable {
     }
 }
 
+/// print out an SCTable for debugging
+pub fn display_sctable(table : &SCTable) {
+    error!("SC table:");
+    for (idx,val) in table.table.iter().enumerate() {
+        error!("%4u : %?",idx,val);
+    }
+}
+
+
 /// Add a value to the end of a vec, return its index
 fn idx_push<T>(vec: &mut ~[T], val: T) -> uint {
     vec.push(val);
-- 
cgit 1.4.1-3-g733a5


From 6c294ba538b987d15fba10f7237ca52654fde64c Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Thu, 5 Sep 2013 09:29:31 -0700
Subject: add test case, cleanup

---
 src/libsyntax/ext/expand.rs  | 29 ++++++++++++++---------------
 src/libsyntax/parse/token.rs |  2 --
 2 files changed, 14 insertions(+), 17 deletions(-)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index ad1420f35cd..9c491c7dfce 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -14,7 +14,6 @@ use ast::{item_mac, Mrk, Stmt_, StmtDecl, StmtMac, StmtExpr, StmtSemi};
 use ast::{token_tree};
 use ast;
 use ast_util::{mtwt_outer_mark, new_rename, new_mark};
-use ast_util;
 use attr;
 use attr::AttrMetaMethods;
 use codemap;
@@ -1549,6 +1548,7 @@ mod test {
     use ast;
     use ast::{Attribute_, AttrOuter, MetaWord, EMPTY_CTXT};
     use ast_util::{get_sctable, mtwt_marksof, mtwt_resolve, new_rename};
+    use ast_util;
     use codemap;
     use codemap::Spanned;
     use fold;
@@ -1557,8 +1557,7 @@ mod test {
     use parse::token;
     use print::pprust;
     use std;
-    use std::vec;
-    use util::parser_testing::{string_to_crate, string_to_crate_and_sess, string_to_item};
+    use util::parser_testing::{string_to_crate, string_to_crate_and_sess};
     use util::parser_testing::{string_to_pat, string_to_tts, strs_to_idents};
     use visit;
 
@@ -1779,9 +1778,11 @@ mod test {
                   macro_rules! user(($x:ident) => ({letty!($x); $x}))
                   fn main() -> int {user!(z)}",
                  ~[~[0]], false),
-                // can't believe I missed this one : a macro def that refers to a local var:
-                ("fn main() {let x = 19; macro_rules! getx(()=>(x)); getx!();}",
-                ~[~[0]], true)
+                // FIXME #8062: this test exposes a *potential* bug; our system does
+                // not behave exactly like MTWT, but I haven't thought of a way that
+                // this could cause a bug in Rust, yet.
+                // ("fn main() {let hrcoo = 19; macro_rules! getx(()=>(hrcoo)); getx!();}",
+                // ~[~[0]], true)
                 // FIXME #6994: the next string exposes the bug referred to in issue 6994, so I'm
                 // commenting it out.
                 // the z flows into and out of two macros (g & f) along one path, and one
@@ -1800,6 +1801,7 @@ mod test {
 
     // run one of the renaming tests
     fn run_renaming_test(t : &renaming_test) {
+        let invalid_name = token::special_idents::invalid.name;
         let (teststr, bound_connections, bound_ident_check) = match *t {
             (ref str,ref conns, bic) => (str.to_managed(), conns.clone(), bic)
         };
@@ -1814,7 +1816,7 @@ mod test {
         assert_eq!(bindings.len(),bound_connections.len());
         for (binding_idx,shouldmatch) in bound_connections.iter().enumerate() {
             let binding_name = mtwt_resolve(bindings[binding_idx]);
-            let binding_marks = mtwt_marksof(bindings[binding_idx].ctxt,binding_name);
+            let binding_marks = mtwt_marksof(bindings[binding_idx].ctxt,invalid_name);
             // shouldmatch can't name varrefs that don't exist:
             assert!((shouldmatch.len() == 0) ||
                     (varrefs.len() > *shouldmatch.iter().max().unwrap()));
@@ -1825,20 +1827,17 @@ mod test {
                     assert_eq!(varref.segments.len(),1);
                     let varref_name = mtwt_resolve(varref.segments[0].identifier);
                     let varref_marks = mtwt_marksof(varref.segments[0].identifier.ctxt,
-                                                    binding_name);
+                                                    invalid_name);
                     if (!(varref_name==binding_name)){
                         std::io::println("uh oh, should match but doesn't:");
                         std::io::println(fmt!("varref: %?",varref));
                         std::io::println(fmt!("binding: %?", bindings[binding_idx]));
-                        let table = get_sctable();
-                        std::io::println("SC table:");
-                        for (idx,val) in table.table.iter().enumerate() {
-                            std::io::println(fmt!("%4u : %?",idx,val));
-                        }
+                        ast_util::display_sctable(get_sctable());
                     }
                     assert_eq!(varref_name,binding_name);
                     if (bound_ident_check) {
-                        // we need to check the marks, too:
+                        // we're checking bound-identifier=?, and the marks
+                        // should be the same, too:
                         assert_eq!(varref_marks,binding_marks.clone());
                     }
                 } else {
@@ -1849,7 +1848,7 @@ mod test {
                         std::io::println("uh oh, matches but shouldn't:");
                         std::io::println(fmt!("varref: %?",varref));
                         std::io::println(fmt!("binding: %?", bindings[binding_idx]));
-                        std::io::println(fmt!("sc_table: %?",get_sctable()));
+                        ast_util::display_sctable(get_sctable());
                     }
                     assert!(!fail);
                 }
diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs
index d0372191084..6b3a95a14f8 100644
--- a/src/libsyntax/parse/token.rs
+++ b/src/libsyntax/parse/token.rs
@@ -729,8 +729,6 @@ pub fn mtwt_token_eq(t1 : &Token, t2 : &Token) -> bool {
 #[cfg(test)]
 mod test {
     use super::*;
-    use std::io;
-    use std::managed;
     use ast;
     use ast_util;
 
-- 
cgit 1.4.1-3-g733a5


From 4664d3320dacb1cbf42081369d3fc4f61f2af1a3 Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Sun, 28 Jul 2013 16:34:19 -0400
Subject: comment fix

---
 src/libsyntax/fold.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/fold.rs b/src/libsyntax/fold.rs
index b805d55830f..aad992706fd 100644
--- a/src/libsyntax/fold.rs
+++ b/src/libsyntax/fold.rs
@@ -475,7 +475,7 @@ fn noop_fold_decl(d: &Decl_, fld: @ast_fold) -> Option<Decl_> {
 }
 
 // lift a function in ast-thingy X fold -> ast-thingy to a function
-// in (ast-thingy X span X fold) -> (ast-thingy X fold). Basically,
+// in (ast-thingy X span X fold) -> (ast-thingy X span). Basically,
 // carries the span around.
 // It seems strange to me that the call to new_fold doesn't happen
 // here but instead in the impl down below.... probably just an
-- 
cgit 1.4.1-3-g733a5


From 1ecc1e51c08f38b6ee42dae9a0f8bd6dc1f1b86f Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Thu, 5 Sep 2013 10:30:23 -0700
Subject: quote_* macros no longer need to be capturing

This is actually almost a problem, because those were my poster-child
macros for "here's how to implement a capturing macro." Following this
change, there will be no macros that use capturing; this will probably
make life unpleasant for the first person that wants to implement a
capturing macro. I should probably create a dummy_capturing macro,
just to show how it works.
---
 src/libsyntax/ext/base.rs   | 13 ++++++------
 src/libsyntax/ext/expand.rs | 52 ---------------------------------------------
 src/libsyntax/ext/quote.rs  | 45 +++++++++++----------------------------
 3 files changed, 19 insertions(+), 91 deletions(-)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/ext/base.rs b/src/libsyntax/ext/base.rs
index 7590125a189..3d5d62aeadf 100644
--- a/src/libsyntax/ext/base.rs
+++ b/src/libsyntax/ext/base.rs
@@ -185,17 +185,18 @@ pub fn syntax_expander_table() -> SyntaxEnv {
 
     // Quasi-quoting expanders
     syntax_expanders.insert(intern(&"quote_tokens"),
-        @SE(NormalTT(ext::quote::expand_quote_tokens, None)));
+                            builtin_normal_tt_no_ctxt(
+                                ext::quote::expand_quote_tokens));
     syntax_expanders.insert(intern(&"quote_expr"),
-        @SE(NormalTT(ext::quote::expand_quote_expr, None)));
+                            builtin_normal_tt_no_ctxt(ext::quote::expand_quote_expr));
     syntax_expanders.insert(intern(&"quote_ty"),
-        @SE(NormalTT(ext::quote::expand_quote_ty, None)));
+                            builtin_normal_tt_no_ctxt(ext::quote::expand_quote_ty));
     syntax_expanders.insert(intern(&"quote_item"),
-        @SE(NormalTT(ext::quote::expand_quote_item, None)));
+                            builtin_normal_tt_no_ctxt(ext::quote::expand_quote_item));
     syntax_expanders.insert(intern(&"quote_pat"),
-        @SE(NormalTT(ext::quote::expand_quote_pat, None)));
+                            builtin_normal_tt_no_ctxt(ext::quote::expand_quote_pat));
     syntax_expanders.insert(intern(&"quote_stmt"),
-        @SE(NormalTT(ext::quote::expand_quote_stmt, None)));
+                            builtin_normal_tt_no_ctxt(ext::quote::expand_quote_stmt));
 
     syntax_expanders.insert(intern(&"line"),
                             builtin_normal_tt_no_ctxt(
diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index 9c491c7dfce..5f57eba5b4c 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -1856,58 +1856,6 @@ mod test {
         }
     }
 
-    #[test] fn quote_expr_test() {
-        quote_ext_cx_test(@"fn main(){let ext_cx = 13; quote_expr!(dontcare);}");
-    }
-    #[test] fn quote_item_test() {
-        quote_ext_cx_test(@"fn main(){let ext_cx = 13; quote_item!(dontcare);}");
-    }
-    #[test] fn quote_pat_test() {
-        quote_ext_cx_test(@"fn main(){let ext_cx = 13; quote_pat!(dontcare);}");
-    }
-    #[test] fn quote_ty_test() {
-        quote_ext_cx_test(@"fn main(){let ext_cx = 13; quote_ty!(dontcare);}");
-    }
-    #[test] fn quote_tokens_test() {
-        quote_ext_cx_test(@"fn main(){let ext_cx = 13; quote_tokens!(dontcare);}");
-    }
-
-    fn quote_ext_cx_test(crate_str : @str) {
-        let crate = expand_crate_str(crate_str);
-        // find the ext_cx binding
-        let bindings = @mut ~[];
-        visit::walk_crate(&mut new_name_finder(bindings), crate, ());
-        let cxbinds : ~[&ast::Ident] =
-            bindings.iter().filter(|b|{@"ext_cx" == (ident_to_str(*b))}).collect();
-        let cxbind = match cxbinds {
-            [b] => b,
-            _ => fail!("expected just one binding for ext_cx")
-        };
-        let resolved_binding = mtwt_resolve(*cxbind);
-        // find all the ext_cx varrefs:
-        let varrefs = @mut ~[];
-        visit::walk_crate(&mut new_path_finder(varrefs), crate, ());
-        // the ext_cx binding should bind all of the ext_cx varrefs:
-        for (idx,v) in varrefs.iter().filter(|p|{ p.segments.len() == 1
-                    && (@"ext_cx" == (ident_to_str(&p.segments[0].identifier)))
-            }).enumerate() {
-            if (mtwt_resolve(v.segments[0].identifier) != resolved_binding) {
-                std::io::println("uh oh, ext_cx binding didn't match ext_cx varref:");
-                std::io::println(fmt!("this is varref # %?",idx));
-                std::io::println(fmt!("binding: %?",cxbind));
-                std::io::println(fmt!("resolves to: %?",resolved_binding));
-                std::io::println(fmt!("varref: %?",v.segments[0]));
-                std::io::println(fmt!("resolves to: %?",mtwt_resolve(v.segments[0].identifier)));
-                let table = get_sctable();
-                std::io::println("SC table:");
-                for (idx,val) in table.table.iter().enumerate() {
-                    std::io::println(fmt!("%4u : %?",idx,val));
-                }
-            }
-            assert_eq!(mtwt_resolve(v.segments[0].identifier),resolved_binding);
-        };
-    }
-
     #[test] fn fmt_in_macro_used_inside_module_macro() {
         let crate_str = @"macro_rules! fmt_wrap(($b:expr)=>(fmt!(\"left: %?\", $b)))
 macro_rules! foo_module (() => (mod generated { fn a() { let xx = 147; fmt_wrap!(xx);}}))
diff --git a/src/libsyntax/ext/quote.rs b/src/libsyntax/ext/quote.rs
index d1299cf56cb..6527b083cc1 100644
--- a/src/libsyntax/ext/quote.rs
+++ b/src/libsyntax/ext/quote.rs
@@ -12,7 +12,6 @@ use ast;
 use codemap::{BytePos, Pos, Span};
 use ext::base::ExtCtxt;
 use ext::base;
-use ext::expand;
 use ext::build::AstBuilder;
 use parse::token::*;
 use parse::token;
@@ -292,73 +291,53 @@ pub mod rt {
 
 pub fn expand_quote_tokens(cx: @ExtCtxt,
                            sp: Span,
-                           tts: &[ast::token_tree],
-                           ctxt: ast::SyntaxContext) -> base::MacResult {
+                           tts: &[ast::token_tree]) -> base::MacResult {
     let (cx_expr, expr) = expand_tts(cx, sp, tts);
     let expanded = expand_wrapper(cx, sp, cx_expr, expr);
-    // repaint the expanded code so it's as though it was the original text.
-    let repainted = expand::replace_ctxts(expanded,ctxt);
-    base::MRExpr(repainted)
+    base::MRExpr(expanded)
 }
 
 pub fn expand_quote_expr(cx: @ExtCtxt,
                          sp: Span,
-                         tts: &[ast::token_tree],
-                         ctxt: ast::SyntaxContext) -> base::MacResult {
+                         tts: &[ast::token_tree]) -> base::MacResult {
     let expanded = expand_parse_call(cx, sp, "parse_expr", ~[], tts);
-    // repaint the expanded code so it's as though it was the original text.
-    let repainted = expand::replace_ctxts(expanded,ctxt);
-    base::MRExpr(repainted)
+    base::MRExpr(expanded)
 }
 
-// these probably need to be capturing, too...
-
 pub fn expand_quote_item(cx: @ExtCtxt,
                          sp: Span,
-                         tts: &[ast::token_tree],
-                         ctxt: ast::SyntaxContext) -> base::MacResult {
+                         tts: &[ast::token_tree]) -> base::MacResult {
     let e_attrs = cx.expr_vec_uniq(sp, ~[]);
     let expanded = expand_parse_call(cx, sp, "parse_item",
                                     ~[e_attrs], tts);
-    // repaint the expanded code so it's as though it was the original text.
-    let repainted = expand::replace_ctxts(expanded,ctxt);
-    base::MRExpr(repainted)
+    base::MRExpr(expanded)
 }
 
 pub fn expand_quote_pat(cx: @ExtCtxt,
                         sp: Span,
-                        tts: &[ast::token_tree],
-                        ctxt: ast::SyntaxContext) -> base::MacResult {
+                        tts: &[ast::token_tree]) -> base::MacResult {
     let e_refutable = cx.expr_lit(sp, ast::lit_bool(true));
     let expanded = expand_parse_call(cx, sp, "parse_pat",
                                     ~[e_refutable], tts);
-    // repaint the expanded code so it's as though it was the original text.
-    let repainted = expand::replace_ctxts(expanded,ctxt);
-    base::MRExpr(repainted)
+    base::MRExpr(expanded)
 }
 
 pub fn expand_quote_ty(cx: @ExtCtxt,
                        sp: Span,
-                       tts: &[ast::token_tree],
-                       ctxt: ast::SyntaxContext) -> base::MacResult {
+                       tts: &[ast::token_tree]) -> base::MacResult {
     let e_param_colons = cx.expr_lit(sp, ast::lit_bool(false));
     let expanded = expand_parse_call(cx, sp, "parse_ty",
                                      ~[e_param_colons], tts);
-    // repaint the expanded code so it's as though it was the original text.
-    let repainted = expand::replace_ctxts(expanded,ctxt);
-    base::MRExpr(repainted)
+    base::MRExpr(expanded)
 }
 
 pub fn expand_quote_stmt(cx: @ExtCtxt,
                          sp: Span,
-                         tts: &[ast::token_tree],
-                         ctxt: ast::SyntaxContext) -> base::MacResult {
+                         tts: &[ast::token_tree]) -> base::MacResult {
     let e_attrs = cx.expr_vec_uniq(sp, ~[]);
     let expanded = expand_parse_call(cx, sp, "parse_stmt",
                                     ~[e_attrs], tts);
-    // repaint the expanded code so it's as though it was the original text.
-    let repainted = expand::replace_ctxts(expanded,ctxt);
-    base::MRExpr(repainted)
+    base::MRExpr(expanded)
 }
 
 fn ids_ext(strs: ~[~str]) -> ~[ast::Ident] {
-- 
cgit 1.4.1-3-g733a5


From eabeba3ef309b09a1acd8adfbca13203a0766d75 Mon Sep 17 00:00:00 2001
From: John Clements <clements@racket-lang.org>
Date: Thu, 5 Sep 2013 13:28:57 -0700
Subject: added index to test cases, more debugging output

---
 src/libsyntax/ext/expand.rs | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

(limited to 'src/libsyntax')

diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index 5f57eba5b4c..cde418ca991 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -1794,13 +1794,13 @@ mod test {
                 // create a really evil test case where a $x appears inside a binding of $x
                 // but *shouldnt* bind because it was inserted by a different macro....
             ];
-        for s in tests.iter() {
-            run_renaming_test(s);
+        for (idx,s) in tests.iter().enumerate() {
+            run_renaming_test(s,idx);
         }
     }
 
     // run one of the renaming tests
-    fn run_renaming_test(t : &renaming_test) {
+    fn run_renaming_test(t : &renaming_test, test_idx: uint) {
         let invalid_name = token::special_idents::invalid.name;
         let (teststr, bound_connections, bound_ident_check) = match *t {
             (ref str,ref conns, bic) => (str.to_managed(), conns.clone(), bic)
@@ -1845,8 +1845,15 @@ mod test {
                         && (mtwt_resolve(varref.segments[0].identifier) == binding_name);
                     // temp debugging:
                     if (fail) {
-                        std::io::println("uh oh, matches but shouldn't:");
+                        println!("failure on test {}",test_idx);
+                        println!("text of test case: \"{}\"", teststr);
+                        println!("");
+                        println!("uh oh, matches but shouldn't:");
                         std::io::println(fmt!("varref: %?",varref));
+                        // good lord, you can't make a path with 0 segments, can you?
+                        println!("varref's first segment's uint: {}, and string: \"{}\"",
+                                 varref.segments[0].identifier.name,
+                                 ident_to_str(&varref.segments[0].identifier));
                         std::io::println(fmt!("binding: %?", bindings[binding_idx]));
                         ast_util::display_sctable(get_sctable());
                     }
-- 
cgit 1.4.1-3-g733a5