auto merge of #5908 : jbclements/rust/add-hygiene-machinery, r=graydon

This pull request changes the representation of identifiers by adding an integer to the side of each one.  This integer will eventually be a reference to a side-table of syntax contexts, presumably stored in TLS. This pull request also adds a bunch of utility functions required for hygiene, and associated tests, but doesn't actually deploy those functions.

Finally, it also has a number of small cleanup items.

8 files changed, 541 insertions, 114 deletions

diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs
index 9c20012d42e..6f4693adb9f 100644
--- a/src/libsyntax/ast.rs
+++ b/src/libsyntax/ast.rs
@@ -22,45 +22,53 @@ use core::to_str::ToStr;
 use std::serialize::{Encodable, Decodable, Encoder, Decoder};
 
 
-/* can't import macros yet, so this is copied from token.rs. See its comment
- * there. */
-macro_rules! interner_key (
-    () => (cast::transmute::<(uint, uint),
-            &fn(+v: @@::parse::token::ident_interner)>(
-        (-3 as uint, 0u)))
-)
-
 // an identifier contains an index into the interner
 // table and a SyntaxContext to track renaming and
 // macro expansion per Flatt et al., "Macros
 // That Work Together"
 #[deriving(Eq)]
-pub struct ident { repr: Name }
+pub struct ident { repr: Name, ctxt: SyntaxContext }
 
 // a SyntaxContext represents a chain of macro-expandings
 // and renamings. Each macro expansion corresponds to
 // a fresh uint
+
+// I'm representing this syntax context as an index into
+// a table, in order to work around a compiler bug
+// that's causing unreleased memory to cause core dumps
+// and also perhaps to save some work in destructor checks.
+// the special uint '0' will be used to indicate an empty
+// syntax context
+
+// this uint is a reference to a table stored in thread-local
+// storage.
+pub type SyntaxContext = uint;
+
+pub type SCTable = ~[SyntaxContext_];
+pub static empty_ctxt : uint = 0;
+
 #[deriving(Eq)]
-pub enum SyntaxContext {
-    MT,
-    Mark (Mrk,~SyntaxContext),
-    Rename (~ident,Name,~SyntaxContext)
+#[auto_encode]
+#[auto_decode]
+pub enum SyntaxContext_ {
+    EmptyCtxt,
+    Mark (Mrk,SyntaxContext),
+    // flattening the name and syntaxcontext into the rename...
+    // HIDDEN INVARIANTS:
+    // 1) the first name in a Rename node
+    // can only be a programmer-supplied name.
+    // 2) Every Rename node with a given Name in the
+    // "to" slot must have the same name and context
+    // in the "from" slot. In essence, they're all
+    // pointers to a single "rename" event node.
+    Rename (ident,Name,SyntaxContext)
 }
 
-/*
-// ** this is going to have to apply to paths, not to idents.
-// Returns true if these two identifiers access the same
-// local binding or top-level binding... that's what it
-// should do. For now, it just compares the names.
-pub fn free_ident_eq (a : ident, b: ident) -> bool{
-    a.repr == b.repr
-}
-*/
-// a name represents a string, interned
-type Name = uint;
+// a name represents an identifier
+pub type Name = uint;
 // a mark represents a unique id associated
 // with a macro expansion
-type Mrk = uint;
+pub type Mrk = uint;
 
 impl<S:Encoder> Encodable<S> for ident {
     fn encode(&self, s: &S) {
@@ -1310,22 +1318,77 @@ pub enum inlined_item {
     ii_dtor(struct_dtor, ident, Generics, def_id /* parent id */)
 }
 
+/* hold off on tests ... they appear in a later merge.
 #[cfg(test)]
 mod test {
-    //are asts encodable?
-
-    // it looks like this *will* be a compiler bug, after
-    // I get deriving_eq for crates into incoming :)
-    /*
+    use core::option::{None, Option, Some};
+    use core::uint;
     use std;
     use codemap::*;
     use super::*;
 
+
+    #[test] fn xorpush_test () {
+        let mut s = ~[];
+        xorPush(&mut s,14);
+        assert_eq!(s,~[14]);
+        xorPush(&mut s,14);
+        assert_eq!(s,~[]);
+        xorPush(&mut s,14);
+        assert_eq!(s,~[14]);
+        xorPush(&mut s,15);
+        assert_eq!(s,~[14,15]);
+        xorPush (&mut s,16);
+        assert_eq! (s,~[14,15,16]);
+        xorPush (&mut s,16);
+        assert_eq! (s,~[14,15]);
+        xorPush (&mut s,15);
+        assert_eq! (s,~[14]);
+    }
+
+    #[test] fn test_marksof () {
+        let stopname = uints_to_name(&~[12,14,78]);
+        let name1 = uints_to_name(&~[4,9,7]);
+        assert_eq!(marksof (MT,stopname),~[]);
+        assert_eq! (marksof (Mark (4,@Mark(98,@MT)),stopname),~[4,98]);
+        // does xoring work?
+        assert_eq! (marksof (Mark (5, @Mark (5, @Mark (16,@MT))),stopname),
+                     ~[16]);
+        // does nested xoring work?
+        assert_eq! (marksof (Mark (5,
+                                    @Mark (10,
+                                           @Mark (10,
+                                                  @Mark (5,
+                                                         @Mark (16,@MT))))),
+                              stopname),
+                     ~[16]);
+        // stop has no effect on marks
+        assert_eq! (marksof (Mark (9, @Mark (14, @Mark (12, @MT))),stopname),
+                     ~[9,14,12]);
+        // rename where stop doesn't match:
+        assert_eq! (marksof (Mark (9, @Rename
+                                    (name1,
+                                     @Mark (4, @MT),
+                                     uints_to_name(&~[100,101,102]),
+                                     @Mark (14, @MT))),
+                              stopname),
+                     ~[9,14]);
+        // rename where stop does match
+        ;
+        assert_eq! (marksof (Mark(9, @Rename (name1,
+                                               @Mark (4, @MT),
+                                               stopname,
+                                               @Mark (14, @MT))),
+                              stopname),
+                     ~[9]);
+    }
+
+    // are ASTs encodable?
     #[test] fn check_asts_encodable() {
         let bogus_span = span {lo:BytePos(10),
                                hi:BytePos(20),
                                expn_info:None};
-        let _e : crate =
+        let e : crate =
             spanned{
             node: crate_{
                 module: _mod {view_items: ~[], items: ~[]},
@@ -1334,10 +1397,13 @@ mod test {
             },
             span: bogus_span};
         // doesn't matter which encoder we use....
-        let _f = (_e as std::serialize::Encodable::<std::json::Encoder>);
+        let _f = (@e as @std::serialize::Encodable<std::json::Encoder>);
     }
-    */
+
+
 }
+
+*/
 //
 // Local Variables:
 // mode: rust
diff --git a/src/libsyntax/ast_util.rs b/src/libsyntax/ast_util.rs
index e83a3ef8bad..59a640bb571 100644
--- a/src/libsyntax/ast_util.rs
+++ b/src/libsyntax/ast_util.rs
@@ -24,6 +24,7 @@ use core::str;
 use core::to_bytes;
 use core::vec;
 
+
 pub fn path_name_i(idents: &[ident], intr: @token::ident_interner) -> ~str {
     // FIXME: Bad copies (#2543 -- same for everything else that says "bad")
     str::connect(idents.map(|i| copy *intr.get(*i)), ~"::")
@@ -587,6 +588,280 @@ pub enum Privacy {
     Public
 }
 
+// HYGIENE FUNCTIONS
+
+/// Construct an identifier with the given repr and an empty context:
+pub fn mk_ident(repr: uint) -> ident { ident {repr: repr, ctxt: 0}}
+
+/// Extend a syntax context with a given mark
+pub fn mk_mark (m:Mrk,ctxt:SyntaxContext,table:&mut SCTable)
+    -> SyntaxContext {
+    idx_push(table,Mark(m,ctxt))
+}
+
+/// Extend a syntax context with a given rename
+pub fn mk_rename (id:ident, to:Name, tail:SyntaxContext, table: &mut SCTable)
+    -> SyntaxContext {
+    idx_push(table,Rename(id,to,tail))
+}
+
+/// Make a fresh syntax context table with EmptyCtxt in slot zero
+pub fn mk_sctable() -> SCTable { ~[EmptyCtxt] }
+
+/// Add a value to the end of a vec, return its index
+fn idx_push<T>(vec: &mut ~[T], +val: T) -> uint {
+    vec.push(val);
+    vec.len() - 1
+}
+
+/// Resolve a syntax object to a name, per MTWT.
+pub fn resolve (id : ident, table : &SCTable) -> Name {
+    match table[id.ctxt] {
+        EmptyCtxt => id.repr,
+        // ignore marks here:
+        Mark(_,subctxt) => resolve (ident{repr:id.repr, ctxt: subctxt},table),
+        // do the rename if necessary:
+        Rename(ident{repr,ctxt},toname,subctxt) => {
+            // this could be cached or computed eagerly:
+            let resolvedfrom = resolve(ident{repr:repr,ctxt:ctxt},table);
+            let resolvedthis = resolve(ident{repr:id.repr,ctxt:subctxt},table);
+            if ((resolvedthis == resolvedfrom)
+                && (marksof (ctxt,resolvedthis,table)
+                    == marksof (subctxt,resolvedthis,table))) {
+                toname
+            } else {
+                resolvedthis
+            }
+        }
+    }
+}
+
+/// Compute the marks associated with a syntax context.
+// it's not clear to me whether it's better to use a [] mutable
+// vector or a cons-list for this.
+pub fn marksof(ctxt: SyntaxContext, stopname: Name, table: &SCTable) -> ~[Mrk] {
+    let mut result = ~[];
+    let mut loopvar = ctxt;
+    loop {
+        match table[loopvar] {
+            EmptyCtxt => {return result;},
+            Mark(mark,tl) => {
+                xorPush(&mut result,mark);
+                loopvar = tl;
+            },
+            Rename(_,name,tl) => {
+                // see MTWT for details on the purpose of the stopname.
+                // short version: it prevents duplication of effort.
+                if (name == stopname) {
+                    return result;
+                } else {
+                    loopvar = tl;
+                }
+            }
+        }
+    }
+}
+
+/// Push a name... unless it matches the one on top, in which
+/// case pop and discard (so two of the same marks cancel)
+pub fn xorPush(marks: &mut ~[uint], mark: uint) {
+    if ((marks.len() > 0) && (getLast(marks) == mark)) {
+        marks.pop();
+    } else {
+        marks.push(mark);
+    }
+}
+
+// get the last element of a mutable array.
+// FIXME #4903: , must be a separate procedure for now.
+pub fn getLast(arr: &~[Mrk]) -> uint {
+    *arr.last()
+}
+
+
+#[cfg(test)]
+mod test {
+    use ast::*;
+    use super::*;
+    use core::io;
+
+    #[test] fn xorpush_test () {
+        let mut s = ~[];
+        xorPush(&mut s,14);
+        assert_eq!(s,~[14]);
+        xorPush(&mut s,14);
+        assert_eq!(s,~[]);
+        xorPush(&mut s,14);
+        assert_eq!(s,~[14]);
+        xorPush(&mut s,15);
+        assert_eq!(s,~[14,15]);
+        xorPush (&mut s,16);
+        assert_eq! (s,~[14,15,16]);
+        xorPush (&mut s,16);
+        assert_eq! (s,~[14,15]);
+        xorPush (&mut s,15);
+        assert_eq! (s,~[14]);
+    }
+
+    // convert a list of uints to an @~[ident]
+    // (ignores the interner completely)
+    fn uints_to_idents (uints: &~[uint]) -> @~[ident] {
+        @uints.map(|u|{ ident {repr:*u, ctxt: empty_ctxt} })
+    }
+
+    fn id (u : uint, s: SyntaxContext) -> ident {
+        ident{repr:u, ctxt: s}
+    }
+
+    // because of the SCTable, I now need a tidy way of
+    // creating syntax objects. Sigh.
+    #[deriving(Eq)]
+    enum TestSC {
+        M(Mrk),
+        R(ident,Name)
+    }
+
+    // unfold a vector of TestSC values into a SCTable,
+    // returning the resulting index
+    fn unfold_test_sc(tscs : ~[TestSC], tail: SyntaxContext, table : &mut SCTable)
+        -> SyntaxContext {
+        tscs.foldr(tail, |tsc : &TestSC,tail : SyntaxContext|
+                  {match *tsc {
+                      M(mrk) => mk_mark(mrk,tail,table),
+                      R(ident,name) => mk_rename(ident,name,tail,table)}})
+    }
+
+    // gather a SyntaxContext back into a vector of TestSCs
+    fn refold_test_sc(mut sc: SyntaxContext, table : &SCTable) -> ~[TestSC] {
+        let mut result = ~[];
+        loop {
+            match table[sc] {
+                EmptyCtxt => {return result;},
+                Mark(mrk,tail) => {
+                    result.push(M(mrk));
+                    sc = tail;
+                    loop;
+                },
+                Rename(id,name,tail) => {
+                    result.push(R(id,name));
+                    sc = tail;
+                    loop;
+                }
+            }
+        }
+    }
+
+    #[test] fn test_unfold_refold(){
+        let mut t = mk_sctable();
+
+        let test_sc = ~[M(3),R(id(101,0),14),M(9)];
+        assert_eq!(unfold_test_sc(test_sc,empty_ctxt,&mut t),3);
+        assert_eq!(t[1],Mark(9,0));
+        assert_eq!(t[2],Rename(id(101,0),14,1));
+        assert_eq!(t[3],Mark(3,2));
+        assert_eq!(refold_test_sc(3,&t),test_sc);
+    }
+
+
+    // extend a syntax context with a sequence of marks given
+    // in a vector. v[0] will be the outermost mark.
+    fn unfold_marks(mrks:~[Mrk],tail:SyntaxContext,table: &mut SCTable) -> SyntaxContext {
+        mrks.foldr(tail, |mrk:&Mrk,tail:SyntaxContext|
+                   {mk_mark(*mrk,tail,table)})
+    }
+
+    #[test] fn unfold_marks_test() {
+        let mut t = ~[EmptyCtxt];
+
+        assert_eq!(unfold_marks(~[3,7],empty_ctxt,&mut t),2);
+        assert_eq!(t[1],Mark(7,0));
+        assert_eq!(t[2],Mark(3,1));
+    }
+
+    #[test] fn test_marksof () {
+        let stopname = 242;
+        let name1 = 243;
+        let mut t = mk_sctable();
+        assert_eq!(marksof (empty_ctxt,stopname,&t),~[]);
+        // FIXME #5074: ANF'd to dodge nested calls
+        { let ans = unfold_marks(~[4,98],empty_ctxt,&mut t);
+         assert_eq! (marksof (ans,stopname,&t),~[4,98]);}
+        // does xoring work?
+        { let ans = unfold_marks(~[5,5,16],empty_ctxt,&mut t);
+         assert_eq! (marksof (ans,stopname,&t), ~[16]);}
+        // does nested xoring work?
+        { let ans = unfold_marks(~[5,10,10,5,16],empty_ctxt,&mut t);
+         assert_eq! (marksof (ans, stopname,&t), ~[16]);}
+        // rename where stop doesn't match:
+        { let chain = ~[M(9),
+                        R(id(name1,
+                             mk_mark (4, empty_ctxt,&mut t)),
+                          100101102),
+                        M(14)];
+         let ans = unfold_test_sc(chain,empty_ctxt,&mut t);
+         assert_eq! (marksof (ans, stopname, &t), ~[9,14]);}
+        // rename where stop does match
+        { let name1sc = mk_mark(4, empty_ctxt, &mut t);
+         let chain = ~[M(9),
+                       R(id(name1, name1sc),
+                         stopname),
+                       M(14)];
+         let ans = unfold_test_sc(chain,empty_ctxt,&mut t);
+         assert_eq! (marksof (ans, stopname, &t), ~[9]); }
+    }
+
+
+    #[test] fn resolve_tests () {
+        let a = 40;
+        let mut t = mk_sctable();
+        // - ctxt is MT
+        assert_eq!(resolve(id(a,empty_ctxt),&t),a);
+        // - simple ignored marks
+        { let sc = unfold_marks(~[1,2,3],empty_ctxt,&mut t);
+         assert_eq!(resolve(id(a,sc),&t),a);}
+        // - orthogonal rename where names don't match
+        { let sc = unfold_test_sc(~[R(id(50,empty_ctxt),51),M(12)],empty_ctxt,&mut t);
+         assert_eq!(resolve(id(a,sc),&t),a);}
+        // - rename where names do match, but marks don't
+        { let sc1 = mk_mark(1,empty_ctxt,&mut t);
+         let sc = unfold_test_sc(~[R(id(a,sc1),50),
+                                   M(1),
+                                   M(2)],
+                                 empty_ctxt,&mut t);
+        assert_eq!(resolve(id(a,sc),&t), a);}
+        // - rename where names and marks match
+        { let sc1 = unfold_test_sc(~[M(1),M(2)],empty_ctxt,&mut t);
+         let sc = unfold_test_sc(~[R(id(a,sc1),50),M(1),M(2)],empty_ctxt,&mut t);
+         assert_eq!(resolve(id(a,sc),&t), 50); }
+        // - rename where names and marks match by literal sharing
+        { let sc1 = unfold_test_sc(~[M(1),M(2)],empty_ctxt,&mut t);
+         let sc = unfold_test_sc(~[R(id(a,sc1),50)],sc1,&mut t);
+         assert_eq!(resolve(id(a,sc),&t), 50); }
+        // - two renames of the same var.. can only happen if you use
+        // local-expand to prevent the inner binding from being renamed
+        // during the rename-pass caused by the first:
+        io::println("about to run bad test");
+        { let sc = unfold_test_sc(~[R(id(a,empty_ctxt),50),
+                                    R(id(a,empty_ctxt),51)],
+                                  empty_ctxt,&mut t);
+         assert_eq!(resolve(id(a,sc),&t), 51); }
+        // the simplest double-rename:
+        { let a_to_a50 = mk_rename(id(a,empty_ctxt),50,empty_ctxt,&mut t);
+         let a50_to_a51 = mk_rename(id(a,a_to_a50),51,a_to_a50,&mut t);
+         assert_eq!(resolve(id(a,a50_to_a51),&t),51);
+         // mark on the outside doesn't stop rename:
+         let sc = mk_mark(9,a50_to_a51,&mut t);
+         assert_eq!(resolve(id(a,sc),&t),51);
+         // but mark on the inside does:
+         let a50_to_a51_b = unfold_test_sc(~[R(id(a,a_to_a50),51),
+                                              M(9)],
+                                           a_to_a50,
+                                           &mut t);
+         assert_eq!(resolve(id(a,a50_to_a51_b),&t),50);}
+    }
+
+}
+
 // Local Variables:
 // mode: rust
 // fill-column: 78;
diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index 09498f09a29..430402a8982 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -547,6 +547,53 @@ pub fn expand_crate(parse_sess: @mut parse::ParseSess,
     @f.fold_crate(&*c)
 }
 
+// given a function from paths to paths, produce
+// an ast_fold that applies that function:
+fn fun_to_path_folder(f: @fn(&ast::Path)->ast::Path) -> @ast_fold{
+    let afp = default_ast_fold();
+    let f_pre = @AstFoldFns{
+        fold_path : |p, _| f(p),
+        .. *afp
+    };
+    make_fold(f_pre)
+}
+/* going to have to figure out whether the table is passed in or
+extracted from TLS...
+// update the ctxts in a path to get a rename node
+fn ctxt_update_rename(from: ast::Name,
+                       fromctx: ast::SyntaxContext, to: ast::Name) ->
+    @fn(&ast::Path,@ast_fold)->ast::Path {
+    return |p:&ast::Path,_|
+    ast::Path {span: p.span,
+               global: p.global,
+               idents: p.idents.map(|id|
+                                    ast::ident{
+                                        repr: id.repr,
+                                        // this needs to be cached....
+                                        ctxt: Some(@ast::Rename(from,fromctx,
+                                                           to,id.ctxt))
+                                    }),
+               rp: p.rp,
+               types: p.types};
+}
+
+// update the ctxts in a path to get a mark node
+fn ctxt_update_mark(mark: uint) ->
+    @fn(&ast::Path,@ast_fold)->ast::Path {
+    return |p:&ast::Path,_|
+    ast::Path {span: p.span,
+               global: p.global,
+               idents: p.idents.map(|id|
+                                    ast::ident{
+                                        repr: id.repr,
+                                        // this needs to be cached....
+                                        ctxt: Some(@ast::Mark(mark,id.ctxt))
+                                    }),
+               rp: p.rp,
+               types: p.types};
+}
+*/
+
 #[cfg(test)]
 mod test {
     use super::*;
diff --git a/src/libsyntax/parse/common.rs b/src/libsyntax/parse/common.rs
index ae7dd8ff96f..f353d94894a 100644
--- a/src/libsyntax/parse/common.rs
+++ b/src/libsyntax/parse/common.rs
@@ -47,17 +47,29 @@ pub fn seq_sep_none() -> SeqSep {
     }
 }
 
+// maps any token back to a string. not necessary if you know it's
+// an identifier....
 pub fn token_to_str(reader: @reader, token: &token::Token) -> ~str {
     token::to_str(reader.interner(), token)
 }
 
 pub impl Parser {
+    // convert a token to a string using self's reader
+    fn token_to_str(&self, token: &token::Token) -> ~str {
+        token::to_str(self.reader.interner(), token)
+    }
+
+    // convert the current token to a string using self's reader
+    fn this_token_to_str(&self) -> ~str {
+        self.token_to_str(self.token)
+    }
+
     fn unexpected_last(&self, t: &token::Token) -> ! {
         self.span_fatal(
             *self.last_span,
             fmt!(
                 "unexpected token: `%s`",
-                token_to_str(self.reader, t)
+                self.token_to_str(t)
             )
         );
     }
@@ -66,7 +78,7 @@ pub impl Parser {
         self.fatal(
             fmt!(
                 "unexpected token: `%s`",
-                token_to_str(self.reader, &copy *self.token)
+                self.this_token_to_str()
             )
         );
     }
@@ -80,8 +92,8 @@ pub impl Parser {
             self.fatal(
                 fmt!(
                     "expected `%s` but found `%s`",
-                    token_to_str(self.reader, t),
-                    token_to_str(self.reader, &copy *self.token)
+                    self.token_to_str(t),
+                    self.this_token_to_str()
                 )
             )
         }
@@ -104,7 +116,7 @@ pub impl Parser {
                 self.fatal(
                     fmt!(
                         "expected ident, found `%s`",
-                        token_to_str(self.reader, &copy *self.token)
+                        self.this_token_to_str()
                     )
                 );
             }
@@ -128,12 +140,15 @@ pub impl Parser {
     // Storing keywords as interned idents instead of strings would be nifty.
 
     // A sanity check that the word we are asking for is a known keyword
+    // NOTE: this could be done statically....
     fn require_keyword(&self, word: &~str) {
         if !self.keywords.contains(word) {
             self.bug(fmt!("unknown keyword: %s", *word));
         }
     }
 
+    // return true when this token represents the given string, and is not
+    // followed immediately by :: .
     fn token_is_word(&self, word: &~str, tok: &token::Token) -> bool {
         match *tok {
             token::IDENT(sid, false) => { *self.id_to_str(sid) == *word }
@@ -150,6 +165,10 @@ pub impl Parser {
         self.token_is_keyword(word, &copy *self.token)
     }
 
+    fn id_is_any_keyword(&self, id: ast::ident) -> bool {
+        self.keywords.contains(self.id_to_str(id))
+    }
+
     fn is_any_keyword(&self, tok: &token::Token) -> bool {
         match *tok {
           token::IDENT(sid, false) => {
@@ -182,7 +201,7 @@ pub impl Parser {
                 fmt!(
                     "expected `%s`, found `%s`",
                     *word,
-                    token_to_str(self.reader, &copy *self.token)
+                    self.this_token_to_str()
                 )
             );
         }
@@ -248,9 +267,9 @@ pub impl Parser {
             );
         } else {
             let mut s: ~str = ~"expected `";
-            s += token_to_str(self.reader, &token::GT);
+            s += self.token_to_str(&token::GT);
             s += ~"`, found `";
-            s += token_to_str(self.reader, &copy *self.token);
+            s += self.this_token_to_str();
             s += ~"`";
             self.fatal(s);
         }
diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs
index fb3e8a5ded5..c1f781f8570 100644
--- a/src/libsyntax/parse/parser.rs
+++ b/src/libsyntax/parse/parser.rs
@@ -66,7 +66,7 @@ use codemap::{span, BytePos, spanned, mk_sp};
 use codemap;
 use parse::attr::parser_attr;
 use parse::classify;
-use parse::common::{seq_sep_none, token_to_str};
+use parse::common::{seq_sep_none};
 use parse::common::{seq_sep_trailing_disallowed, seq_sep_trailing_allowed};
 use parse::lexer::reader;
 use parse::lexer::TokenAndSpan;
@@ -252,8 +252,11 @@ pub fn Parser(sess: @mut ParseSess,
 pub struct Parser {
     sess: @mut ParseSess,
     cfg: crate_cfg,
+    // the current token:
     token: @mut token::Token,
+    // the span of the current token:
     span: @mut span,
+    // the span of the prior token:
     last_span: @mut span,
     buffer: @mut [TokenAndSpan, ..4],
     buffer_start: @mut int,
@@ -499,7 +502,7 @@ pub impl Parser {
             let hi = p.last_span.hi;
             debug!("parse_trait_methods(): trait method signature ends in \
                     `%s`",
-                   token_to_str(p.reader, &copy *p.token));
+                   self.this_token_to_str());
             match *p.token {
               token::SEMI => {
                 p.bump();
@@ -541,7 +544,7 @@ pub impl Parser {
                     p.fatal(
                         fmt!(
                             "expected `;` or `}` but found `%s`",
-                            token_to_str(p.reader, &copy *p.token)
+                            self.this_token_to_str()
                         )
                     );
                 }
@@ -698,7 +701,8 @@ pub impl Parser {
             let path = self.parse_path_with_tps(false);
             ty_path(path, self.get_id())
         } else {
-            self.fatal(~"expected type");
+            self.fatal(fmt!("expected type, found token %?",
+                            *self.token));
         };
 
         let sp = mk_sp(lo, self.last_span.hi);
@@ -1455,6 +1459,11 @@ pub impl Parser {
     fn parse_token_tree(&self) -> token_tree {
         maybe_whole!(deref self, nt_tt);
 
+        // this is the fall-through for the 'match' below.
+        // invariants: the current token is not a left-delimiter,
+        // not an EOF, and not the desired right-delimiter (if
+        // it were, parse_seq_to_before_end would have prevented
+        // reaching this point.
         fn parse_non_delim_tt_tok(p: &Parser) -> token_tree {
             maybe_whole!(deref p, nt_tt);
             match *p.token {
@@ -1463,7 +1472,7 @@ pub impl Parser {
                 p.fatal(
                     fmt!(
                         "incorrect close delimiter: `%s`",
-                        token_to_str(p.reader, &copy *p.token)
+                        p.this_token_to_str()
                     )
                 );
               }
@@ -1505,18 +1514,17 @@ pub impl Parser {
 
         match *self.token {
             token::EOF => {
-                self.fatal(~"file ended in the middle of a macro invocation");
+                self.fatal(~"file ended with unbalanced delimiters");
             }
             token::LPAREN | token::LBRACE | token::LBRACKET => {
-                // tjc: ??????
-                let ket = token::flip_delimiter(&*self.token);
+                let close_delim = token::flip_delimiter(&*self.token);
                 tt_delim(
                     vec::append(
                         // the open delimiter:
                         ~[parse_any_tt_tok(self)],
                         vec::append(
                             self.parse_seq_to_before_end(
-                                &ket,
+                                &close_delim,
                                 seq_sep_none(),
                                 |p| p.parse_token_tree()
                             ),
@@ -1530,6 +1538,8 @@ pub impl Parser {
         }
     }
 
+    // parse a stream of tokens into a list of token_trees,
+    // up to EOF.
     fn parse_all_token_trees(&self) -> ~[token_tree] {
         let mut tts = ~[];
         while *self.token != token::EOF {
@@ -2052,6 +2062,7 @@ pub impl Parser {
         return e;
     }
 
+    // parse the RHS of a local variable declaration (e.g. '= 14;')
     fn parse_initializer(&self) -> Option<@expr> {
         match *self.token {
           token::EQ => {
@@ -2138,7 +2149,7 @@ pub impl Parser {
                     self.fatal(
                         fmt!(
                             "expected `}`, found `%s`",
-                            token_to_str(self.reader, &copy *self.token)
+                            self.this_token_to_str()
                         )
                     );
                 }
@@ -2406,6 +2417,7 @@ pub impl Parser {
         pat_ident(binding_mode, name, sub)
     }
 
+    // parse a local variable declaration
     fn parse_local(&self, is_mutbl: bool,
                    allow_init: bool) -> @local {
         let lo = self.span.lo;
@@ -2651,7 +2663,7 @@ pub impl Parser {
                                             fmt!(
                                                 "expected `;` or `}` after \
                                                 expression but found `%s`",
-                                                token_to_str(self.reader, &t)
+                                                self.token_to_str(&t)
                                             )
                                         );
                                     }
@@ -2866,7 +2878,7 @@ pub impl Parser {
             self.fatal(
                 fmt!(
                     "expected `self` but found `%s`",
-                    token_to_str(self.reader, &copy *self.token)
+                    self.this_token_to_str()
                 )
             );
         }
@@ -2990,7 +3002,7 @@ pub impl Parser {
                     self.fatal(
                         fmt!(
                             "expected `,` or `)`, found `%s`",
-                            token_to_str(self.reader, &copy *self.token)
+                            self.this_token_to_str()
                         )
                     );
                 }
@@ -3270,7 +3282,7 @@ pub impl Parser {
                 fmt!(
                     "expected `{`, `(`, or `;` after struct name \
                     but found `%s`",
-                    token_to_str(self.reader, &copy *self.token)
+                    self.this_token_to_str()
                 )
             );
         }
@@ -3320,7 +3332,7 @@ pub impl Parser {
                     copy *self.span,
                     fmt!(
                         "expected `;`, `,`, or '}' but found `%s`",
-                        token_to_str(self.reader, &copy *self.token)
+                        self.this_token_to_str()
                     )
                 );
             }
@@ -3422,7 +3434,7 @@ pub impl Parser {
                 self.fatal(
                     fmt!(
                         "expected item but found `%s`",
-                        token_to_str(self.reader, &copy *self.token)
+                        self.this_token_to_str()
                     )
                 );
               }
@@ -3682,7 +3694,7 @@ pub impl Parser {
                 copy *self.span,
                 fmt!(
                     "expected `{` or `mod` but found `%s`",
-                    token_to_str(self.reader, &copy *self.token)
+                    self.this_token_to_str()
                 )
             );
         }
@@ -3695,7 +3707,7 @@ pub impl Parser {
                         copy *self.span,
                         fmt!(
                             "expected foreign module name but found `%s`",
-                            token_to_str(self.reader, &copy *self.token)
+                            self.this_token_to_str()
                         )
                     );
                 }
@@ -4279,7 +4291,7 @@ pub impl Parser {
                                             rp: None,
                                             types: ~[] };
                     return @spanned(lo, self.span.hi,
-                                 view_path_glob(path, self.get_id()));
+                                    view_path_glob(path, self.get_id()));
                   }
 
                   _ => break
diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs
index 54b2ad85147..cf05a4375a8 100644
--- a/src/libsyntax/parse/token.rs
+++ b/src/libsyntax/parse/token.rs
@@ -309,50 +309,50 @@ pub fn is_bar(t: &Token) -> bool {
 pub mod special_idents {
     use ast::ident;
 
-    pub static underscore : ident = ident { repr: 0u };
-    pub static anon : ident = ident { repr: 1u };
-    pub static dtor : ident = ident { repr: 2u }; // 'drop', but that's
+    pub static underscore : ident = ident { repr: 0u, ctxt: 0};
+    pub static anon : ident = ident { repr: 1u, ctxt: 0};
+    pub static dtor : ident = ident { repr: 2u, ctxt: 0}; // 'drop', but that's
                                                  // reserved
-    pub static invalid : ident = ident { repr: 3u }; // ''
-    pub static unary : ident = ident { repr: 4u };
-    pub static not_fn : ident = ident { repr: 5u };
-    pub static idx_fn : ident = ident { repr: 6u };
-    pub static unary_minus_fn : ident = ident { repr: 7u };
-    pub static clownshoes_extensions : ident = ident { repr: 8u };
+    pub static invalid : ident = ident { repr: 3u, ctxt: 0}; // ''
+    pub static unary : ident = ident { repr: 4u, ctxt: 0};
+    pub static not_fn : ident = ident { repr: 5u, ctxt: 0};
+    pub static idx_fn : ident = ident { repr: 6u, ctxt: 0};
+    pub static unary_minus_fn : ident = ident { repr: 7u, ctxt: 0};
+    pub static clownshoes_extensions : ident = ident { repr: 8u, ctxt: 0};
 
-    pub static self_ : ident = ident { repr: 9u }; // 'self'
+    pub static self_ : ident = ident { repr: 9u, ctxt: 0}; // 'self'
 
     /* for matcher NTs */
-    pub static item : ident = ident { repr: 10u };
-    pub static block : ident = ident { repr: 11u };
-    pub static stmt : ident = ident { repr: 12u };
-    pub static pat : ident = ident { repr: 13u };
-    pub static expr : ident = ident { repr: 14u };
-    pub static ty : ident = ident { repr: 15u };
-    pub static ident : ident = ident { repr: 16u };
-    pub static path : ident = ident { repr: 17u };
-    pub static tt : ident = ident { repr: 18u };
-    pub static matchers : ident = ident { repr: 19u };
-
-    pub static str : ident = ident { repr: 20u }; // for the type
+    pub static item : ident = ident { repr: 10u, ctxt: 0};
+    pub static block : ident = ident { repr: 11u, ctxt: 0};
+    pub static stmt : ident = ident { repr: 12u, ctxt: 0};
+    pub static pat : ident = ident { repr: 13u, ctxt: 0};
+    pub static expr : ident = ident { repr: 14u, ctxt: 0};
+    pub static ty : ident = ident { repr: 15u, ctxt: 0};
+    pub static ident : ident = ident { repr: 16u, ctxt: 0};
+    pub static path : ident = ident { repr: 17u, ctxt: 0};
+    pub static tt : ident = ident { repr: 18u, ctxt: 0};
+    pub static matchers : ident = ident { repr: 19u, ctxt: 0};
+
+    pub static str : ident = ident { repr: 20u, ctxt: 0}; // for the type
 
     /* outside of libsyntax */
-    pub static ty_visitor : ident = ident { repr: 21u };
-    pub static arg : ident = ident { repr: 22u };
-    pub static descrim : ident = ident { repr: 23u };
-    pub static clownshoe_abi : ident = ident { repr: 24u };
-    pub static clownshoe_stack_shim : ident = ident { repr: 25u };
-    pub static tydesc : ident = ident { repr: 26u };
-    pub static literally_dtor : ident = ident { repr: 27u };
-    pub static main : ident = ident { repr: 28u };
-    pub static opaque : ident = ident { repr: 29u };
-    pub static blk : ident = ident { repr: 30u };
-    pub static static : ident = ident { repr: 31u };
-    pub static intrinsic : ident = ident { repr: 32u };
-    pub static clownshoes_foreign_mod: ident = ident { repr: 33 };
-    pub static unnamed_field: ident = ident { repr: 34 };
-    pub static c_abi: ident = ident { repr: 35 };
-    pub static type_self: ident = ident { repr: 36 };    // `Self`
+    pub static ty_visitor : ident = ident { repr: 21u, ctxt: 0};
+    pub static arg : ident = ident { repr: 22u, ctxt: 0};
+    pub static descrim : ident = ident { repr: 23u, ctxt: 0};
+    pub static clownshoe_abi : ident = ident { repr: 24u, ctxt: 0};
+    pub static clownshoe_stack_shim : ident = ident { repr: 25u, ctxt: 0};
+    pub static tydesc : ident = ident { repr: 26u, ctxt: 0};
+    pub static literally_dtor : ident = ident { repr: 27u, ctxt: 0};
+    pub static main : ident = ident { repr: 28u, ctxt: 0};
+    pub static opaque : ident = ident { repr: 29u, ctxt: 0};
+    pub static blk : ident = ident { repr: 30u, ctxt: 0};
+    pub static static : ident = ident { repr: 31u, ctxt: 0};
+    pub static intrinsic : ident = ident { repr: 32u, ctxt: 0};
+    pub static clownshoes_foreign_mod: ident = ident { repr: 33u, ctxt: 0};
+    pub static unnamed_field: ident = ident { repr: 34u, ctxt: 0};
+    pub static c_abi: ident = ident { repr: 35u, ctxt: 0};
+    pub static type_self: ident = ident { repr: 36u, ctxt: 0};    // `Self`
 }
 
 pub struct ident_interner {
@@ -361,10 +361,10 @@ pub struct ident_interner {
 
 pub impl ident_interner {
     fn intern(&self, val: @~str) -> ast::ident {
-        ast::ident { repr: self.interner.intern(val) }
+        ast::ident { repr: self.interner.intern(val), ctxt: 0}
     }
     fn gensym(&self, val: @~str) -> ast::ident {
-        ast::ident { repr: self.interner.gensym(val) }
+        ast::ident { repr: self.interner.gensym(val), ctxt: 0}
     }
     fn get(&self, idx: ast::ident) -> @~str {
         self.interner.get(idx.repr)
@@ -374,16 +374,6 @@ pub impl ident_interner {
     }
 }
 
-/* Key for thread-local data for sneaking interner information to the
- * encoder/decoder. It sounds like a hack because it is one.
- * Bonus ultra-hack: functions as keys don't work across crates,
- * so we have to use a unique number. See taskgroup_key! in task.rs
- * for another case of this. */
-macro_rules! interner_key (
-    () => (cast::transmute::<(uint, uint), &fn(+v: @@token::ident_interner)>(
-        (-3 as uint, 0u)))
-)
-
 pub fn mk_ident_interner() -> @ident_interner {
     unsafe {
         match task::local_data::local_data_get(interner_key!()) {
diff --git a/src/libsyntax/syntax.rc b/src/libsyntax/syntax.rc
index 56b17fb8d0a..c1b857a6cdb 100644
--- a/src/libsyntax/syntax.rc
+++ b/src/libsyntax/syntax.rc
@@ -30,6 +30,13 @@ extern mod std(vers = "0.7-pre");
 
 use core::*;
 
+// allow the interner_key macro
+// to escape this module:
+#[macro_escape]
+pub mod util {
+    pub mod interner;
+}
+
 pub mod syntax {
     pub use ext;
     pub use parse;
@@ -45,9 +52,6 @@ pub mod ast_util;
 pub mod ast_map;
 pub mod visit;
 pub mod fold;
-pub mod util {
-    pub mod interner;
-}
 
 
 #[path = "parse/mod.rs"]
diff --git a/src/libsyntax/util/interner.rs b/src/libsyntax/util/interner.rs
index 1133fd850d8..75bcac1b163 100644
--- a/src/libsyntax/util/interner.rs
+++ b/src/libsyntax/util/interner.rs
@@ -12,6 +12,9 @@
 // allows bidirectional lookup; i.e. given a value, one can easily find the
 // type, and vice versa.
 
+// allow the interner_key macro to escape this module:
+#[macro_escape];
+
 use core::prelude::*;
 use core::hashmap::HashMap;
 
@@ -66,6 +69,17 @@ pub impl<T:Eq + IterBytes + Hash + Const + Copy> Interner<T> {
     fn len(&self) -> uint { let vect = &*self.vect; vect.len() }
 }
 
+/* Key for thread-local data for sneaking interner information to the
+* encoder/decoder. It sounds like a hack because it is one.
+* Bonus ultra-hack: functions as keys don't work across crates,
+* so we have to use a unique number. See taskgroup_key! in task.rs
+* for another case of this. */
+macro_rules! interner_key (
+    () => (cast::transmute::<(uint, uint),
+           &fn(+v: @@::parse::token::ident_interner)>(
+        (-3 as uint, 0u)))
+)
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -109,4 +123,4 @@ mod tests {
         assert_eq!(i.get(2), @~"Carol");
         assert_eq!(i.intern(@~"Bob"), 1);
     }
-}
\ No newline at end of file
+}