auto merge of #6938 : jbclements/rust/hygiene_fns_and_cleanup, r=jbclements

I'm close to flipping the switch on hygiene for let-bound identifiers.  This commit adds a bunch of support functions for that change... but also a huge amount of cleanup in syntax.rc. The most interesting of these are

- the use of TLS for the interners everywhere.  We had already breached the "no-global-state" dam by using TLS for encoding, and it saves a lot of code just to use it everywhere.
Also, there were many places where two or more interners were passed in attached to different structures, and the danger of having those diverge seemed greater that the danger of having a single one get big and heavy. If the interner size proves to be a problem, it should be quite simple to add a "parameterize"-like dynamic binding form--because we don't have interesting continuation operations (or tail calling, mostly) this should just be a case of a mutation followed by another later mutation. Again, this is only if the interner gets too big.

- I renamed the "repr" field of the identifier to "name". I can see the case for "repr" when there's only one field in the structure, but that's no longer the case; there's now a name and a context (both are uints).

- the interner now just maps between strings and uints, rather than between idents and uints. The former state made perfect sense when identifiers didn't have syntax contexts, but that's no longer the case.

I've run this patch against a fairly recent incoming, and it appears to pass all tests. Let's see if it can be merged....

6 files changed, 217 insertions, 182 deletions

diff --git a/src/libsyntax/parse/comments.rs b/src/libsyntax/parse/comments.rs
index 2f166ae89ef..dab8dd3b4b6 100644
--- a/src/libsyntax/parse/comments.rs
+++ b/src/libsyntax/parse/comments.rs
@@ -18,6 +18,7 @@ use parse::lexer::{StringReader, bump, is_eof, nextch, TokenAndSpan};
 use parse::lexer::{is_line_non_doc_comment, is_block_non_doc_comment};
 use parse::lexer;
 use parse::token;
+use parse::token::{get_ident_interner};
 use parse;
 
 use core::io;
@@ -323,12 +324,9 @@ pub fn gather_comments_and_literals(span_diagnostic:
                                     srdr: @io::Reader)
                                  -> (~[cmnt], ~[lit]) {
     let src = @str::from_bytes(srdr.read_whole_stream());
-    let itr = parse::token::mk_fake_ident_interner();
     let cm = CodeMap::new();
     let filemap = cm.new_filemap(path, src);
-    let rdr = lexer::new_low_level_string_reader(span_diagnostic,
-                                                 filemap,
-                                                 itr);
+    let rdr = lexer::new_low_level_string_reader(span_diagnostic, filemap);
 
     let mut comments: ~[cmnt] = ~[];
     let mut literals: ~[lit] = ~[];
@@ -358,7 +356,7 @@ pub fn gather_comments_and_literals(span_diagnostic:
             debug!("tok lit: %s", s);
             literals.push(lit {lit: s, pos: sp.lo});
         } else {
-            debug!("tok: %s", token::to_str(rdr.interner, &tok));
+            debug!("tok: %s", token::to_str(get_ident_interner(), &tok));
         }
         first_read = false;
     }
diff --git a/src/libsyntax/parse/common.rs b/src/libsyntax/parse/common.rs
index 8a930cf9afd..9fb69821953 100644
--- a/src/libsyntax/parse/common.rs
+++ b/src/libsyntax/parse/common.rs
@@ -16,6 +16,7 @@ use parse::lexer::reader;
 use parse::parser::Parser;
 use parse::token::keywords;
 use parse::token;
+use parse::token::{get_ident_interner};
 
 use opt_vec;
 use opt_vec::OptVec;
@@ -49,13 +50,13 @@ pub fn seq_sep_none() -> SeqSep {
 // maps any token back to a string. not necessary if you know it's
 // an identifier....
 pub fn token_to_str(reader: @reader, token: &token::Token) -> ~str {
-    token::to_str(reader.interner(), token)
+    token::to_str(get_ident_interner(), token)
 }
 
 impl Parser {
     // convert a token to a string using self's reader
     pub fn token_to_str(&self, token: &token::Token) -> ~str {
-        token::to_str(self.reader.interner(), token)
+        token::to_str(get_ident_interner(), token)
     }
 
     // convert the current token to a string using self's reader
@@ -142,7 +143,7 @@ impl Parser {
     // true. Otherwise, return false.
     pub fn eat_keyword(&self, kw: keywords::Keyword) -> bool {
         let is_kw = match *self.token {
-            token::IDENT(sid, false) => kw.to_ident().repr == sid.repr,
+            token::IDENT(sid, false) => kw.to_ident().name == sid.name,
             _ => false
         };
         if is_kw { self.bump() }
diff --git a/src/libsyntax/parse/lexer.rs b/src/libsyntax/parse/lexer.rs
index 0eb933e6c3a..84700f052c9 100644
--- a/src/libsyntax/parse/lexer.rs
+++ b/src/libsyntax/parse/lexer.rs
@@ -17,6 +17,7 @@ use diagnostic::span_handler;
 use ext::tt::transcribe::{tt_next_token};
 use ext::tt::transcribe::{dup_tt_reader};
 use parse::token;
+use parse::token::{str_to_ident};
 
 use core::char;
 use core::either;
@@ -30,7 +31,6 @@ pub trait reader {
     fn next_token(@mut self) -> TokenAndSpan;
     fn fatal(@mut self, ~str) -> !;
     fn span_diag(@mut self) -> @span_handler;
-    fn interner(@mut self) -> @token::ident_interner;
     fn peek(@mut self) -> TokenAndSpan;
     fn dup(@mut self) -> @reader;
 }
@@ -50,25 +50,22 @@ pub struct StringReader {
     // The last character to be read
     curr: char,
     filemap: @codemap::FileMap,
-    interner: @token::ident_interner,
     /* cached: */
     peek_tok: token::Token,
     peek_span: span
 }
 
 pub fn new_string_reader(span_diagnostic: @span_handler,
-                         filemap: @codemap::FileMap,
-                         itr: @token::ident_interner)
+                         filemap: @codemap::FileMap)
                       -> @mut StringReader {
-    let r = new_low_level_string_reader(span_diagnostic, filemap, itr);
+    let r = new_low_level_string_reader(span_diagnostic, filemap);
     string_advance_token(r); /* fill in peek_* */
     return r;
 }
 
 /* For comments.rs, which hackily pokes into 'pos' and 'curr' */
 pub fn new_low_level_string_reader(span_diagnostic: @span_handler,
-                                   filemap: @codemap::FileMap,
-                                   itr: @token::ident_interner)
+                                   filemap: @codemap::FileMap)
                                 -> @mut StringReader {
     // Force the initial reader bump to start on a fresh line
     let initial_char = '\n';
@@ -79,7 +76,6 @@ pub fn new_low_level_string_reader(span_diagnostic: @span_handler,
         col: CharPos(0),
         curr: initial_char,
         filemap: filemap,
-        interner: itr,
         /* dummy values; not read */
         peek_tok: token::EOF,
         peek_span: codemap::dummy_sp()
@@ -100,7 +96,6 @@ fn dup_string_reader(r: @mut StringReader) -> @mut StringReader {
         col: r.col,
         curr: r.curr,
         filemap: r.filemap,
-        interner: r.interner,
         peek_tok: copy r.peek_tok,
         peek_span: copy r.peek_span
     }
@@ -121,7 +116,6 @@ impl reader for StringReader {
         self.span_diagnostic.span_fatal(copy self.peek_span, m)
     }
     fn span_diag(@mut self) -> @span_handler { self.span_diagnostic }
-    fn interner(@mut self) -> @token::ident_interner { self.interner }
     fn peek(@mut self) -> TokenAndSpan {
         TokenAndSpan {
             tok: copy self.peek_tok,
@@ -138,7 +132,6 @@ impl reader for TtReader {
         self.sp_diag.span_fatal(copy self.cur_span, m);
     }
     fn span_diag(@mut self) -> @span_handler { self.sp_diag }
-    fn interner(@mut self) -> @token::ident_interner { self.interner }
     fn peek(@mut self) -> TokenAndSpan {
         TokenAndSpan {
             tok: copy self.cur_tok,
@@ -277,7 +270,7 @@ fn consume_any_line_comment(rdr: @mut StringReader)
                 // but comments with only more "/"s are not
                 if !is_line_non_doc_comment(acc) {
                     return Some(TokenAndSpan{
-                        tok: token::DOC_COMMENT(rdr.interner.intern(acc)),
+                        tok: token::DOC_COMMENT(str_to_ident(acc)),
                         sp: codemap::mk_sp(start_bpos, rdr.pos)
                     });
                 }
@@ -331,7 +324,7 @@ fn consume_block_comment(rdr: @mut StringReader)
             // but comments with only "*"s between two "/"s are not
             if !is_block_non_doc_comment(acc) {
                 return Some(TokenAndSpan{
-                    tok: token::DOC_COMMENT(rdr.interner.intern(acc)),
+                    tok: token::DOC_COMMENT(str_to_ident(acc)),
                     sp: codemap::mk_sp(start_bpos, rdr.pos)
                 });
             }
@@ -477,12 +470,12 @@ fn scan_number(c: char, rdr: @mut StringReader) -> token::Token {
         if c == '3' && n == '2' {
             bump(rdr);
             bump(rdr);
-            return token::LIT_FLOAT(rdr.interner.intern(num_str),
+            return token::LIT_FLOAT(str_to_ident(num_str),
                                  ast::ty_f32);
         } else if c == '6' && n == '4' {
             bump(rdr);
             bump(rdr);
-            return token::LIT_FLOAT(rdr.interner.intern(num_str),
+            return token::LIT_FLOAT(str_to_ident(num_str),
                                  ast::ty_f64);
             /* FIXME (#2252): if this is out of range for either a
             32-bit or 64-bit float, it won't be noticed till the
@@ -494,9 +487,9 @@ fn scan_number(c: char, rdr: @mut StringReader) -> token::Token {
     }
     if is_float {
         if is_machine_float {
-            return token::LIT_FLOAT(rdr.interner.intern(num_str), ast::ty_f);
+            return token::LIT_FLOAT(str_to_ident(num_str), ast::ty_f);
         }
-        return token::LIT_FLOAT_UNSUFFIXED(rdr.interner.intern(num_str));
+        return token::LIT_FLOAT_UNSUFFIXED(str_to_ident(num_str));
     } else {
         if str::len(num_str) == 0u {
             rdr.fatal(~"no valid digits found for number");
@@ -559,7 +552,7 @@ fn next_token_inner(rdr: @mut StringReader) -> token::Token {
         let is_mod_name = c == ':' && nextch(rdr) == ':';
 
         // FIXME: perform NFKC normalization here. (Issue #2253)
-        return token::IDENT(rdr.interner.intern(accum_str), is_mod_name);
+        return token::IDENT(str_to_ident(accum_str), is_mod_name);
     }
     if is_dec_digit(c) {
         return scan_number(c, rdr);
@@ -669,7 +662,7 @@ fn next_token_inner(rdr: @mut StringReader) -> token::Token {
                 lifetime_name.push_char(rdr.curr);
                 bump(rdr);
             }
-            return token::LIFETIME(rdr.interner.intern(lifetime_name));
+            return token::LIFETIME(str_to_ident(lifetime_name));
         }
 
         // Otherwise it is a character constant:
@@ -742,7 +735,7 @@ fn next_token_inner(rdr: @mut StringReader) -> token::Token {
             }
         }
         bump(rdr);
-        return token::LIT_STR(rdr.interner.intern(accum_str));
+        return token::LIT_STR(str_to_ident(accum_str));
       }
       '-' => {
         if nextch(rdr) == '>' {
@@ -786,10 +779,10 @@ mod test {
     use core::option::None;
     use diagnostic;
     use parse::token;
+    use parse::token::{str_to_ident};
 
     // represents a testing reader (incl. both reader and interner)
     struct Env {
-        interner: @token::ident_interner,
         string_reader: @mut StringReader
     }
 
@@ -797,20 +790,18 @@ mod test {
     fn setup(teststr: ~str) -> Env {
         let cm = CodeMap::new();
         let fm = cm.new_filemap(~"zebra.rs", @teststr);
-        let ident_interner = token::get_ident_interner();
         let span_handler =
             diagnostic::mk_span_handler(diagnostic::mk_handler(None),@cm);
         Env {
-            interner: ident_interner,
-            string_reader: new_string_reader(span_handler,fm,ident_interner)
+            string_reader: new_string_reader(span_handler,fm)
         }
     }
 
     #[test] fn t1 () {
-        let Env {interner: ident_interner, string_reader} =
+        let Env {string_reader} =
             setup(~"/* my source file */ \
                     fn main() { io::println(~\"zebra\"); }\n");
-        let id = ident_interner.intern("fn");
+        let id = str_to_ident("fn");
         let tok1 = string_reader.next_token();
         let tok2 = TokenAndSpan{
             tok:token::IDENT(id, false),
@@ -821,7 +812,7 @@ mod test {
         // read another token:
         let tok3 = string_reader.next_token();
         let tok4 = TokenAndSpan{
-            tok:token::IDENT(ident_interner.intern("main"), false),
+            tok:token::IDENT(str_to_ident("main"), false),
             sp:span {lo:BytePos(24),hi:BytePos(28),expn_info: None}};
         assert_eq!(tok3,tok4);
         // the lparen is already read:
@@ -839,39 +830,39 @@ mod test {
     }
 
     // make the identifier by looking up the string in the interner
-    fn mk_ident (env: Env, id: &str, is_mod_name: bool) -> token::Token {
-        token::IDENT (env.interner.intern(id),is_mod_name)
+    fn mk_ident (id: &str, is_mod_name: bool) -> token::Token {
+        token::IDENT (str_to_ident(id),is_mod_name)
     }
 
     #[test] fn doublecolonparsing () {
         let env = setup (~"a b");
         check_tokenization (env,
-                           ~[mk_ident (env,"a",false),
-                             mk_ident (env,"b",false)]);
+                           ~[mk_ident("a",false),
+                             mk_ident("b",false)]);
     }
 
     #[test] fn dcparsing_2 () {
         let env = setup (~"a::b");
         check_tokenization (env,
-                           ~[mk_ident (env,"a",true),
+                           ~[mk_ident("a",true),
                              token::MOD_SEP,
-                             mk_ident (env,"b",false)]);
+                             mk_ident("b",false)]);
     }
 
     #[test] fn dcparsing_3 () {
         let env = setup (~"a ::b");
         check_tokenization (env,
-                           ~[mk_ident (env,"a",false),
+                           ~[mk_ident("a",false),
                              token::MOD_SEP,
-                             mk_ident (env,"b",false)]);
+                             mk_ident("b",false)]);
     }
 
     #[test] fn dcparsing_4 () {
         let env = setup (~"a:: b");
         check_tokenization (env,
-                           ~[mk_ident (env,"a",true),
+                           ~[mk_ident("a",true),
                              token::MOD_SEP,
-                             mk_ident (env,"b",false)]);
+                             mk_ident("b",false)]);
     }
 
     #[test] fn character_a() {
@@ -899,7 +890,7 @@ mod test {
         let env = setup(~"'abc");
         let TokenAndSpan {tok, sp: _} =
             env.string_reader.next_token();
-        let id = env.interner.intern("abc");
+        let id = token::str_to_ident("abc");
         assert_eq!(tok, token::LIFETIME(id));
     }
 
diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs
index 9c716f5631f..d7248204e1c 100644
--- a/src/libsyntax/parse/mod.rs
+++ b/src/libsyntax/parse/mod.rs
@@ -19,7 +19,6 @@ use diagnostic::{span_handler, mk_span_handler, mk_handler, Emitter};
 use parse::attr::parser_attr;
 use parse::lexer::reader;
 use parse::parser::Parser;
-use parse::token::{ident_interner, get_ident_interner};
 
 use core::io;
 use core::option::{None, Option, Some};
@@ -43,14 +42,10 @@ pub mod classify;
 pub mod obsolete;
 
 // info about a parsing session.
-// This structure and the reader both have
-// an interner associated with them. If they're
-// not the same, bad things can happen.
 pub struct ParseSess {
     cm: @codemap::CodeMap, // better be the same as the one in the reader!
     next_id: node_id,
     span_diagnostic: @span_handler, // better be the same as the one in the reader!
-    interner: @ident_interner,
 }
 
 pub fn new_parse_sess(demitter: Option<Emitter>) -> @mut ParseSess {
@@ -59,7 +54,6 @@ pub fn new_parse_sess(demitter: Option<Emitter>) -> @mut ParseSess {
         cm: cm,
         next_id: 1,
         span_diagnostic: mk_span_handler(mk_handler(demitter), cm),
-        interner: get_ident_interner(),
     }
 }
 
@@ -70,7 +64,6 @@ pub fn new_parse_sess_special_handler(sh: @span_handler,
         cm: cm,
         next_id: 1,
         span_diagnostic: sh,
-        interner: get_ident_interner(),
     }
 }
 
@@ -312,9 +305,7 @@ pub fn filemap_to_tts(sess: @mut ParseSess, filemap: @FileMap)
     // it appears to me that the cfg doesn't matter here... indeed,
     // parsing tt's probably shouldn't require a parser at all.
     let cfg = ~[];
-    let srdr = lexer::new_string_reader(copy sess.span_diagnostic,
-                                        filemap,
-                                        sess.interner);
+    let srdr = lexer::new_string_reader(copy sess.span_diagnostic, filemap);
     let p1 = Parser(sess, cfg, srdr as @reader);
     p1.parse_all_token_trees()
 }
@@ -325,7 +316,6 @@ pub fn tts_to_parser(sess: @mut ParseSess,
                      cfg: ast::crate_cfg) -> Parser {
     let trdr = lexer::new_tt_reader(
         copy sess.span_diagnostic,
-        sess.interner,
         None,
         tts
     );
@@ -351,12 +341,13 @@ mod test {
     use codemap::{span, BytePos, spanned};
     use opt_vec;
     use ast;
+    use ast::{new_ident};
     use abi;
     use parse::parser::Parser;
-    use parse::token::intern;
-    use util::parser_testing::{string_to_tts_and_sess,string_to_parser};
+    use parse::token::{intern, str_to_ident};
+    use util::parser_testing::{string_to_tts_and_sess, string_to_parser};
     use util::parser_testing::{string_to_expr, string_to_item};
-    use util::parser_testing::{string_to_stmt};
+    use util::parser_testing::{string_to_stmt, strs_to_idents};
 
     // map a string to tts, return the tt without its parsesess
     fn string_to_tts_only(source_str : @~str) -> ~[ast::token_tree] {
@@ -377,17 +368,12 @@ mod test {
         span{lo:BytePos(a),hi:BytePos(b),expn_info:None}
     }
 
-    // convert a vector of uints to a vector of ast::idents
-    fn ints_to_idents(ids: ~[~str]) -> ~[ast::ident] {
-        ids.map(|u| intern(*u))
-    }
-
     #[test] fn path_exprs_1 () {
         assert_eq!(string_to_expr(@~"a"),
                    @ast::expr{id:1,
                               node:ast::expr_path(@ast::Path {span:sp(0,1),
                                                               global:false,
-                                                              idents:~[intern("a")],
+                                                              idents:~[str_to_ident("a")],
                                                               rp:None,
                                                               types:~[]}),
                               span:sp(0,1)})
@@ -399,7 +385,7 @@ mod test {
                                node:ast::expr_path(
                                    @ast::Path {span:sp(0,6),
                                                global:true,
-                                               idents:ints_to_idents(~[~"a",~"b"]),
+                                               idents:strs_to_idents(~["a","b"]),
                                                rp:None,
                                                types:~[]}),
                               span:sp(0,6)})
@@ -449,7 +435,7 @@ mod test {
                                                   node:ast::expr_path(
                                                       @ast::Path{span:sp(7,8),
                                                                  global:false,
-                                                                 idents:~[intern("d")],
+                                                                 idents:~[str_to_ident("d")],
                                                                  rp:None,
                                                                  types:~[]
                                                                 }),
@@ -466,7 +452,7 @@ mod test {
                                @ast::Path{
                                    span:sp(0,1),
                                    global:false,
-                                   idents:~[intern("b")],
+                                   idents:~[str_to_ident("b")],
                                    rp:None,
                                    types: ~[]}),
                            span: sp(0,1)},
@@ -487,7 +473,7 @@ mod test {
                                                   @ast::Path{
                                                       span:sp(0,1),
                                                       global:false,
-                                                      idents:~[intern("b")],
+                                                      idents:~[str_to_ident("b")],
                                                       rp: None,
                                                       types: ~[]},
                                                   None // no idea
@@ -506,7 +492,7 @@ mod test {
                                         span:sp(4,4), // this is bizarre...
                                         // check this in the original parser?
                                         global:false,
-                                        idents:~[intern("int")],
+                                        idents:~[str_to_ident("int")],
                                         rp: None,
                                         types: ~[]},
                                                        2),
@@ -516,7 +502,7 @@ mod test {
                                                            @ast::Path{
                                                                span:sp(0,1),
                                                                global:false,
-                                                               idents:~[intern("b")],
+                                                               idents:~[str_to_ident("b")],
                                                                rp: None,
                                                                types: ~[]},
                                                            None // no idea
@@ -532,7 +518,7 @@ mod test {
         // assignment order of the node_ids.
         assert_eq!(string_to_item(@~"fn a (b : int) { b; }"),
                   Some(
-                      @ast::item{ident:intern("a"),
+                      @ast::item{ident:str_to_ident("a"),
                             attrs:~[],
                             id: 9, // fixme
                             node: ast::item_fn(ast::fn_decl{
@@ -542,7 +528,7 @@ mod test {
                                                 node: ast::ty_path(@ast::Path{
                                         span:sp(10,13),
                                         global:false,
-                                        idents:~[intern("int")],
+                                        idents:~[str_to_ident("int")],
                                         rp: None,
                                         types: ~[]},
                                                        2),
@@ -553,7 +539,7 @@ mod test {
                                                        @ast::Path{
                                                            span:sp(6,7),
                                                            global:false,
-                                                           idents:~[intern("b")],
+                                                           idents:~[str_to_ident("b")],
                                                            rp: None,
                                                            types: ~[]},
                                                        None // no idea
@@ -583,7 +569,7 @@ mod test {
                                                         @ast::Path{
                                                             span:sp(17,18),
                                                             global:false,
-                                                            idents:~[intern("b")],
+                                                            idents:~[str_to_ident("b")],
                                                             rp:None,
                                                             types: ~[]}),
                                                     span: sp(17,18)},
diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs
index 23e3f145398..2fd0a7e33ff 100644
--- a/src/libsyntax/parse/parser.rs
+++ b/src/libsyntax/parse/parser.rs
@@ -85,7 +85,7 @@ use parse::obsolete::{ObsoleteLifetimeNotation, ObsoleteConstManagedPointer};
 use parse::obsolete::{ObsoletePurity, ObsoleteStaticMethod};
 use parse::obsolete::{ObsoleteConstItem, ObsoleteFixedLengthVectorType};
 use parse::obsolete::{ObsoleteNamedExternModule, ObsoleteMultipleLocalDecl};
-use parse::token::{can_begin_expr, is_ident, is_ident_or_path};
+use parse::token::{can_begin_expr, get_ident_interner, ident_to_str, is_ident, is_ident_or_path};
 use parse::token::{is_plain_ident, INTERPOLATED, keywords, special_idents, token_to_binop};
 use parse::token;
 use parse::{new_sub_parser_from_file, next_node_id, ParseSess};
@@ -219,7 +219,7 @@ pub fn Parser(sess: @mut ParseSess,
               rdr: @reader)
            -> Parser {
     let tok0 = copy rdr.next_token();
-    let interner = rdr.interner();
+    let interner = get_ident_interner();
 
     Parser {
         reader: rdr,
@@ -333,7 +333,7 @@ impl Parser {
     pub fn get_id(&self) -> node_id { next_node_id(self.sess) }
 
     pub fn id_to_str(&self, id: ident) -> @~str {
-        self.sess.interner.get(id)
+        get_ident_interner().get(id.name)
     }
 
     // is this one of the keywords that signals a closure type?
@@ -2628,6 +2628,13 @@ impl Parser {
             // to the macro clause of parse_item_or_view_item. This
             // could use some cleanup, it appears to me.
 
+            // whoops! I now have a guess: I'm guessing the "parens-only"
+            // rule here is deliberate, to allow macro users to use parens
+            // for things that should be parsed as stmt_mac, and braces
+            // for things that should expand into items. Tricky, and
+            // somewhat awkward... and probably undocumented. Of course,
+            // I could just be wrong.
+
             check_expected_item(self, item_attrs);
 
             // Potential trouble: if we allow macros with paths instead of
@@ -3363,7 +3370,7 @@ impl Parser {
             }
             if fields.len() == 0 {
                 self.fatal(fmt!("Unit-like struct should be written as `struct %s;`",
-                                *self.interner.get(class_name)));
+                                *get_ident_interner().get(class_name.name)));
             }
             self.bump();
         } else if *self.token == token::LPAREN {
@@ -3575,7 +3582,7 @@ impl Parser {
     }
 
     fn push_mod_path(&self, id: ident, attrs: ~[ast::attribute]) {
-        let default_path = self.sess.interner.get(id);
+        let default_path = token::interner_get(id.name);
         let file_path = match ::attr::first_attr_value_str_by_name(
             attrs, "path") {
 
@@ -3598,7 +3605,7 @@ impl Parser {
         let prefix = prefix.dir_path();
         let mod_path_stack = &*self.mod_path_stack;
         let mod_path = Path(".").push_many(*mod_path_stack);
-        let default_path = *self.sess.interner.get(id) + ".rs";
+        let default_path = *token::interner_get(id.name) + ".rs";
         let file_path = match ::attr::first_attr_value_str_by_name(
             outer_attrs, "path") {
             Some(d) => {
@@ -3973,7 +3980,7 @@ impl Parser {
         match *self.token {
             token::LIT_STR(s) => {
                 self.bump();
-                let the_string = self.id_to_str(s);
+                let the_string = ident_to_str(&s);
                 let mut words = ~[];
                 for str::each_word(*the_string) |s| { words.push(s) }
                 let mut abis = AbiSet::empty();
@@ -4535,7 +4542,7 @@ impl Parser {
         match *self.token {
             token::LIT_STR(s) => {
                 self.bump();
-                self.id_to_str(s)
+                ident_to_str(&s)
             }
             _ =>  self.fatal("expected string literal")
         }
diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs
index b716384c6cc..ecf83483c21 100644
--- a/src/libsyntax/parse/token.rs
+++ b/src/libsyntax/parse/token.rs
@@ -11,6 +11,7 @@
 use core::prelude::*;
 
 use ast;
+use ast::Name;
 use ast_util;
 use parse::token;
 use util::interner::StrInterner;
@@ -21,6 +22,9 @@ use core::char;
 use core::cmp::Equiv;
 use core::local_data;
 use core::str;
+use core::hashmap::HashSet;
+use core::rand;
+use core::rand::RngUtil;
 use core::to_bytes;
 
 #[deriving(Encodable, Decodable, Eq)]
@@ -176,29 +180,29 @@ pub fn to_str(in: @ident_interner, t: &Token) -> ~str {
           u.to_str() + ast_util::uint_ty_to_str(t)
       }
       LIT_INT_UNSUFFIXED(i) => { i.to_str() }
-      LIT_FLOAT(s, t) => {
-        let mut body = copy *in.get(s);
+      LIT_FLOAT(ref s, t) => {
+        let mut body = copy *ident_to_str(s);
         if body.ends_with(".") {
             body += "0";  // `10.f` is not a float literal
         }
         body + ast_util::float_ty_to_str(t)
       }
-      LIT_FLOAT_UNSUFFIXED(s) => {
-        let mut body = copy *in.get(s);
+      LIT_FLOAT_UNSUFFIXED(ref s) => {
+        let mut body = copy *ident_to_str(s);
         if body.ends_with(".") {
             body += "0";  // `10.f` is not a float literal
         }
         body
       }
-      LIT_STR(s) => { ~"\"" + str::escape_default(*in.get(s)) + "\"" }
+      LIT_STR(ref s) => { ~"\"" + str::escape_default(*ident_to_str(s)) + "\"" }
 
       /* Name components */
-      IDENT(s, _) => copy *in.get(s),
-      LIFETIME(s) => fmt!("'%s", *in.get(s)),
+      IDENT(s, _) => copy *in.get(s.name),
+      LIFETIME(s) => fmt!("'%s", *in.get(s.name)),
       UNDERSCORE => ~"_",
 
       /* Other */
-      DOC_COMMENT(s) => copy *in.get(s),
+      DOC_COMMENT(ref s) => copy *ident_to_str(s),
       EOF => ~"<eof>",
       INTERPOLATED(ref nt) => {
         match nt {
@@ -304,47 +308,47 @@ pub fn is_bar(t: &Token) -> bool {
 pub mod special_idents {
     use ast::ident;
 
-    pub static underscore : ident = ident { repr: 0, ctxt: 0};
-    pub static anon : ident = ident { repr: 1, ctxt: 0};
-    pub static invalid : ident = ident { repr: 2, ctxt: 0}; // ''
-    pub static unary : ident = ident { repr: 3, ctxt: 0};
-    pub static not_fn : ident = ident { repr: 4, ctxt: 0};
-    pub static idx_fn : ident = ident { repr: 5, ctxt: 0};
-    pub static unary_minus_fn : ident = ident { repr: 6, ctxt: 0};
-    pub static clownshoes_extensions : ident = ident { repr: 7, ctxt: 0};
+    pub static underscore : ident = ident { name: 0, ctxt: 0};
+    pub static anon : ident = ident { name: 1, ctxt: 0};
+    pub static invalid : ident = ident { name: 2, ctxt: 0}; // ''
+    pub static unary : ident = ident { name: 3, ctxt: 0};
+    pub static not_fn : ident = ident { name: 4, ctxt: 0};
+    pub static idx_fn : ident = ident { name: 5, ctxt: 0};
+    pub static unary_minus_fn : ident = ident { name: 6, ctxt: 0};
+    pub static clownshoes_extensions : ident = ident { name: 7, ctxt: 0};
 
-    pub static self_ : ident = ident { repr: 8, ctxt: 0}; // 'self'
+    pub static self_ : ident = ident { name: 8, ctxt: 0}; // 'self'
 
     /* for matcher NTs */
-    pub static item : ident = ident { repr: 9, ctxt: 0};
-    pub static block : ident = ident { repr: 10, ctxt: 0};
-    pub static stmt : ident = ident { repr: 11, ctxt: 0};
-    pub static pat : ident = ident { repr: 12, ctxt: 0};
-    pub static expr : ident = ident { repr: 13, ctxt: 0};
-    pub static ty : ident = ident { repr: 14, ctxt: 0};
-    pub static ident : ident = ident { repr: 15, ctxt: 0};
-    pub static path : ident = ident { repr: 16, ctxt: 0};
-    pub static tt : ident = ident { repr: 17, ctxt: 0};
-    pub static matchers : ident = ident { repr: 18, ctxt: 0};
-
-    pub static str : ident = ident { repr: 19, ctxt: 0}; // for the type
+    pub static item : ident = ident { name: 9, ctxt: 0};
+    pub static block : ident = ident { name: 10, ctxt: 0};
+    pub static stmt : ident = ident { name: 11, ctxt: 0};
+    pub static pat : ident = ident { name: 12, ctxt: 0};
+    pub static expr : ident = ident { name: 13, ctxt: 0};
+    pub static ty : ident = ident { name: 14, ctxt: 0};
+    pub static ident : ident = ident { name: 15, ctxt: 0};
+    pub static path : ident = ident { name: 16, ctxt: 0};
+    pub static tt : ident = ident { name: 17, ctxt: 0};
+    pub static matchers : ident = ident { name: 18, ctxt: 0};
+
+    pub static str : ident = ident { name: 19, ctxt: 0}; // for the type
 
     /* outside of libsyntax */
-    pub static ty_visitor : ident = ident { repr: 20, ctxt: 0};
-    pub static arg : ident = ident { repr: 21, ctxt: 0};
-    pub static descrim : ident = ident { repr: 22, ctxt: 0};
-    pub static clownshoe_abi : ident = ident { repr: 23, ctxt: 0};
-    pub static clownshoe_stack_shim : ident = ident { repr: 24, ctxt: 0};
-    pub static tydesc : ident = ident { repr: 25, ctxt: 0};
-    pub static main : ident = ident { repr: 26, ctxt: 0};
-    pub static opaque : ident = ident { repr: 27, ctxt: 0};
-    pub static blk : ident = ident { repr: 28, ctxt: 0};
-    pub static statik : ident = ident { repr: 29, ctxt: 0};
-    pub static intrinsic : ident = ident { repr: 30, ctxt: 0};
-    pub static clownshoes_foreign_mod: ident = ident { repr: 31, ctxt: 0};
-    pub static unnamed_field: ident = ident { repr: 32, ctxt: 0};
-    pub static c_abi: ident = ident { repr: 33, ctxt: 0};
-    pub static type_self: ident = ident { repr: 34, ctxt: 0};    // `Self`
+    pub static ty_visitor : ident = ident { name: 20, ctxt: 0};
+    pub static arg : ident = ident { name: 21, ctxt: 0};
+    pub static descrim : ident = ident { name: 22, ctxt: 0};
+    pub static clownshoe_abi : ident = ident { name: 23, ctxt: 0};
+    pub static clownshoe_stack_shim : ident = ident { name: 24, ctxt: 0};
+    pub static tydesc : ident = ident { name: 25, ctxt: 0};
+    pub static main : ident = ident { name: 26, ctxt: 0};
+    pub static opaque : ident = ident { name: 27, ctxt: 0};
+    pub static blk : ident = ident { name: 28, ctxt: 0};
+    pub static statik : ident = ident { name: 29, ctxt: 0};
+    pub static intrinsic : ident = ident { name: 30, ctxt: 0};
+    pub static clownshoes_foreign_mod: ident = ident { name: 31, ctxt: 0};
+    pub static unnamed_field: ident = ident { name: 32, ctxt: 0};
+    pub static c_abi: ident = ident { name: 33, ctxt: 0};
+    pub static type_self: ident = ident { name: 34, ctxt: 0};    // `Self`
 }
 
 pub struct StringRef<'self>(&'self str);
@@ -394,25 +398,22 @@ pub struct ident_interner {
 }
 
 impl ident_interner {
-    pub fn intern(&self, val: &str) -> ast::ident {
-        ast::ident { repr: self.interner.intern(val), ctxt: 0 }
+    pub fn intern(&self, val: &str) -> Name {
+        self.interner.intern(val)
     }
-    pub fn gensym(&self, val: &str) -> ast::ident {
-        ast::ident { repr: self.interner.gensym(val), ctxt: 0 }
+    pub fn gensym(&self, val: &str) -> Name {
+        self.interner.gensym(val)
     }
-    pub fn get(&self, idx: ast::ident) -> @~str {
-        self.interner.get(idx.repr)
+    pub fn get(&self, idx: Name) -> @~str {
+        self.interner.get(idx)
     }
+    // is this really something that should be exposed?
     pub fn len(&self) -> uint {
         self.interner.len()
     }
-    pub fn find_equiv<Q:Hash +
-                        IterBytes +
-                        Equiv<@~str>>(&self, val: &Q) -> Option<ast::ident> {
-        match self.interner.find_equiv(val) {
-            Some(v) => Some(ast::ident { repr: v, ctxt: 0 }),
-            None => None,
-        }
+    pub fn find_equiv<Q:Hash + IterBytes + Equiv<@~str>>(&self, val: &Q)
+                                                     -> Option<Name> {
+        self.interner.find_equiv(val)
     }
 }
 
@@ -530,11 +531,51 @@ pub fn mk_fake_ident_interner() -> @ident_interner {
 }
 
 // maps a string to its interned representation
-pub fn intern(str : &str) -> ast::ident {
+pub fn intern(str : &str) -> Name {
     let interner = get_ident_interner();
     interner.intern(str)
 }
 
+// gensyms a new uint, using the current interner
+pub fn gensym(str : &str) -> Name {
+    let interner = get_ident_interner();
+    interner.gensym(str)
+}
+
+// map an interned representation back to a string
+pub fn interner_get(name : Name) -> @~str {
+    get_ident_interner().get(name)
+}
+
+// maps an identifier to the string that it corresponds to
+pub fn ident_to_str(id : &ast::ident) -> @~str {
+    interner_get(id.name)
+}
+
+// maps a string to an identifier with an empty syntax context
+pub fn str_to_ident(str : &str) -> ast::ident {
+    ast::new_ident(intern(str))
+}
+
+// maps a string to a gensym'ed identifier
+pub fn gensym_ident(str : &str) -> ast::ident {
+    ast::new_ident(gensym(str))
+}
+
+
+// create a fresh name. In principle, this is just a
+// gensym, but for debugging purposes, you'd like the
+// resulting name to have a suggestive stringify, without
+// paying the cost of guaranteeing that the name is
+// truly unique.  I'm going to try to strike a balance
+// by using a gensym with a name that has a random number
+// at the end. So, the gensym guarantees the uniqueness,
+// and the int helps to avoid confusion.
+pub fn fresh_name(src_name : &str) -> Name {
+    let num = rand::rng().gen_uint_range(0,0xffff);
+   gensym(fmt!("%s_%u",src_name,num))
+}
+
 /**
  * All the valid words that have meaning in the Rust language.
  *
@@ -590,42 +631,42 @@ pub mod keywords {
     impl Keyword {
         pub fn to_ident(&self) -> ident {
             match *self {
-                As => ident { repr: 35, ctxt: 0 },
-                   Break => ident { repr: 36, ctxt: 0 },
-                   Const => ident { repr: 37, ctxt: 0 },
-                   Copy => ident { repr: 38, ctxt: 0 },
-                   Do => ident { repr: 39, ctxt: 0 },
-                   Else => ident { repr: 41, ctxt: 0 },
-                   Enum => ident { repr: 42, ctxt: 0 },
-                   Extern => ident { repr: 43, ctxt: 0 },
-                   False => ident { repr: 44, ctxt: 0 },
-                   Fn => ident { repr: 45, ctxt: 0 },
-                   For => ident { repr: 46, ctxt: 0 },
-                   If => ident { repr: 47, ctxt: 0 },
-                   Impl => ident { repr: 48, ctxt: 0 },
-                   Let => ident { repr: 49, ctxt: 0 },
-                   __Log => ident { repr: 50, ctxt: 0 },
-                   Loop => ident { repr: 51, ctxt: 0 },
-                   Match => ident { repr: 52, ctxt: 0 },
-                   Mod => ident { repr: 53, ctxt: 0 },
-                   Mut => ident { repr: 54, ctxt: 0 },
-                   Once => ident { repr: 55, ctxt: 0 },
-                   Priv => ident { repr: 56, ctxt: 0 },
-                   Pub => ident { repr: 57, ctxt: 0 },
-                   Pure => ident { repr: 58, ctxt: 0 },
-                   Ref => ident { repr: 59, ctxt: 0 },
-                   Return => ident { repr: 60, ctxt: 0 },
-                   Static => ident { repr: 29, ctxt: 0 },
-                   Self => ident { repr: 8, ctxt: 0 },
-                   Struct => ident { repr: 61, ctxt: 0 },
-                   Super => ident { repr: 62, ctxt: 0 },
-                   True => ident { repr: 63, ctxt: 0 },
-                   Trait => ident { repr: 64, ctxt: 0 },
-                   Type => ident { repr: 65, ctxt: 0 },
-                   Unsafe => ident { repr: 66, ctxt: 0 },
-                   Use => ident { repr: 67, ctxt: 0 },
-                   While => ident { repr: 68, ctxt: 0 },
-                   Be => ident { repr: 69, ctxt: 0 },
+                As => ident { name: 35, ctxt: 0 },
+                   Break => ident { name: 36, ctxt: 0 },
+                   Const => ident { name: 37, ctxt: 0 },
+                   Copy => ident { name: 38, ctxt: 0 },
+                   Do => ident { name: 39, ctxt: 0 },
+                   Else => ident { name: 41, ctxt: 0 },
+                   Enum => ident { name: 42, ctxt: 0 },
+                   Extern => ident { name: 43, ctxt: 0 },
+                   False => ident { name: 44, ctxt: 0 },
+                   Fn => ident { name: 45, ctxt: 0 },
+                   For => ident { name: 46, ctxt: 0 },
+                   If => ident { name: 47, ctxt: 0 },
+                   Impl => ident { name: 48, ctxt: 0 },
+                   Let => ident { name: 49, ctxt: 0 },
+                   __Log => ident { name: 50, ctxt: 0 },
+                   Loop => ident { name: 51, ctxt: 0 },
+                   Match => ident { name: 52, ctxt: 0 },
+                   Mod => ident { name: 53, ctxt: 0 },
+                   Mut => ident { name: 54, ctxt: 0 },
+                   Once => ident { name: 55, ctxt: 0 },
+                   Priv => ident { name: 56, ctxt: 0 },
+                   Pub => ident { name: 57, ctxt: 0 },
+                   Pure => ident { name: 58, ctxt: 0 },
+                   Ref => ident { name: 59, ctxt: 0 },
+                   Return => ident { name: 60, ctxt: 0 },
+                   Static => ident { name: 29, ctxt: 0 },
+                   Self => ident { name: 8, ctxt: 0 },
+                   Struct => ident { name: 61, ctxt: 0 },
+                   Super => ident { name: 62, ctxt: 0 },
+                   True => ident { name: 63, ctxt: 0 },
+                   Trait => ident { name: 64, ctxt: 0 },
+                   Type => ident { name: 65, ctxt: 0 },
+                   Unsafe => ident { name: 66, ctxt: 0 },
+                   Use => ident { name: 67, ctxt: 0 },
+                   While => ident { name: 68, ctxt: 0 },
+                   Be => ident { name: 69, ctxt: 0 },
             }
         }
     }
@@ -633,14 +674,14 @@ pub mod keywords {
 
 pub fn is_keyword(kw: keywords::Keyword, tok: &Token) -> bool {
     match *tok {
-        token::IDENT(sid, false) => { kw.to_ident().repr == sid.repr }
+        token::IDENT(sid, false) => { kw.to_ident().name == sid.name }
         _ => { false }
     }
 }
 
 pub fn is_any_keyword(tok: &Token) -> bool {
     match *tok {
-        token::IDENT(sid, false) => match sid.repr {
+        token::IDENT(sid, false) => match sid.name {
             8 | 29 | 35 .. 69 => true,
             _ => false,
         },
@@ -650,7 +691,7 @@ pub fn is_any_keyword(tok: &Token) -> bool {
 
 pub fn is_strict_keyword(tok: &Token) -> bool {
     match *tok {
-        token::IDENT(sid, false) => match sid.repr {
+        token::IDENT(sid, false) => match sid.name {
             8 | 29 | 35 .. 68 => true,
             _ => false,
         },
@@ -660,10 +701,21 @@ pub fn is_strict_keyword(tok: &Token) -> bool {
 
 pub fn is_reserved_keyword(tok: &Token) -> bool {
     match *tok {
-        token::IDENT(sid, false) => match sid.repr {
+        token::IDENT(sid, false) => match sid.name {
             69 => true,
             _ => false,
         },
         _ => false,
     }
 }
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use std::io;
+    #[test] fn t1() {
+        let a = fresh_name("ghi");
+        io::println(fmt!("interned name: %u,\ntextual name: %s\n",
+                         a,*interner_get(a)));
+    }
+}