auto merge of #6559 : jbclements/rust/hygiene-fns-and-cleanup, r=jbclements

This includes new, tested, hygiene support functions. It also removes the interner_key! macro and replaces it with a function, which should be inline-able. It also contains some parser patch-ups and some docfixes.

On my machine, this patch passes all tests.

4 files changed, 102 insertions, 130 deletions

diff --git a/src/libsyntax/parse/lexer.rs b/src/libsyntax/parse/lexer.rs
index 764dec0eeb3..5340293bb02 100644
--- a/src/libsyntax/parse/lexer.rs
+++ b/src/libsyntax/parse/lexer.rs
@@ -786,7 +786,7 @@ mod test {
     fn setup(teststr: ~str) -> Env {
         let cm = CodeMap::new();
         let fm = cm.new_filemap(~"zebra.rs", @teststr);
-        let ident_interner = token::mk_ident_interner(); // interner::mk();
+        let ident_interner = token::get_ident_interner();
         let span_handler =
             diagnostic::mk_span_handler(diagnostic::mk_handler(None),@cm);
         Env {
diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs
index fa52f3dde3d..8eb7ca2923c 100644
--- a/src/libsyntax/parse/mod.rs
+++ b/src/libsyntax/parse/mod.rs
@@ -19,7 +19,7 @@ use diagnostic::{span_handler, mk_span_handler, mk_handler, Emitter};
 use parse::attr::parser_attr;
 use parse::lexer::reader;
 use parse::parser::Parser;
-use parse::token::{ident_interner, mk_ident_interner};
+use parse::token::{ident_interner, get_ident_interner};
 
 use core::io;
 use core::option::{None, Option, Some};
@@ -59,7 +59,7 @@ pub fn new_parse_sess(demitter: Option<Emitter>) -> @mut ParseSess {
         cm: cm,
         next_id: 1,
         span_diagnostic: mk_span_handler(mk_handler(demitter), cm),
-        interner: mk_ident_interner(),
+        interner: get_ident_interner(),
     }
 }
 
@@ -70,7 +70,7 @@ pub fn new_parse_sess_special_handler(sh: @span_handler,
         cm: cm,
         next_id: 1,
         span_diagnostic: sh,
-        interner: mk_ident_interner(),
+        interner: get_ident_interner(),
     }
 }
 
@@ -346,76 +346,24 @@ mod test {
     use std::serialize::Encodable;
     use std;
     use core::io;
-    use core::option::Option;
     use core::option::Some;
     use core::option::None;
-    use core::int;
-    use core::num::NumCast;
-    use codemap::{CodeMap, span, BytePos, spanned};
+    use codemap::{span, BytePos, spanned};
     use opt_vec;
     use ast;
     use abi;
-    use ast_util::mk_ident;
     use parse::parser::Parser;
-    use parse::token::{ident_interner, mk_fresh_ident_interner};
-    use diagnostic::{mk_span_handler, mk_handler};
-
-    // add known names to interner for testing
-    fn mk_testing_interner() -> @ident_interner {
-        let i = mk_fresh_ident_interner();
-        // baby hack; in order to put the identifiers
-        // 'a' and 'b' at known locations, we're going
-        // to fill up the interner to length 100. If
-        // the # of preloaded items on the interner
-        // ever gets larger than 100, we'll have to
-        // adjust this number (say, to 200) and
-        // change the numbers in the identifier
-        // test cases below.
-
-        assert!(i.len() < 100);
-        for int::range(0,100-((i.len()).to_int())) |_dc| {
-            i.gensym("dontcare");
-        }
-        i.intern("a");
-        i.intern("b");
-        i.intern("c");
-        i.intern("d");
-        i.intern("return");
-        assert_eq!(i.get(ast::ident{repr:101,ctxt:0}), @~"b");
-        i
-    }
-
-    // make a parse_sess that's closed over a
-    // testing interner (where a -> 100, b -> 101)
-    fn mk_testing_parse_sess() -> @mut ParseSess {
-        let interner = mk_testing_interner();
-        let cm = @CodeMap::new();
-        @mut ParseSess {
-            cm: cm,
-            next_id: 1,
-            span_diagnostic: mk_span_handler(mk_handler(None), cm),
-            interner: interner,
-        }
-    }
-
-    // map a string to tts, using a made-up filename: return both the token_trees
-    // and the ParseSess
-    fn string_to_tts_t (source_str : @~str) -> (~[ast::token_tree],@mut ParseSess) {
-        let ps = mk_testing_parse_sess();
-        (filemap_to_tts(ps,string_to_filemap(ps,source_str,~"bogofile")),ps)
-    }
+    use parse::token::intern;
+    use util::parser_testing::{string_to_tts_and_sess,string_to_parser};
+    use util::parser_testing::{string_to_expr, string_to_item};
+    use util::parser_testing::{string_to_stmt};
 
     // map a string to tts, return the tt without its parsesess
     fn string_to_tts_only(source_str : @~str) -> ~[ast::token_tree] {
-        let (tts,_ps) = string_to_tts_t(source_str);
+        let (tts,_ps) = string_to_tts_and_sess(source_str);
         tts
     }
 
-    // map string to parser (via tts)
-    fn string_to_parser(source_str: @~str) -> Parser {
-        let ps = mk_testing_parse_sess();
-        new_parser_from_source_str(ps,~[],~"bogofile",source_str)
-    }
 
     #[cfg(test)] fn to_json_str<E : Encodable<std::json::Encoder>>(val: @E) -> ~str {
         do io::with_str_writer |writer| {
@@ -424,30 +372,14 @@ mod test {
         }
     }
 
-    fn string_to_crate (source_str : @~str) -> @ast::crate {
-        string_to_parser(source_str).parse_crate_mod()
-    }
-
-    fn string_to_expr (source_str : @~str) -> @ast::expr {
-        string_to_parser(source_str).parse_expr()
-    }
-
-    fn string_to_item (source_str : @~str) -> Option<@ast::item> {
-        string_to_parser(source_str).parse_item(~[])
-    }
-
-    fn string_to_stmt (source_str : @~str) -> @ast::stmt {
-        string_to_parser(source_str).parse_stmt(~[])
-    }
-
     // produce a codemap::span
     fn sp (a: uint, b: uint) -> span {
         span{lo:BytePos(a),hi:BytePos(b),expn_info:None}
     }
 
     // convert a vector of uints to a vector of ast::idents
-    fn ints_to_idents(ids: ~[uint]) -> ~[ast::ident] {
-        ids.map(|u| mk_ident(*u))
+    fn ints_to_idents(ids: ~[~str]) -> ~[ast::ident] {
+        ids.map(|u| intern(*u))
     }
 
     #[test] fn path_exprs_1 () {
@@ -456,7 +388,7 @@ mod test {
                               callee_id:2,
                               node:ast::expr_path(@ast::Path {span:sp(0,1),
                                                               global:false,
-                                                              idents:~[mk_ident(100)],
+                                                              idents:~[intern("a")],
                                                               rp:None,
                                                               types:~[]}),
                               span:sp(0,1)})
@@ -466,11 +398,12 @@ mod test {
         assert_eq!(string_to_expr(@~"::a::b"),
                    @ast::expr{id:1,
                                callee_id:2,
-                               node:ast::expr_path(@ast::Path {span:sp(0,6),
-                                                               global:true,
-                                                               idents:ints_to_idents(~[100,101]),
-                                                               rp:None,
-                                                               types:~[]}),
+                               node:ast::expr_path(
+                                   @ast::Path {span:sp(0,6),
+                                               global:true,
+                                               idents:ints_to_idents(~[~"a",~"b"]),
+                                               rp:None,
+                                               types:~[]}),
                               span:sp(0,6)})
     }
 
@@ -482,7 +415,7 @@ mod test {
     }*/
 
     #[test] fn string_to_tts_1 () {
-        let (tts,_ps) = string_to_tts_t(@~"fn a (b : int) { b; }");
+        let (tts,_ps) = string_to_tts_and_sess(@~"fn a (b : int) { b; }");
         assert_eq!(to_json_str(@tts),
                    ~"[\
                 [\"tt_tok\",null,[\"IDENT\",\"fn\",false]],\
@@ -519,7 +452,7 @@ mod test {
                                                   node:ast::expr_path(
                                                       @ast::Path{span:sp(7,8),
                                                                  global:false,
-                                                                 idents:~[mk_ident(103)],
+                                                                 idents:~[intern("d")],
                                                                  rp:None,
                                                                  types:~[]
                                                                 }),
@@ -537,7 +470,7 @@ mod test {
                                @ast::Path{
                                    span:sp(0,1),
                                    global:false,
-                                   idents:~[mk_ident(101)],
+                                   idents:~[intern("b")],
                                    rp:None,
                                    types: ~[]}),
                            span: sp(0,1)},
@@ -558,7 +491,7 @@ mod test {
                                                   @ast::Path{
                                                       span:sp(0,1),
                                                       global:false,
-                                                      idents:~[mk_ident(101)],
+                                                      idents:~[intern("b")],
                                                       rp: None,
                                                       types: ~[]},
                                                   None // no idea
@@ -577,7 +510,7 @@ mod test {
                                         span:sp(4,4), // this is bizarre...
                                         // check this in the original parser?
                                         global:false,
-                                        idents:~[mk_ident(105)],
+                                        idents:~[intern("int")],
                                         rp: None,
                                         types: ~[]},
                                                        2),
@@ -587,7 +520,7 @@ mod test {
                                                            @ast::Path{
                                                                span:sp(0,1),
                                                                global:false,
-                                                               idents:~[mk_ident(101)],
+                                                               idents:~[intern("b")],
                                                                rp: None,
                                                                types: ~[]},
                                                            None // no idea
@@ -603,7 +536,7 @@ mod test {
         // assignment order of the node_ids.
         assert_eq!(string_to_item(@~"fn a (b : int) { b; }"),
                   Some(
-                      @ast::item{ident:mk_ident(100),
+                      @ast::item{ident:intern("a"),
                             attrs:~[],
                             id: 10, // fixme
                             node: ast::item_fn(ast::fn_decl{
@@ -613,7 +546,7 @@ mod test {
                                                 node: ast::ty_path(@ast::Path{
                                         span:sp(10,13),
                                         global:false,
-                                        idents:~[mk_ident(106)],
+                                        idents:~[intern("int")],
                                         rp: None,
                                         types: ~[]},
                                                        2),
@@ -624,7 +557,7 @@ mod test {
                                                        @ast::Path{
                                                            span:sp(6,7),
                                                            global:false,
-                                                           idents:~[mk_ident(101)],
+                                                           idents:~[intern("b")],
                                                            rp: None,
                                                            types: ~[]},
                                                        None // no idea
@@ -655,7 +588,7 @@ mod test {
                                                         @ast::Path{
                                                             span:sp(17,18),
                                                             global:false,
-                                                            idents:~[mk_ident(101)],
+                                                            idents:~[intern("b")],
                                                             rp:None,
                                                             types: ~[]}),
                                                     span: sp(17,18)},
@@ -675,4 +608,20 @@ mod test {
         string_to_expr(@~"3 + 4");
         string_to_expr(@~"a::z.froob(b,@(987+3))");
     }
+
+    #[test] fn attrs_fix_bug () {
+        string_to_item(@~"pub fn mk_file_writer(path: &Path, flags: &[FileFlag])
+                   -> Result<@Writer, ~str> {
+    #[cfg(windows)]
+    fn wb() -> c_int {
+      (O_WRONLY | libc::consts::os::extra::O_BINARY) as c_int
+    }
+
+    #[cfg(unix)]
+    fn wb() -> c_int { O_WRONLY as c_int }
+
+    let mut fflags: c_int = wb();
+}");
+    }
+
 }
diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs
index cfef9c49879..fddeea93024 100644
--- a/src/libsyntax/parse/parser.rs
+++ b/src/libsyntax/parse/parser.rs
@@ -2496,6 +2496,7 @@ pub impl Parser {
         @ast::pat { id: self.get_id(), node: pat, span: mk_sp(lo, hi) }
     }
 
+    // parse ident or ident @ pat
     // used by the copy foo and ref foo patterns to give a good
     // error message when parsing mistakes like ref foo(a,b)
     fn parse_pat_ident(&self,
@@ -2587,20 +2588,22 @@ pub impl Parser {
         })
     }
 
-    // parse a statement. may include decl
-    fn parse_stmt(&self, first_item_attrs: ~[attribute]) -> @stmt {
+    // parse a statement. may include decl.
+    // precondition: any attributes are parsed already
+    fn parse_stmt(&self, item_attrs: ~[attribute]) -> @stmt {
         maybe_whole!(self, nt_stmt);
 
         fn check_expected_item(p: &Parser, current_attrs: &[attribute]) {
             // If we have attributes then we should have an item
             if !current_attrs.is_empty() {
-                p.fatal(~"expected item after attrs");
+                p.span_err(*p.last_span,
+                           ~"expected item after attributes");
             }
         }
 
         let lo = self.span.lo;
         if self.is_keyword("let") {
-            check_expected_item(self, first_item_attrs);
+            check_expected_item(self, item_attrs);
             self.expect_keyword("let");
             let decl = self.parse_let();
             return @spanned(lo, decl.span.hi, stmt_decl(decl, self.get_id()));
@@ -2613,7 +2616,7 @@ pub impl Parser {
             // to the macro clause of parse_item_or_view_item. This
             // could use some cleanup, it appears to me.
 
-            check_expected_item(self, first_item_attrs);
+            check_expected_item(self, item_attrs);
 
             // Potential trouble: if we allow macros with paths instead of
             // idents, we'd need to look ahead past the whole path here...
@@ -2649,9 +2652,6 @@ pub impl Parser {
             }
 
         } else {
-            let item_attrs = vec::append(first_item_attrs,
-                                         self.parse_outer_attributes());
-
             match self.parse_item_or_view_item(/*bad*/ copy item_attrs,
                                                            false) {
                 iovi_item(i) => {
@@ -2726,6 +2726,7 @@ pub impl Parser {
         let mut stmts = ~[];
         let mut expr = None;
 
+        // wouldn't it be more uniform to parse view items only, here?
         let ParsedItemsAndViewItems {
             attrs_remaining: attrs_remaining,
             view_items: view_items,
@@ -2740,23 +2741,29 @@ pub impl Parser {
                                 stmt_decl(decl, self.get_id())));
         }
 
-        let mut initial_attrs = attrs_remaining;
-
-        if *self.token == token::RBRACE && !vec::is_empty(initial_attrs) {
-            self.fatal(~"expected item");
-        }
+        let mut attributes_box = attrs_remaining;
 
-        while *self.token != token::RBRACE {
+        while (*self.token != token::RBRACE) {
+            // parsing items even when they're not allowed lets us give
+            // better error messages and recover more gracefully.
+            attributes_box.push_all(self.parse_outer_attributes());
             match *self.token {
                 token::SEMI => {
+                    if !vec::is_empty(attributes_box) {
+                        self.span_err(*self.last_span,~"expected item after attributes");
+                        attributes_box = ~[];
+                    }
                     self.bump(); // empty
                 }
+                token::RBRACE => {
+                    // fall through and out.
+                }
                 _ => {
-                    let stmt = self.parse_stmt(initial_attrs);
-                    initial_attrs = ~[];
+                    let stmt = self.parse_stmt(attributes_box);
+                    attributes_box = ~[];
                     match stmt.node {
                         stmt_expr(e, stmt_id) => {
-                            // Expression without semicolon
+                            // expression without semicolon
                             match *self.token {
                                 token::SEMI => {
                                     self.bump();
@@ -2772,7 +2779,7 @@ pub impl Parser {
                                         self.fatal(
                                             fmt!(
                                                 "expected `;` or `}` after \
-                                                expression but found `%s`",
+                                                 expression but found `%s`",
                                                 self.token_to_str(&t)
                                             )
                                         );
@@ -2781,9 +2788,8 @@ pub impl Parser {
                                 }
                             }
                         }
-
                         stmt_mac(ref m, _) => {
-                            // Statement macro; might be an expr
+                            // statement macro; might be an expr
                             match *self.token {
                                 token::SEMI => {
                                     self.bump();
@@ -2802,8 +2808,7 @@ pub impl Parser {
                                 _ => { stmts.push(stmt); }
                             }
                         }
-
-                        _ => { // All other kinds of statements:
+                        _ => { // all other kinds of statements:
                             stmts.push(stmt);
 
                             if classify::stmt_ends_with_semi(stmt) {
@@ -2814,6 +2819,11 @@ pub impl Parser {
                 }
             }
         }
+
+        if !vec::is_empty(attributes_box) {
+            self.span_err(*self.last_span,~"expected item after attributes");
+        }
+
         let hi = self.span.hi;
         self.bump();
         let bloc = ast::blk_ {
@@ -3518,7 +3528,7 @@ pub impl Parser {
 
         if first && attrs_remaining_len > 0u {
             // We parsed attributes for the first item but didn't find it
-            self.fatal(~"expected item");
+            self.span_err(*self.last_span,~"expected item after attributes");
         }
 
         ast::_mod { view_items: view_items, items: items }
@@ -3723,11 +3733,15 @@ pub impl Parser {
                                first_item_attrs: ~[attribute])
                                -> foreign_mod {
         let ParsedItemsAndViewItems {
-            attrs_remaining: _,
+            attrs_remaining: attrs_remaining,
             view_items: view_items,
             items: _,
             foreign_items: foreign_items
         } = self.parse_foreign_items(first_item_attrs, true);
+        if (! attrs_remaining.is_empty()) {
+            self.span_err(*self.last_span,
+                          ~"expected item after attributes");
+        }
         assert!(*self.token == token::RBRACE);
         ast::foreign_mod {
             sort: sort,
diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs
index 88fa5389089..b4bad5abbf9 100644
--- a/src/libsyntax/parse/token.rs
+++ b/src/libsyntax/parse/token.rs
@@ -411,8 +411,7 @@ pub impl ident_interner {
 }
 
 // return a fresh interner, preloaded with special identifiers.
-// EFFECT: stores this interner in TLS
-pub fn mk_fresh_ident_interner() -> @ident_interner {
+fn mk_fresh_ident_interner() -> @ident_interner {
     // the indices here must correspond to the numbers in
     // special_idents.
     let init_vec = ~[
@@ -453,23 +452,27 @@ pub fn mk_fresh_ident_interner() -> @ident_interner {
         "Self",               // 34
     ];
 
-    let rv = @ident_interner {
+    @ident_interner {
         interner: interner::StrInterner::prefill(init_vec)
-    };
-    unsafe {
-        local_data::local_data_set(interner_key!(), @rv);
     }
-    rv
 }
 
 // if an interner exists in TLS, return it. Otherwise, prepare a
 // fresh one.
-pub fn mk_ident_interner() -> @ident_interner {
+pub fn get_ident_interner() -> @ident_interner {
     unsafe {
-        match local_data::local_data_get(interner_key!()) {
+        let key =
+            (cast::transmute::<(uint, uint),
+             &fn(v: @@::parse::token::ident_interner)>(
+                 (-3 as uint, 0u)));
+        match local_data::local_data_get(key) {
             Some(interner) => *interner,
             None => {
-                mk_fresh_ident_interner()
+                let interner = mk_fresh_ident_interner();
+                unsafe {
+                    local_data::local_data_set(key, @interner);
+                }
+                interner
             }
         }
     }
@@ -481,6 +484,12 @@ pub fn mk_fake_ident_interner() -> @ident_interner {
     @ident_interner { interner: interner::StrInterner::new() }
 }
 
+// maps a string to its interned representation
+pub fn intern(str : &str) -> ast::ident {
+    let interner = get_ident_interner();
+    interner.intern(str)
+}
+
 /**
  * All the valid words that have meaning in the Rust language.
  *