Auto merge of #21452 - bleibig:bison-grammar, r=nikomatsakis

This adds a new lexer/parser combo for the entire Rust language can be generated with with flex and bison, taken from my project at https://github.com/bleibig/rust-grammar. There is also a testing script that runs the generated parser with all *.rs files in the repository (except for tests in compile-fail or ones that marked as "ignore-test" or "ignore-lexer-test"). If you have flex and bison installed, you can run these tests using the new "check-grammar" make target.

This does not depend on or interact with the existing testing code in the grammar, which only provides and tests a lexer specification.

OS X users should take note that the version of bison that comes with the Xcode toolchain (2.3) is too old to work with this grammar, they need to download and install version 3.0 or later.

The parser builds up an S-expression-based AST, which can be displayed by giving the "-v" argument to parser-lalr (normally it only gives output on error). It is only a rough approximation of what is parsed and doesn't capture every detail and nuance of the program.

Hopefully this should be sufficient for issue #2234, or at least a good starting point.

5 files changed, 2613 insertions, 0 deletions

diff --git a/src/grammar/lexer.l b/src/grammar/lexer.l
new file mode 100644
index 00000000000..58202a08c37
--- /dev/null
+++ b/src/grammar/lexer.l
@@ -0,0 +1,342 @@
+%{
+// Copyright 2015 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+#include <stdio.h>
+#include <ctype.h>
+
+static int num_hashes;
+static int end_hashes;
+static int saw_non_hash;
+
+%}
+
+%option stack
+%option yylineno
+
+%x str
+%x rawstr
+%x rawstr_esc_begin
+%x rawstr_esc_body
+%x rawstr_esc_end
+%x byte
+%x bytestr
+%x rawbytestr
+%x rawbytestr_nohash
+%x pound
+%x shebang_or_attr
+%x ltorchar
+%x linecomment
+%x doc_line
+%x blockcomment
+%x doc_block
+%x suffix
+
+ident [a-zA-Z\x80-\xff_][a-zA-Z0-9\x80-\xff_]*
+
+%%
+
+<suffix>{ident}            { BEGIN(INITIAL); }
+<suffix>(.|\n)  { yyless(0); BEGIN(INITIAL); }
+
+[ \n\t\r]             { }
+
+\xef\xbb\xbf {
+  // UTF-8 byte order mark (BOM), ignore if in line 1, error otherwise
+  if (yyget_lineno() != 1) {
+    return -1;
+  }
+}
+
+\/\/(\/|\!)           { BEGIN(doc_line); yymore(); }
+<doc_line>\n          { BEGIN(INITIAL);
+                        yyleng--;
+                        yytext[yyleng] = 0;
+                        return ((yytext[2] == '!') ? INNER_DOC_COMMENT : OUTER_DOC_COMMENT);
+                      }
+<doc_line>[^\n]*      { yymore(); }
+
+\/\/|\/\/\/\/         { BEGIN(linecomment); }
+<linecomment>\n       { BEGIN(INITIAL); }
+<linecomment>[^\n]*   { }
+
+\/\*(\*|\!)[^*]       { yy_push_state(INITIAL); yy_push_state(doc_block); yymore(); }
+<doc_block>\/\*       { yy_push_state(doc_block); yymore(); }
+<doc_block>\*\/       {
+    yy_pop_state();
+    if (yy_top_state() == doc_block) {
+        yymore();
+    } else {
+        return ((yytext[2] == '!') ? INNER_DOC_COMMENT : OUTER_DOC_COMMENT);
+    }
+}
+<doc_block>(.|\n)     { yymore(); }
+
+\/\*                  { yy_push_state(blockcomment); }
+<blockcomment>\/\*    { yy_push_state(blockcomment); }
+<blockcomment>\*\/    { yy_pop_state(); }
+<blockcomment>(.|\n)   { }
+
+_        { return UNDERSCORE; }
+as       { return AS; }
+box      { return BOX; }
+break    { return BREAK; }
+const    { return CONST; }
+continue { return CONTINUE; }
+crate    { return CRATE; }
+else     { return ELSE; }
+enum     { return ENUM; }
+extern   { return EXTERN; }
+false    { return FALSE; }
+fn       { return FN; }
+for      { return FOR; }
+if       { return IF; }
+impl     { return IMPL; }
+in       { return IN; }
+let      { return LET; }
+loop     { return LOOP; }
+match    { return MATCH; }
+mod      { return MOD; }
+move     { return MOVE; }
+mut      { return MUT; }
+priv     { return PRIV; }
+proc     { return PROC; }
+pub      { return PUB; }
+ref      { return REF; }
+return   { return RETURN; }
+self     { return SELF; }
+static   { return STATIC; }
+struct   { return STRUCT; }
+trait    { return TRAIT; }
+true     { return TRUE; }
+type     { return TYPE; }
+typeof   { return TYPEOF; }
+unsafe   { return UNSAFE; }
+use      { return USE; }
+where    { return WHERE; }
+while    { return WHILE; }
+
+{ident}  { return IDENT; }
+
+0x[0-9a-fA-F_]+                                    { BEGIN(suffix); return LIT_INTEGER; }
+0o[0-8_]+                                          { BEGIN(suffix); return LIT_INTEGER; }
+0b[01_]+                                           { BEGIN(suffix); return LIT_INTEGER; }
+[0-9][0-9_]*                                       { BEGIN(suffix); return LIT_INTEGER; }
+[0-9][0-9_]*\.(\.|[a-zA-Z])    { yyless(yyleng - 2); BEGIN(suffix); return LIT_INTEGER; }
+
+[0-9][0-9_]*\.[0-9_]*([eE][-\+]?[0-9_]+)?          { BEGIN(suffix); return LIT_FLOAT; }
+[0-9][0-9_]*(\.[0-9_]*)?[eE][-\+]?[0-9_]+          { BEGIN(suffix); return LIT_FLOAT; }
+
+;      { return ';'; }
+,      { return ','; }
+\.\.\. { return DOTDOTDOT; }
+\.\.   { return DOTDOT; }
+\.     { return '.'; }
+\(     { return '('; }
+\)     { return ')'; }
+\{     { return '{'; }
+\}     { return '}'; }
+\[     { return '['; }
+\]     { return ']'; }
+@      { return '@'; }
+#      { BEGIN(pound); yymore(); }
+<pound>\! { BEGIN(shebang_or_attr); yymore(); }
+<shebang_or_attr>\[ {
+  BEGIN(INITIAL);
+  yyless(2);
+  return SHEBANG;
+}
+<shebang_or_attr>[^\[\n]*\n {
+  // Since the \n was eaten as part of the token, yylineno will have
+  // been incremented to the value 2 if the shebang was on the first
+  // line. This yyless undoes that, setting yylineno back to 1.
+  yyless(yyleng - 1);
+  if (yyget_lineno() == 1) {
+    BEGIN(INITIAL);
+    return SHEBANG_LINE;
+  } else {
+    BEGIN(INITIAL);
+    yyless(2);
+    return SHEBANG;
+  }
+}
+<pound>. { BEGIN(INITIAL); yyless(1); return '#'; }
+
+\~     { return '~'; }
+::     { return MOD_SEP; }
+:      { return ':'; }
+\$     { return '$'; }
+\?     { return '?'; }
+
+==    { return EQEQ; }
+=>    { return FAT_ARROW; }
+=     { return '='; }
+\!=   { return NE; }
+\!    { return '!'; }
+\<=   { return LE; }
+\<\<  { return SHL; }
+\<\<= { return SHLEQ; }
+\<    { return '<'; }
+\>=   { return GE; }
+\>\>  { return SHR; }
+\>\>= { return SHREQ; }
+\>    { return '>'; }
+
+\x27                                  { BEGIN(ltorchar); yymore(); }
+<ltorchar>static                      { BEGIN(INITIAL); return STATIC_LIFETIME; }
+<ltorchar>{ident}                     { BEGIN(INITIAL); return LIFETIME; }
+<ltorchar>\\[nrt\\\x27\x220]\x27      { BEGIN(suffix); return LIT_CHAR; }
+<ltorchar>\\x[0-9a-fA-F]{2}\x27       { BEGIN(suffix); return LIT_CHAR; }
+<ltorchar>\\u\{[0-9a-fA-F]?{6}\}\x27  { BEGIN(suffix); return LIT_CHAR; }
+<ltorchar>.\x27                       { BEGIN(suffix); return LIT_CHAR; }
+<ltorchar>[\x80-\xff]{2,4}\x27        { BEGIN(suffix); return LIT_CHAR; }
+<ltorchar><<EOF>>                     { BEGIN(INITIAL); return -1; }
+
+b\x22              { BEGIN(bytestr); yymore(); }
+<bytestr>\x22      { BEGIN(suffix); return LIT_BINARY; }
+
+<bytestr><<EOF>>                { return -1; }
+<bytestr>\\[n\nrt\\\x27\x220]   { yymore(); }
+<bytestr>\\x[0-9a-fA-F]{2}      { yymore(); }
+<bytestr>\\u\{[0-9a-fA-F]?{6}\} { yymore(); }
+<bytestr>\\[^n\nrt\\\x27\x220]  { return -1; }
+<bytestr>(.|\n)                 { yymore(); }
+
+br\x22                      { BEGIN(rawbytestr_nohash); yymore(); }
+<rawbytestr_nohash>\x22     { BEGIN(suffix); return LIT_BINARY_RAW; }
+<rawbytestr_nohash>(.|\n)   { yymore(); }
+<rawbytestr_nohash><<EOF>>  { return -1; }
+
+br/# {
+    BEGIN(rawbytestr);
+    yymore();
+    num_hashes = 0;
+    saw_non_hash = 0;
+    end_hashes = 0;
+}
+<rawbytestr># {
+    if (!saw_non_hash) {
+        num_hashes++;
+    } else if (end_hashes != 0) {
+        end_hashes++;
+        if (end_hashes == num_hashes) {
+            BEGIN(INITIAL);
+            return LIT_BINARY_RAW;
+        }
+    }
+    yymore();
+}
+<rawbytestr>\x22# {
+    end_hashes = 1;
+    if (end_hashes == num_hashes) {
+        BEGIN(INITIAL);
+        return LIT_BINARY_RAW;
+    }
+    yymore();
+}
+<rawbytestr>(.|\n) {
+    if (!saw_non_hash) {
+        saw_non_hash = 1;
+    }
+    if (end_hashes != 0) {
+        end_hashes = 0;
+    }
+    yymore();
+}
+<rawbytestr><<EOF>> { return -1; }
+
+b\x27                        { BEGIN(byte); yymore(); }
+<byte>\\[nrt\\\x27\x220]\x27 { BEGIN(INITIAL); return LIT_BYTE; }
+<byte>\\x[0-9a-fA-F]{2}\x27  { BEGIN(INITIAL); return LIT_BYTE; }
+<byte>\\u[0-9a-fA-F]{4}\x27  { BEGIN(INITIAL); return LIT_BYTE; }
+<byte>\\U[0-9a-fA-F]{8}\x27  { BEGIN(INITIAL); return LIT_BYTE; }
+<byte>.\x27                  { BEGIN(INITIAL); return LIT_BYTE; }
+<byte><<EOF>>                { BEGIN(INITIAL); return -1; }
+
+r\x22           { BEGIN(rawstr); yymore(); }
+<rawstr>\x22    { BEGIN(suffix); return LIT_STR_RAW; }
+<rawstr>(.|\n)  { yymore(); }
+<rawstr><<EOF>> { return -1; }
+
+r/#             {
+    BEGIN(rawstr_esc_begin);
+    yymore();
+    num_hashes = 0;
+    saw_non_hash = 0;
+    end_hashes = 0;
+}
+
+<rawstr_esc_begin># {
+    num_hashes++;
+    yymore();
+}
+<rawstr_esc_begin>\x22 {
+    BEGIN(rawstr_esc_body);
+    yymore();
+}
+<rawstr_esc_begin>(.|\n) { return -1; }
+
+<rawstr_esc_body>\x22/# {
+  BEGIN(rawstr_esc_end);
+  yymore();
+ }
+<rawstr_esc_body>(.|\n) {
+  yymore();
+ }
+
+<rawstr_esc_end># {
+  end_hashes++;
+  if (end_hashes == num_hashes) {
+    BEGIN(INITIAL);
+    return LIT_STR_RAW;
+  }
+  yymore();
+ }
+<rawstr_esc_end>[^#] {
+  end_hashes = 0;
+  BEGIN(rawstr_esc_body);
+  yymore();
+ }
+
+<rawstr_esc_begin,rawstr_esc_body,rawstr_esc_end><<EOF>> { return -1; }
+
+\x22                     { BEGIN(str); yymore(); }
+<str>\x22                { BEGIN(suffix); return LIT_STR; }
+
+<str><<EOF>>                { return -1; }
+<str>\\[n\nrt\\\x27\x220]   { yymore(); }
+<str>\\x[0-9a-fA-F]{2}      { yymore(); }
+<str>\\u\{[0-9a-fA-F]?{6}\} { yymore(); }
+<str>\\[^n\nrt\\\x27\x220]  { return -1; }
+<str>(.|\n)                 { yymore(); }
+
+-\>  { return RARROW; }
+-    { return '-'; }
+-=   { return MINUSEQ; }
+&&   { return ANDAND; }
+&    { return '&'; }
+&=   { return ANDEQ; }
+\|\| { return OROR; }
+\|   { return '|'; }
+\|=  { return OREQ; }
+\+   { return '+'; }
+\+=  { return PLUSEQ; }
+\*   { return '*'; }
+\*=  { return STAREQ; }
+\/   { return '/'; }
+\/=  { return SLASHEQ; }
+\^   { return '^'; }
+\^=  { return CARETEQ; }
+%    { return '%'; }
+%=   { return PERCENTEQ; }
+
+<<EOF>> { return 0; }
+
+%%
diff --git a/src/grammar/parser-lalr-main.c b/src/grammar/parser-lalr-main.c
new file mode 100644
index 00000000000..db88a1f2999
--- /dev/null
+++ b/src/grammar/parser-lalr-main.c
@@ -0,0 +1,203 @@
+// Copyright 2015 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+#include <stdio.h>
+#include <stdarg.h>
+#include <stdlib.h>
+#include <string.h>
+
+extern int yylex();
+extern int rsparse();
+
+#define PUSHBACK_LEN 4
+
+static char pushback[PUSHBACK_LEN];
+static int verbose;
+
+void print(const char* format, ...) {
+  va_list args;
+  va_start(args, format);
+  if (verbose) {
+    vprintf(format, args);
+  }
+  va_end(args);
+}
+
+// If there is a non-null char at the head of the pushback queue,
+// dequeue it and shift the rest of the queue forwards. Otherwise,
+// return the token from calling yylex.
+int rslex() {
+  if (pushback[0] == '\0') {
+    return yylex();
+  } else {
+    char c = pushback[0];
+    memmove(pushback, pushback + 1, PUSHBACK_LEN - 1);
+    pushback[PUSHBACK_LEN - 1] = '\0';
+    return c;
+  }
+}
+
+// Note: this does nothing if the pushback queue is full. As long as
+// there aren't more than PUSHBACK_LEN consecutive calls to push_back
+// in an action, this shouldn't be a problem.
+void push_back(char c) {
+  for (int i = 0; i < PUSHBACK_LEN; ++i) {
+    if (pushback[i] == '\0') {
+      pushback[i] = c;
+      break;
+    }
+  }
+}
+
+extern int rsdebug;
+
+struct node {
+  struct node *next;
+  struct node *prev;
+  int own_string;
+  char const *name;
+  int n_elems;
+  struct node *elems[];
+};
+
+struct node *nodes = NULL;
+int n_nodes;
+
+struct node *mk_node(char const *name, int n, ...) {
+  va_list ap;
+  int i = 0;
+  unsigned sz = sizeof(struct node) + (n * sizeof(struct node *));
+  struct node *nn, *nd = (struct node *)malloc(sz);
+
+  print("# New %d-ary node: %s = %p\n", n, name, nd);
+
+  nd->own_string = 0;
+  nd->prev = NULL;
+  nd->next = nodes;
+  if (nodes) {
+    nodes->prev = nd;
+  }
+  nodes = nd;
+
+  nd->name = name;
+  nd->n_elems = n;
+
+  va_start(ap, n);
+  while (i < n) {
+    nn = va_arg(ap, struct node *);
+    print("#   arg[%d]: %p\n", i, nn);
+    print("#            (%s ...)\n", nn->name);
+    nd->elems[i++] = nn;
+  }
+  va_end(ap);
+  n_nodes++;
+  return nd;
+}
+
+struct node *mk_atom(char *name) {
+  struct node *nd = mk_node((char const *)strdup(name), 0);
+  nd->own_string = 1;
+  return nd;
+}
+
+struct node *mk_none() {
+  return mk_atom("<none>");
+}
+
+struct node *ext_node(struct node *nd, int n, ...) {
+  va_list ap;
+  int i = 0, c = nd->n_elems + n;
+  unsigned sz = sizeof(struct node) + (c * sizeof(struct node *));
+  struct node *nn;
+
+  print("# Extending %d-ary node by %d nodes: %s = %p",
+        nd->n_elems, c, nd->name, nd);
+
+  if (nd->next) {
+    nd->next->prev = nd->prev;
+  }
+  if (nd->prev) {
+    nd->prev->next = nd->next;
+  }
+  nd = realloc(nd, sz);
+  nd->prev = NULL;
+  nd->next = nodes;
+  nodes->prev = nd;
+  nodes = nd;
+
+  print(" ==> %p\n", nd);
+
+  va_start(ap, n);
+  while (i < n) {
+    nn = va_arg(ap, struct node *);
+    print("#   arg[%d]: %p\n", i, nn);
+    print("#            (%s ...)\n", nn->name);
+    nd->elems[nd->n_elems++] = nn;
+    ++i;
+  }
+  va_end(ap);
+  return nd;
+}
+
+int const indent_step = 4;
+
+void print_indent(int depth) {
+  while (depth) {
+    if (depth-- % indent_step == 0) {
+      print("|");
+    } else {
+      print(" ");
+    }
+  }
+}
+
+void print_node(struct node *n, int depth) {
+  int i = 0;
+  print_indent(depth);
+  if (n->n_elems == 0) {
+    print("%s\n", n->name);
+  } else {
+    print("(%s\n", n->name);
+    for (i = 0; i < n->n_elems; ++i) {
+      print_node(n->elems[i], depth + indent_step);
+    }
+    print_indent(depth);
+    print(")\n");
+  }
+}
+
+int main(int argc, char **argv) {
+  if (argc == 2 && strcmp(argv[1], "-v") == 0) {
+    verbose = 1;
+  } else {
+    verbose = 0;
+  }
+  int ret = 0;
+  struct node *tmp;
+  memset(pushback, '\0', PUSHBACK_LEN);
+  ret = rsparse();
+  print("--- PARSE COMPLETE: ret:%d, n_nodes:%d ---\n", ret, n_nodes);
+  if (nodes) {
+    print_node(nodes, 0);
+  }
+  while (nodes) {
+    tmp = nodes;
+    nodes = tmp->next;
+    if (tmp->own_string) {
+      free((void*)tmp->name);
+    }
+    free(tmp);
+  }
+  return ret;
+}
+
+void rserror(char const *s) {
+  fprintf(stderr, "%s\n", s);
+}
diff --git a/src/grammar/parser-lalr.y b/src/grammar/parser-lalr.y
new file mode 100644
index 00000000000..24185ed65d5
--- /dev/null
+++ b/src/grammar/parser-lalr.y
@@ -0,0 +1,1912 @@
+// Copyright 2015 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+%{
+#define YYERROR_VERBOSE
+#define YYSTYPE struct node *
+struct node;
+extern int yylex();
+extern void yyerror(char const *s);
+extern struct node *mk_node(char const *name, int n, ...);
+extern struct node *mk_atom(char *text);
+extern struct node *mk_none();
+extern struct node *ext_node(struct node *nd, int n, ...);
+extern void push_back(char c);
+extern char *yytext;
+%}
+%debug
+
+%token SHL
+%token SHR
+%token LE
+%token EQEQ
+%token NE
+%token GE
+%token ANDAND
+%token OROR
+%token SHLEQ
+%token SHREQ
+%token MINUSEQ
+%token ANDEQ
+%token OREQ
+%token PLUSEQ
+%token STAREQ
+%token SLASHEQ
+%token CARETEQ
+%token PERCENTEQ
+%token DOTDOT
+%token DOTDOTDOT
+%token MOD_SEP
+%token RARROW
+%token FAT_ARROW
+%token LIT_BYTE
+%token LIT_CHAR
+%token LIT_INTEGER
+%token LIT_FLOAT
+%token LIT_STR
+%token LIT_STR_RAW
+%token LIT_BINARY
+%token LIT_BINARY_RAW
+%token IDENT
+%token UNDERSCORE
+%token LIFETIME
+
+// keywords
+%token SELF
+%token STATIC
+%token AS
+%token BREAK
+%token CRATE
+%token ELSE
+%token ENUM
+%token EXTERN
+%token FALSE
+%token FN
+%token FOR
+%token IF
+%token IMPL
+%token IN
+%token LET
+%token LOOP
+%token MATCH
+%token MOD
+%token MOVE
+%token MUT
+%token PRIV
+%token PUB
+%token REF
+%token RETURN
+%token STRUCT
+%token TRUE
+%token TRAIT
+%token TYPE
+%token UNSAFE
+%token USE
+%token WHILE
+%token CONTINUE
+%token PROC
+%token BOX
+%token CONST
+%token WHERE
+%token TYPEOF
+%token INNER_DOC_COMMENT
+%token OUTER_DOC_COMMENT
+
+%token SHEBANG
+%token SHEBANG_LINE
+%token STATIC_LIFETIME
+
+ /*
+   Quoting from the Bison manual:
+
+   "Finally, the resolution of conflicts works by comparing the precedence
+   of the rule being considered with that of the lookahead token. If the
+   token's precedence is higher, the choice is to shift. If the rule's
+   precedence is higher, the choice is to reduce. If they have equal
+   precedence, the choice is made based on the associativity of that
+   precedence level. The verbose output file made by ‘-v’ (see Invoking
+   Bison) says how each conflict was resolved"
+ */
+
+// We expect no shift/reduce or reduce/reduce conflicts in this grammar;
+// all potential ambiguities are scrutinized and eliminated manually.
+%expect 0
+
+// fake-precedence symbol to cause '|' bars in lambda context to parse
+// at low precedence, permit things like |x| foo = bar, where '=' is
+// otherwise lower-precedence than '|'. Also used for proc() to cause
+// things like proc() a + b to parse as proc() { a + b }.
+%precedence LAMBDA
+
+%precedence SELF
+
+// MUT should be lower precedence than IDENT so that in the pat rule,
+// "& MUT pat" has higher precedence than "binding_mode ident [@ pat]"
+%precedence MUT
+
+// IDENT needs to be lower than '{' so that 'foo {' is shifted when
+// trying to decide if we've got a struct-construction expr (esp. in
+// contexts like 'if foo { .')
+//
+// IDENT also needs to be lower precedence than '<' so that '<' in
+// 'foo:bar . <' is shifted (in a trait reference occurring in a
+// bounds list), parsing as foo:(bar<baz>) rather than (foo:bar)<baz>.
+%precedence IDENT
+
+// A couple fake-precedence symbols to use in rules associated with +
+// and < in trailing type contexts. These come up when you have a type
+// in the RHS of operator-AS, such as "foo as bar<baz>". The "<" there
+// has to be shifted so the parser keeps trying to parse a type, even
+// though it might well consider reducing the type "bar" and then
+// going on to "<" as a subsequent binop. The "+" case is with
+// trailing type-bounds ("foo as bar:A+B"), for the same reason.
+%precedence SHIFTPLUS
+
+%precedence MOD_SEP
+%precedence RARROW ':'
+
+// Binops & unops, and their precedences
+%precedence BOX
+%precedence BOXPLACE
+%nonassoc DOTDOT
+
+// RETURN needs to be lower-precedence than tokens that start
+// prefix_exprs
+%precedence RETURN
+
+%left '=' SHLEQ SHREQ MINUSEQ ANDEQ OREQ PLUSEQ STAREQ SLASHEQ CARETEQ PERCENTEQ
+%left OROR
+%left ANDAND
+%left EQEQ NE
+%left '<' '>' LE GE
+%left '|'
+%left '^'
+%left '&'
+%left SHL SHR
+%left '+' '-'
+%precedence AS
+%left '*' '/' '%'
+%precedence '!'
+
+%precedence '{' '[' '(' '.'
+
+%start crate
+
+%%
+
+////////////////////////////////////////////////////////////////////////
+// Part 1: Items and attributes
+////////////////////////////////////////////////////////////////////////
+
+crate
+: maybe_shebang inner_attrs maybe_mod_items  { mk_node("crate", 2, $2, $3); }
+| maybe_shebang maybe_mod_items  { mk_node("crate", 1, $2); }
+;
+
+maybe_shebang
+: SHEBANG_LINE
+| %empty
+;
+
+maybe_inner_attrs
+: inner_attrs
+| %empty                   { $$ = mk_none(); }
+;
+
+inner_attrs
+: inner_attr               { $$ = mk_node("InnerAttrs", 1, $1); }
+| inner_attrs inner_attr   { $$ = ext_node($1, 1, $2); }
+;
+
+inner_attr
+: SHEBANG '[' meta_item ']'   { $$ = mk_node("InnerAttr", 1, $3); }
+| INNER_DOC_COMMENT           { $$ = mk_node("InnerAttr", 1, mk_node("doc-comment", 1, mk_atom(yytext))); }
+;
+
+maybe_outer_attrs
+: outer_attrs
+| %empty                   { $$ = mk_none(); }
+;
+
+outer_attrs
+: outer_attr               { $$ = mk_node("OuterAttrs", 1, $1); }
+| outer_attrs outer_attr   { $$ = ext_node($1, 1, $2); }
+;
+
+outer_attr
+: '#' '[' meta_item ']'    { $$ = $3; }
+| OUTER_DOC_COMMENT        { $$ = mk_node("doc-comment", 1, mk_atom(yytext)); }
+;
+
+meta_item
+: ident                      { $$ = mk_node("MetaWord", 1, $1); }
+| ident '=' lit              { $$ = mk_node("MetaNameValue", 2, $1, $3); }
+| ident '(' meta_seq ')'     { $$ = mk_node("MetaList", 2, $1, $3); }
+| ident '(' meta_seq ',' ')' { $$ = mk_node("MetaList", 2, $1, $3); }
+;
+
+meta_seq
+: %empty                   { $$ = mk_none(); }
+| meta_item                { $$ = mk_node("MetaItems", 1, $1); }
+| meta_seq ',' meta_item   { $$ = ext_node($1, 1, $3); }
+;
+
+maybe_mod_items
+: mod_items
+| %empty             { $$ = mk_none(); }
+;
+
+mod_items
+: mod_item                               { $$ = mk_node("Items", 1, $1); }
+| mod_items mod_item                     { $$ = ext_node($1, 1, $2); }
+;
+
+attrs_and_vis
+: maybe_outer_attrs visibility           { $$ = mk_node("AttrsAndVis", 2, $1, $2); }
+;
+
+mod_item
+: attrs_and_vis item    { $$ = mk_node("Item", 2, $1, $2); }
+;
+
+// items that can appear outside of a fn block
+item
+: item_static
+| item_const
+| item_type
+| block_item
+| view_item
+| item_macro
+;
+
+// items that can appear in "stmts"
+stmt_item
+: item_static
+| item_const
+| item_type
+| block_item
+| use_item
+| extern_fn_item
+;
+
+item_static
+: STATIC ident ':' ty '=' expr ';'  { $$ = mk_node("ItemStatic", 3, $2, $4, $6); }
+| STATIC MUT ident ':' ty '=' expr ';'  { $$ = mk_node("ItemStatic", 3, $3, $5, $7); }
+;
+
+item_const
+: CONST ident ':' ty '=' expr ';'  { $$ = mk_node("ItemConst", 3, $2, $4, $6); }
+;
+
+item_macro
+: path_expr '!' maybe_ident parens_delimited_token_trees ';'
+| path_expr '!' maybe_ident braces_delimited_token_trees
+| path_expr '!' maybe_ident brackets_delimited_token_trees ';'
+;
+
+view_item
+: use_item
+| extern_fn_item
+| EXTERN CRATE ident ';'                      { $$ = mk_node("ViewItemExternCrate", 1, $3); }
+| EXTERN CRATE ident '=' str ';'              { $$ = mk_node("ViewItemExternCrate", 2, $3, $5); }
+| EXTERN CRATE str AS ident ';'               { $$ = mk_node("ViewItemExternCrate", 2, $3, $5); }
+;
+
+extern_fn_item
+: EXTERN maybe_abi item_fn                    { $$ = mk_node("ViewItemExternFn", 2, $2, $3); }
+;
+
+use_item
+: USE view_path ';'                           { $$ = mk_node("ViewItemUse", 1, $2); }
+;
+
+view_path
+: path_no_types_allowed                                    { $$ = mk_node("ViewPathSimple", 1, $1); }
+| path_no_types_allowed MOD_SEP '{'                '}'     { $$ = mk_node("ViewPathList", 2, $1, mk_atom("ViewPathListEmpty")); }
+| path_no_types_allowed MOD_SEP '{' idents_or_self '}'     { $$ = mk_node("ViewPathList", 2, $1, $4); }
+| path_no_types_allowed MOD_SEP '{' idents_or_self ',' '}' { $$ = mk_node("ViewPathList", 2, $1, $4); }
+| path_no_types_allowed MOD_SEP '*'                        { $$ = mk_node("ViewPathGlob", 1, $1); }
+|                               '{'                '}'     { $$ = mk_atom("ViewPathListEmpty"); }
+|                               '{' idents_or_self '}'     { $$ = mk_node("ViewPathList", 1, $2); }
+|                               '{' idents_or_self ',' '}' { $$ = mk_node("ViewPathList", 1, $2); }
+| path_no_types_allowed AS ident                           { $$ = mk_node("ViewPathSimple", 2, $1, $3); }
+;
+
+block_item
+: item_fn
+| item_unsafe_fn
+| item_mod
+| item_foreign_mod          { $$ = mk_node("ItemForeignMod", 1, $1); }
+| item_struct
+| item_enum
+| item_trait
+| item_impl
+;
+
+maybe_ty_ascription
+: ':' ty { $$ = $2; }
+| %empty { $$ = mk_none(); }
+;
+
+maybe_init_expr
+: '=' expr { $$ = $2; }
+| %empty   { $$ = mk_none(); }
+;
+
+// structs
+item_struct
+: STRUCT ident generic_params maybe_where_clause struct_decl_args
+{
+  $$ = mk_node("ItemStruct", 4, $2, $3, $4, $5);
+}
+| STRUCT ident generic_params struct_tuple_args maybe_where_clause ';'
+{
+  $$ = mk_node("ItemStruct", 4, $2, $3, $4, $5);
+}
+| STRUCT ident generic_params maybe_where_clause ';'
+{
+  $$ = mk_node("ItemStruct", 3, $2, $3, $4);
+}
+;
+
+struct_decl_args
+: '{' struct_decl_fields '}'                  { $$ = $2; }
+| '{' struct_decl_fields ',' '}'              { $$ = $2; }
+;
+
+struct_tuple_args
+: '(' struct_tuple_fields ')'                 { $$ = $2; }
+| '(' struct_tuple_fields ',' ')'             { $$ = $2; }
+;
+
+struct_decl_fields
+: struct_decl_field                           { $$ = mk_node("StructFields", 1, $1); }
+| struct_decl_fields ',' struct_decl_field    { $$ = ext_node($1, 1, $3); }
+| %empty                                      { $$ = mk_none(); }
+;
+
+struct_decl_field
+: attrs_and_vis ident ':' ty_sum              { $$ = mk_node("StructField", 3, $1, $2, $4); }
+;
+
+struct_tuple_fields
+: struct_tuple_field                          { $$ = mk_node("StructFields", 1, $1); }
+| struct_tuple_fields ',' struct_tuple_field  { $$ = ext_node($1, 1, $3); }
+;
+
+struct_tuple_field
+: attrs_and_vis ty_sum                    { $$ = mk_node("StructField", 2, $1, $2); }
+;
+
+// enums
+item_enum
+: ENUM ident generic_params maybe_where_clause '{' enum_defs '}'     { $$ = mk_node("ItemEnum", 0); }
+| ENUM ident generic_params maybe_where_clause '{' enum_defs ',' '}' { $$ = mk_node("ItemEnum", 0); }
+;
+
+enum_defs
+: enum_def               { $$ = mk_node("EnumDefs", 1, $1); }
+| enum_defs ',' enum_def { $$ = ext_node($1, 1, $3); }
+| %empty                 { $$ = mk_none(); }
+;
+
+enum_def
+: attrs_and_vis ident enum_args { $$ = mk_node("EnumDef", 3, $1, $2, $3); }
+;
+
+enum_args
+: '{' struct_decl_fields '}'     { $$ = mk_node("EnumArgs", 1, $2); }
+| '{' struct_decl_fields ',' '}' { $$ = mk_node("EnumArgs", 1, $2); }
+| '(' maybe_ty_sums ')'          { $$ = mk_node("EnumArgs", 1, $2); }
+| '=' expr                       { $$ = mk_node("EnumArgs", 1, $2); }
+| %empty                         { $$ = mk_none(); }
+;
+
+item_mod
+: MOD ident ';'                                 { $$ = mk_node("ItemMod", 1, $2); }
+| MOD ident '{' maybe_mod_items '}'             { $$ = mk_node("ItemMod", 2, $2, $4); }
+| MOD ident '{' inner_attrs maybe_mod_items '}' { $$ = mk_node("ItemMod", 3, $2, $4, $5); }
+;
+
+item_foreign_mod
+: EXTERN maybe_abi '{' maybe_foreign_items '}'             { $$ = mk_node("ItemForeignMod", 1, $4); }
+| EXTERN maybe_abi '{' inner_attrs maybe_foreign_items '}' { $$ = mk_node("ItemForeignMod", 2, $4, $5); }
+;
+
+maybe_abi
+: str
+| %empty { $$ = mk_none(); }
+;
+
+maybe_foreign_items
+: foreign_items
+| %empty { $$ = mk_none(); }
+;
+
+foreign_items
+: foreign_item               { $$ = mk_node("ForeignItems", 1, $1); }
+| foreign_items foreign_item { $$ = ext_node($1, 1, $2); }
+;
+
+foreign_item
+: attrs_and_vis STATIC item_foreign_static { $$ = mk_node("ForeignItem", 2, $1, $3); }
+| attrs_and_vis item_foreign_fn            { $$ = mk_node("ForeignItem", 2, $1, $2); }
+| attrs_and_vis UNSAFE item_foreign_fn     { $$ = mk_node("ForeignItem", 2, $1, $3); }
+;
+
+item_foreign_static
+: maybe_mut ident ':' ty ';'               { $$ = mk_node("StaticItem", 3, $1, $2, $4); }
+;
+
+item_foreign_fn
+: FN ident generic_params fn_decl_allow_variadic maybe_where_clause ';' { $$ = mk_node("ForeignFn", 4, $2, $3, $4, $5); }
+;
+
+fn_decl_allow_variadic
+: fn_params_allow_variadic ret_ty { $$ = mk_node("FnDecl", 2, $1, $2); }
+;
+
+fn_params_allow_variadic
+: '(' ')'                      { $$ = mk_none(); }
+| '(' params ')'               { $$ = $2; }
+| '(' params ',' ')'           { $$ = $2; }
+| '(' params ',' DOTDOTDOT ')' { $$ = $2; }
+;
+
+visibility
+: PUB      { $$ = mk_atom("Public"); }
+| %empty   { $$ = mk_atom("Inherited"); }
+;
+
+idents_or_self
+: ident_or_self                    { $$ = mk_node("IdentsOrSelf", 1, $1); }
+| idents_or_self ',' ident_or_self { $$ = ext_node($1, 1, $3); }
+;
+
+ident_or_self
+: ident
+| SELF  { $$ = mk_atom(yytext); }
+;
+
+item_type
+: TYPE ident generic_params maybe_where_clause '=' ty_sum ';'  { $$ = mk_node("ItemTy", 4, $2, $3, $4, $6); }
+;
+
+for_sized
+: FOR '?' ident { $$ = mk_node("ForSized", 1, $3); }
+| FOR ident '?' { $$ = mk_node("ForSized", 1, $2); }
+| %empty        { $$ = mk_none(); }
+;
+
+item_trait
+: maybe_unsafe TRAIT ident generic_params for_sized maybe_ty_param_bounds maybe_where_clause '{' maybe_trait_items '}'
+{
+  $$ = mk_node("ItemTrait", 7, $1, $3, $4, $5, $6, $7, $9);
+}
+;
+
+maybe_trait_items
+: trait_items
+| %empty { $$ = mk_none(); }
+;
+
+trait_items
+: trait_item               { $$ = mk_node("TraitItems", 1, $1); }
+| trait_items trait_item   { $$ = ext_node($1, 1, $2); }
+;
+
+trait_item
+: trait_type
+| trait_method
+;
+
+trait_type
+: maybe_outer_attrs TYPE ty_param ';' { $$ = mk_node("TypeTraitItem", 2, $1, $3); }
+;
+
+maybe_unsafe
+: UNSAFE { $$ = mk_atom("Unsafe"); }
+| %empty { $$ = mk_none(); }
+;
+
+trait_method
+: type_method { $$ = mk_node("Required", 1, $1); }
+| method      { $$ = mk_node("Provided", 1, $1); }
+;
+
+type_method
+: attrs_and_vis maybe_unsafe FN ident generic_params fn_decl_with_self_allow_anon_params maybe_where_clause ';'
+{
+  $$ = mk_node("TypeMethod", 6, $1, $2, $4, $5, $6, $7);
+}
+| attrs_and_vis maybe_unsafe EXTERN maybe_abi FN ident generic_params fn_decl_with_self_allow_anon_params maybe_where_clause ';'
+{
+  $$ = mk_node("TypeMethod", 7, $1, $2, $4, $6, $7, $8, $9);
+}
+;
+
+method
+: attrs_and_vis maybe_unsafe FN ident generic_params fn_decl_with_self_allow_anon_params maybe_where_clause inner_attrs_and_block
+{
+  $$ = mk_node("Method", 7, $1, $2, $4, $5, $6, $7, $8);
+}
+| attrs_and_vis maybe_unsafe EXTERN maybe_abi FN ident generic_params fn_decl_with_self_allow_anon_params maybe_where_clause inner_attrs_and_block
+{
+  $$ = mk_node("Method", 8, $1, $2, $4, $6, $7, $8, $9, $10);
+}
+;
+
+impl_method
+: attrs_and_vis maybe_unsafe FN ident generic_params fn_decl_with_self maybe_where_clause inner_attrs_and_block
+{
+  $$ = mk_node("Method", 7, $1, $2, $4, $5, $6, $7, $8);
+}
+| attrs_and_vis maybe_unsafe EXTERN maybe_abi FN ident generic_params fn_decl_with_self maybe_where_clause inner_attrs_and_block
+{
+  $$ = mk_node("Method", 8, $1, $2, $4, $6, $7, $8, $9, $10);
+}
+;
+
+// There are two forms of impl:
+//
+// impl (<...>)? TY { ... }
+// impl (<...>)? TRAIT for TY { ... }
+//
+// Unfortunately since TY can begin with '<' itself -- as part of a
+// TyQualifiedPath type -- there's an s/r conflict when we see '<' after IMPL:
+// should we reduce one of the early rules of TY (such as maybe_once)
+// or shall we continue shifting into the generic_params list for the
+// impl?
+//
+// The production parser disambiguates a different case here by
+// permitting / requiring the user to provide parens around types when
+// they are ambiguous with traits. We do the same here, regrettably,
+// by splitting ty into ty and ty_prim.
+item_impl
+: maybe_unsafe IMPL generic_params ty_prim_sum maybe_where_clause '{' maybe_inner_attrs maybe_impl_items '}'
+{
+  $$ = mk_node("ItemImpl", 6, $1, $3, $4, $5, $7, $8);
+}
+| maybe_unsafe IMPL generic_params '(' ty ')' maybe_where_clause '{' maybe_inner_attrs maybe_impl_items '}'
+{
+  $$ = mk_node("ItemImpl", 6, $1, $3, 5, $6, $9, $10);
+}
+| maybe_unsafe IMPL generic_params trait_ref FOR ty maybe_where_clause '{' maybe_inner_attrs maybe_impl_items '}'
+{
+  $$ = mk_node("ItemImpl", 6, $3, $4, $6, $7, $9, $10);
+}
+| maybe_unsafe IMPL generic_params '!' trait_ref FOR ty maybe_where_clause '{' maybe_inner_attrs maybe_impl_items '}'
+{
+  $$ = mk_node("ItemImplNeg", 7, $1, $3, $5, $7, $8, $10, $11);
+}
+;
+
+maybe_impl_items
+: impl_items
+| %empty { $$ = mk_none(); }
+;
+
+impl_items
+: impl_item               { $$ = mk_node("ImplItems", 1, $1); }
+| impl_item impl_items    { $$ = ext_node($1, 1, $2); }
+;
+
+impl_item
+: impl_method
+| item_macro
+| trait_type
+;
+
+item_fn
+: FN ident generic_params fn_decl maybe_where_clause inner_attrs_and_block
+{
+  $$ = mk_node("ItemFn", 5, $2, $3, $4, $5, $6);
+}
+;
+
+item_unsafe_fn
+: UNSAFE FN ident generic_params fn_decl maybe_where_clause inner_attrs_and_block
+{
+  $$ = mk_node("ItemUnsafeFn", 5, $3, $4, $5, $6, $7);
+}
+| UNSAFE EXTERN maybe_abi FN ident generic_params fn_decl maybe_where_clause inner_attrs_and_block
+{
+  $$ = mk_node("ItemUnsafeFn", 6, $3, $5, $6, $7, $8, $9);
+}
+;
+
+fn_decl
+: fn_params ret_ty   { $$ = mk_node("FnDecl", 2, $1, $2); }
+;
+
+fn_decl_with_self
+: fn_params_with_self ret_ty   { $$ = mk_node("FnDecl", 2, $1, $2); }
+;
+
+fn_decl_with_self_allow_anon_params
+: fn_anon_params_with_self ret_ty   { $$ = mk_node("FnDecl", 2, $1, $2); }
+;
+
+fn_params
+: '(' maybe_params ')'  { $$ = $2; }
+;
+
+fn_anon_params
+: '(' anon_param anon_params_allow_variadic_tail ')' { $$ = ext_node($2, 1, $3); }
+| '(' ')'                                            { $$ = mk_none(); }
+;
+
+fn_params_with_self
+: '(' maybe_mut SELF maybe_ty_ascription maybe_comma_params ')'              { $$ = mk_node("SelfValue", 3, $2, $4, $5); }
+| '(' '&' maybe_mut SELF maybe_ty_ascription maybe_comma_params ')'          { $$ = mk_node("SelfRegion", 3, $3, $5, $6); }
+| '(' '&' lifetime maybe_mut SELF maybe_ty_ascription maybe_comma_params ')' { $$ = mk_node("SelfRegion", 4, $3, $4, $6, $7); }
+| '(' maybe_params ')'                                                       { $$ = mk_node("SelfStatic", 1, $2); }
+;
+
+fn_anon_params_with_self
+: '(' maybe_mut SELF maybe_ty_ascription maybe_comma_anon_params ')'              { $$ = mk_node("SelfValue", 3, $2, $4, $5); }
+| '(' '&' maybe_mut SELF maybe_ty_ascription maybe_comma_anon_params ')'          { $$ = mk_node("SelfRegion", 3, $3, $5, $6); }
+| '(' '&' lifetime maybe_mut SELF maybe_ty_ascription maybe_comma_anon_params ')' { $$ = mk_node("SelfRegion", 4, $3, $4, $6, $7); }
+| '(' maybe_anon_params ')'                                                       { $$ = mk_node("SelfStatic", 1, $2); }
+;
+
+maybe_params
+: params
+| params ','
+| %empty  { $$ = mk_none(); }
+;
+
+params
+: param                { $$ = mk_node("Args", 1, $1); }
+| params ',' param     { $$ = ext_node($1, 1, $3); }
+;
+
+param
+: pat ':' ty   { $$ = mk_node("Arg", 2, $1, $3); }
+;
+
+inferrable_params
+: inferrable_param                       { $$ = mk_node("InferrableParams", 1, $1); }
+| inferrable_params ',' inferrable_param { $$ = ext_node($1, 1, $3); }
+;
+
+inferrable_param
+: pat maybe_ty_ascription { $$ = mk_node("InferrableParam", 2, $1, $2); }
+;
+
+maybe_unboxed_closure_kind
+: %empty
+| ':'
+| '&' maybe_mut ':'
+;
+
+maybe_comma_params
+: ','            { $$ = mk_none(); }
+| ',' params     { $$ = $2; }
+| ',' params ',' { $$ = $2; }
+| %empty         { $$ = mk_none(); }
+;
+
+maybe_comma_anon_params
+: ','                 { $$ = mk_none(); }
+| ',' anon_params     { $$ = $2; }
+| ',' anon_params ',' { $$ = $2; }
+| %empty              { $$ = mk_none(); }
+;
+
+maybe_anon_params
+: anon_params
+| anon_params ','
+| %empty      { $$ = mk_none(); }
+;
+
+anon_params
+: anon_param                 { $$ = mk_node("Args", 1, $1); }
+| anon_params ',' anon_param { $$ = ext_node($1, 1, $3); }
+;
+
+// anon means it's allowed to be anonymous (type-only), but it can
+// still have a name
+anon_param
+: named_arg ':' ty   { $$ = mk_node("Arg", 2, $1, $3); }
+| ty
+;
+
+anon_params_allow_variadic_tail
+: ',' DOTDOTDOT                                  { $$ = mk_none(); }
+| ',' anon_param anon_params_allow_variadic_tail { $$ = mk_node("Args", 2, $2, $3); }
+| %empty                                         { $$ = mk_none(); }
+;
+
+named_arg
+: ident
+| UNDERSCORE        { $$ = mk_atom("PatWild"); }
+| '&' ident         { $$ = $2; }
+| '&' UNDERSCORE    { $$ = mk_atom("PatWild"); }
+| ANDAND ident      { $$ = $2; }
+| ANDAND UNDERSCORE { $$ = mk_atom("PatWild"); }
+| MUT ident         { $$ = $2; }
+;
+
+ret_ty
+: RARROW '!'         { $$ = mk_none(); }
+| RARROW ty          { $$ = mk_node("ret-ty", 1, $2); }
+| %prec IDENT %empty { $$ = mk_none(); }
+;
+
+generic_params
+: '<' lifetimes '>'                   { $$ = mk_node("Generics", 2, $2, mk_none()); }
+| '<' lifetimes ',' '>'               { $$ = mk_node("Generics", 2, $2, mk_none()); }
+| '<' lifetimes SHR                   { push_back('>'); $$ = mk_node("Generics", 2, $2, mk_none()); }
+| '<' lifetimes ',' SHR               { push_back('>'); $$ = mk_node("Generics", 2, $2, mk_none()); }
+| '<' lifetimes ',' ty_params '>'     { $$ = mk_node("Generics", 2, $2, $4); }
+| '<' lifetimes ',' ty_params ',' '>' { $$ = mk_node("Generics", 2, $2, $4); }
+| '<' lifetimes ',' ty_params SHR     { push_back('>'); $$ = mk_node("Generics", 2, $2, $4); }
+| '<' lifetimes ',' ty_params ',' SHR { push_back('>'); $$ = mk_node("Generics", 2, $2, $4); }
+| '<' ty_params '>'                   { $$ = mk_node("Generics", 2, mk_none(), $2); }
+| '<' ty_params ',' '>'               { $$ = mk_node("Generics", 2, mk_none(), $2); }
+| '<' ty_params SHR                   { push_back('>'); $$ = mk_node("Generics", 2, mk_none(), $2); }
+| '<' ty_params ',' SHR               { push_back('>'); $$ = mk_node("Generics", 2, mk_none(), $2); }
+| %empty                              { $$ = mk_none(); }
+;
+
+maybe_where_clause
+: %empty                              { $$ = mk_none(); }
+| where_clause
+;
+
+where_clause
+: WHERE where_predicates              { $$ = mk_node("WhereClause", 1, $2); }
+| WHERE where_predicates ','          { $$ = mk_node("WhereClause", 1, $2); }
+;
+
+where_predicates
+: where_predicate                      { $$ = mk_node("WherePredicates", 1, $1); }
+| where_predicates ',' where_predicate { $$ = ext_node($1, 1, $3); }
+;
+
+where_predicate
+: lifetime ':' bounds    { $$ = mk_node("WherePredicate", 2, $1, $3); }
+| ty ':' ty_param_bounds { $$ = mk_node("WherePredicate", 2, $1, $3); }
+;
+
+ty_params
+: ty_param               { $$ = mk_node("TyParams", 1, $1); }
+| ty_params ',' ty_param { $$ = ext_node($1, 1, $3); }
+;
+
+// A path with no type parameters; e.g. `foo::bar::Baz`
+//
+// These show up in 'use' view-items, because these are processed
+// without respect to types.
+path_no_types_allowed
+: ident                               { $$ = mk_node("ViewPath", 1, $1); }
+| MOD_SEP ident                       { $$ = mk_node("ViewPath", 1, $2); }
+| SELF                                { $$ = mk_node("ViewPath", 1, mk_atom("Self")); }
+| MOD_SEP SELF                        { $$ = mk_node("ViewPath", 1, mk_atom("Self")); }
+| path_no_types_allowed MOD_SEP ident { $$ = ext_node($1, 1, $3); }
+;
+
+// A path with a lifetime and type parameters, with no double colons
+// before the type parameters; e.g. `foo::bar<'a>::Baz<T>`
+//
+// These show up in "trait references", the components of
+// type-parameter bounds lists, as well as in the prefix of the
+// path_generic_args_and_bounds rule, which is the full form of a
+// named typed expression.
+//
+// They do not have (nor need) an extra '::' before '<' because
+// unlike in expr context, there are no "less-than" type exprs to
+// be ambiguous with.
+path_generic_args_without_colons
+: %prec IDENT
+  ident                                                                       { $$ = mk_node("components", 1, $1); }
+| %prec IDENT
+  ident generic_args                                                          { $$ = mk_node("components", 2, $1, $2); }
+| %prec IDENT
+  ident '(' maybe_ty_sums ')' ret_ty                                          { $$ = mk_node("components", 2, $1, $3); }
+| %prec IDENT
+  path_generic_args_without_colons MOD_SEP ident                              { $$ = ext_node($1, 1, $3); }
+| %prec IDENT
+  path_generic_args_without_colons MOD_SEP ident generic_args                 { $$ = ext_node($1, 2, $3, $4); }
+| %prec IDENT
+  path_generic_args_without_colons MOD_SEP ident '(' maybe_ty_sums ')' ret_ty { $$ = ext_node($1, 2, $3, $5); }
+;
+
+generic_args
+: '<' generic_values '>'   { $$ = $2; }
+| '<' generic_values SHR   { push_back('>'); $$ = $2; }
+| '<' generic_values GE    { push_back('='); $$ = $2; }
+| '<' generic_values SHREQ { push_back('>'); push_back('='); $$ = $2; }
+// If generic_args starts with "<<", the first arg must be a
+// TyQualifiedPath because that's the only type that can start with a
+// '<'. This rule parses that as the first ty_sum and then continues
+// with the rest of generic_values.
+| SHL ty_qualified_path_and_generic_values '>'   { $$ = $2; }
+| SHL ty_qualified_path_and_generic_values SHR   { push_back('>'); $$ = $2; }
+| SHL ty_qualified_path_and_generic_values GE    { push_back('='); $$ = $2; }
+| SHL ty_qualified_path_and_generic_values SHREQ { push_back('>'); push_back('='); $$ = $2; }
+;
+
+generic_values
+: maybe_lifetimes maybe_ty_sums_and_or_bindings { $$ = mk_node("GenericValues", 2, $1, $2); }
+;
+
+maybe_ty_sums_and_or_bindings
+: ty_sums
+| ty_sums ','
+| ty_sums ',' bindings { $$ = mk_node("TySumsAndBindings", 2, $1, $3); }
+| bindings
+| bindings ','
+| %empty               { $$ = mk_none(); }
+;
+
+maybe_bindings
+: ',' bindings { $$ = $2; }
+| %empty       { $$ = mk_none(); }
+;
+
+////////////////////////////////////////////////////////////////////////
+// Part 2: Patterns
+////////////////////////////////////////////////////////////////////////
+
+pat
+: UNDERSCORE                                      { $$ = mk_atom("PatWild"); }
+| '&' pat                                         { $$ = mk_node("PatRegion", 1, $2); }
+| '&' MUT pat                                     { $$ = mk_node("PatRegion", 1, $3); }
+| ANDAND pat                                      { $$ = mk_node("PatRegion", 1, mk_node("PatRegion", 1, $2)); }
+| '(' ')'                                         { $$ = mk_atom("PatUnit"); }
+| '(' pat_tup ')'                                 { $$ = mk_node("PatTup", 1, $2); }
+| '(' pat_tup ',' ')'                             { $$ = mk_node("PatTup", 1, $2); }
+| '[' pat_vec ']'                                 { $$ = mk_node("PatVec", 1, $2); }
+| lit_or_path
+| lit_or_path DOTDOTDOT lit_or_path               { $$ = mk_node("PatRange", 2, $1, $3); }
+| path_expr '{' pat_struct '}'                    { $$ = mk_node("PatStruct", 2, $1, $3); }
+| path_expr '(' DOTDOT ')'                        { $$ = mk_node("PatEnum", 1, $1); }
+| path_expr '(' pat_tup ')'                       { $$ = mk_node("PatEnum", 2, $1, $3); }
+| path_expr '!' maybe_ident delimited_token_trees { $$ = mk_node("PatMac", 3, $1, $3, $4); }
+| binding_mode ident                              { $$ = mk_node("PatIdent", 2, $1, $2); }
+|              ident '@' pat                      { $$ = mk_node("PatIdent", 3, mk_node("BindByValue", 1, mk_atom("MutImmutable")), $1, $3); }
+| binding_mode ident '@' pat                      { $$ = mk_node("PatIdent", 3, $1, $2, $4); }
+| BOX pat                                         { $$ = mk_node("PatUniq", 1, $2); }
+;
+
+pats_or
+: pat              { $$ = mk_node("Pats", 1, $1); }
+| pats_or '|' pat  { $$ = ext_node($1, 1, $3); }
+;
+
+binding_mode
+: REF         { $$ = mk_node("BindByRef", 1, mk_atom("MutImmutable")); }
+| REF MUT     { $$ = mk_node("BindByRef", 1, mk_atom("MutMutable")); }
+| MUT         { $$ = mk_node("BindByValue", 1, mk_atom("MutMutable")); }
+;
+
+lit_or_path
+: path_expr    { $$ = mk_node("PatLit", 1, $1); }
+| lit          { $$ = mk_node("PatLit", 1, $1); }
+| '-' lit      { $$ = mk_node("PatLit", 1, $2); }
+;
+
+pat_field
+:                  ident        { $$ = mk_node("PatField", 1, $1); }
+|     binding_mode ident        { $$ = mk_node("PatField", 2, $1, $2); }
+| BOX              ident        { $$ = mk_node("PatField", 2, mk_atom("box"), $2); }
+| BOX binding_mode ident        { $$ = mk_node("PatField", 3, mk_atom("box"), $2, $3); }
+|              ident ':' pat    { $$ = mk_node("PatField", 2, $1, $3); }
+| binding_mode ident ':' pat    { $$ = mk_node("PatField", 3, $1, $2, $4); }
+;
+
+pat_fields
+: pat_field                  { $$ = mk_node("PatFields", 1, $1); }
+| pat_fields ',' pat_field   { $$ = ext_node($1, 1, $3); }
+;
+
+pat_struct
+: pat_fields                 { $$ = mk_node("PatStruct", 2, $1, mk_atom("false")); }
+| pat_fields ','             { $$ = mk_node("PatStruct", 2, $1, mk_atom("false")); }
+| pat_fields ',' DOTDOT      { $$ = mk_node("PatStruct", 2, $1, mk_atom("true")); }
+| DOTDOT                     { $$ = mk_node("PatStruct", 1, mk_atom("true")); }
+;
+
+pat_tup
+: pat               { $$ = mk_node("pat_tup", 1, $1); }
+| pat_tup ',' pat   { $$ = ext_node($1, 1, $3); }
+;
+
+pat_vec
+: pat_vec_elts                                  { $$ = mk_node("PatVec", 2, $1, mk_none()); }
+| pat_vec_elts                             ','  { $$ = mk_node("PatVec", 2, $1, mk_none()); }
+| pat_vec_elts     DOTDOT                       { $$ = mk_node("PatVec", 2, $1, mk_none()); }
+| pat_vec_elts ',' DOTDOT                       { $$ = mk_node("PatVec", 2, $1, mk_none()); }
+| pat_vec_elts     DOTDOT ',' pat_vec_elts      { $$ = mk_node("PatVec", 2, $1, $4); }
+| pat_vec_elts     DOTDOT ',' pat_vec_elts ','  { $$ = mk_node("PatVec", 2, $1, $4); }
+| pat_vec_elts ',' DOTDOT ',' pat_vec_elts      { $$ = mk_node("PatVec", 2, $1, $5); }
+| pat_vec_elts ',' DOTDOT ',' pat_vec_elts ','  { $$ = mk_node("PatVec", 2, $1, $5); }
+|                  DOTDOT ',' pat_vec_elts      { $$ = mk_node("PatVec", 2, mk_none(), $3); }
+|                  DOTDOT ',' pat_vec_elts ','  { $$ = mk_node("PatVec", 2, mk_none(), $3); }
+|                  DOTDOT                       { $$ = mk_node("PatVec", 2, mk_none(), mk_none()); }
+| %empty                                        { $$ = mk_node("PatVec", 2, mk_none(), mk_none()); }
+;
+
+pat_vec_elts
+: pat                    { $$ = mk_node("PatVecElts", 1, $1); }
+| pat_vec_elts ',' pat   { $$ = ext_node($1, 1, $3); }
+;
+
+////////////////////////////////////////////////////////////////////////
+// Part 3: Types
+////////////////////////////////////////////////////////////////////////
+
+ty
+: ty_prim
+| ty_closure
+| '<' ty_sum AS trait_ref '>' MOD_SEP ident                                { $$ = mk_node("TyQualifiedPath", 3, $2, $4, $7); }
+| SHL ty_sum AS trait_ref '>' MOD_SEP ident AS trait_ref '>' MOD_SEP ident { $$ = mk_node("TyQualifiedPath", 3, mk_node("TyQualifiedPath", 3, $2, $4, $7), $9, $12); }
+| '(' ty_sums ')'                                                          { $$ = mk_node("TyTup", 1, $2); }
+| '(' ty_sums ',' ')'                                                      { $$ = mk_node("TyTup", 1, $2); }
+| '(' ')'                                                                  { $$ = mk_atom("TyNil"); }
+;
+
+ty_prim
+: %prec IDENT path_generic_args_without_colons              { $$ = mk_node("TyPath", 2, mk_node("global", 1, mk_atom("false")), $1); }
+| %prec IDENT MOD_SEP path_generic_args_without_colons      { $$ = mk_node("TyPath", 2, mk_node("global", 1, mk_atom("true")), $2); }
+| %prec IDENT SELF MOD_SEP path_generic_args_without_colons { $$ = mk_node("TyPath", 2, mk_node("self", 1, mk_atom("true")), $3); }
+| BOX ty                                                    { $$ = mk_node("TyBox", 1, $2); }
+| '*' maybe_mut_or_const ty                                 { $$ = mk_node("TyPtr", 2, $2, $3); }
+| '&' ty                                                    { $$ = mk_node("TyRptr", 2, mk_atom("MutImmutable"), $2); }
+| '&' MUT ty                                                { $$ = mk_node("TyRptr", 2, mk_atom("MutMutable"), $3); }
+| ANDAND ty                                                 { $$ = mk_node("TyRptr", 1, mk_node("TyRptr", 2, mk_atom("MutImmutable"), $2)); }
+| ANDAND MUT ty                                             { $$ = mk_node("TyRptr", 1, mk_node("TyRptr", 2, mk_atom("MutMutable"), $3)); }
+| '&' lifetime maybe_mut ty                                 { $$ = mk_node("TyRptr", 3, $2, $3, $4); }
+| ANDAND lifetime maybe_mut ty                              { $$ = mk_node("TyRptr", 1, mk_node("TyRptr", 3, $2, $3, $4)); }
+| '[' ty ']'                                                { $$ = mk_node("TyVec", 1, $2); }
+| '[' ty ',' DOTDOT expr ']'                                { $$ = mk_node("TyFixedLengthVec", 2, $2, $5); }
+| '[' ty ';' expr ']'                                       { $$ = mk_node("TyFixedLengthVec", 2, $2, $4); }
+| TYPEOF '(' expr ')'                                       { $$ = mk_node("TyTypeof", 1, $3); }
+| UNDERSCORE                                                { $$ = mk_atom("TyInfer"); }
+| ty_bare_fn
+| ty_proc
+| for_in_type
+;
+
+ty_bare_fn
+:                         FN ty_fn_decl { $$ = $2; }
+| UNSAFE                  FN ty_fn_decl { $$ = $3; }
+|        EXTERN maybe_abi FN ty_fn_decl { $$ = $4; }
+| UNSAFE EXTERN maybe_abi FN ty_fn_decl { $$ = $5; }
+;
+
+ty_fn_decl
+: generic_params fn_anon_params ret_ty { $$ = mk_node("TyFnDecl", 3, $1, $2, $3); }
+;
+
+ty_closure
+: UNSAFE '|' anon_params '|' maybe_bounds ret_ty { $$ = mk_node("TyClosure", 3, $3, $5, $6); }
+|        '|' anon_params '|' maybe_bounds ret_ty { $$ = mk_node("TyClosure", 3, $2, $4, $5); }
+| UNSAFE OROR maybe_bounds ret_ty                { $$ = mk_node("TyClosure", 2, $3, $4); }
+|        OROR maybe_bounds ret_ty                { $$ = mk_node("TyClosure", 2, $2, $3); }
+;
+
+ty_proc
+: PROC generic_params fn_params maybe_bounds ret_ty { $$ = mk_node("TyProc", 4, $2, $3, $4, $5); }
+;
+
+for_in_type
+: FOR '<' maybe_lifetimes '>' for_in_type_suffix { $$ = mk_node("ForInType", 2, $3, $5); }
+;
+
+for_in_type_suffix
+: ty_proc
+| ty_bare_fn
+| trait_ref
+| ty_closure
+;
+
+maybe_mut
+: MUT              { $$ = mk_atom("MutMutable"); }
+| %prec MUT %empty { $$ = mk_atom("MutImmutable"); }
+;
+
+maybe_mut_or_const
+: MUT    { $$ = mk_atom("MutMutable"); }
+| CONST  { $$ = mk_atom("MutImmutable"); }
+| %empty { $$ = mk_atom("MutImmutable"); }
+;
+
+ty_qualified_path_and_generic_values
+: ty_qualified_path maybe_bindings
+{
+  $$ = mk_node("GenericValues", 3, mk_none(), mk_node("TySums", 1, mk_node("TySum", 1, $1)), $2);
+}
+| ty_qualified_path ',' ty_sums maybe_bindings
+{
+  $$ = mk_node("GenericValues", 3, mk_none(), ext_node(mk_node("TySums", 1, $1), 1, $3), $4); }
+;
+
+ty_qualified_path
+: ty_sum AS trait_ref '>' MOD_SEP ident                     { $$ = mk_node("TyQualifiedPath", 3, $1, $3, $6); }
+| ty_sum AS trait_ref '>' MOD_SEP ident '+' ty_param_bounds { $$ = mk_node("TyQualifiedPath", 3, $1, $3, $6); }
+;
+
+maybe_ty_sums
+: ty_sums
+| ty_sums ','
+| %empty { $$ = mk_none(); }
+;
+
+ty_sums
+: ty_sum             { $$ = mk_node("TySums", 1, $1); }
+| ty_sums ',' ty_sum { $$ = ext_node($1, 1, $3); }
+;
+
+ty_sum
+: ty                     { $$ = mk_node("TySum", 1, $1); }
+| ty '+' ty_param_bounds { $$ = mk_node("TySum", 2, $1, $3); }
+;
+
+ty_prim_sum
+: ty_prim                     { $$ = mk_node("TySum", 1, $1); }
+| ty_prim '+' ty_param_bounds { $$ = mk_node("TySum", 2, $1, $3); }
+;
+
+maybe_ty_param_bounds
+: ':' ty_param_bounds { $$ = $2; }
+| %empty              { $$ = mk_none(); }
+;
+
+ty_param_bounds
+: boundseq
+| %empty { $$ = mk_none(); }
+;
+
+boundseq
+: polybound
+| boundseq '+' polybound { $$ = ext_node($1, 1, $3); }
+;
+
+polybound
+: FOR '<' maybe_lifetimes '>' bound { $$ = mk_node("PolyBound", 2, $3, $5); }
+| bound
+| '?' bound { $$ = $2; }
+;
+
+bindings
+: binding              { $$ = mk_node("Bindings", 1, $1); }
+| bindings ',' binding { $$ = ext_node($1, 1, $3); }
+;
+
+binding
+: ident '=' ty { mk_node("Binding", 2, $1, $3); }
+;
+
+ty_param
+: ident maybe_ty_param_bounds maybe_ty_default           { $$ = mk_node("TyParam", 3, $1, $2, $3); }
+| ident '?' ident maybe_ty_param_bounds maybe_ty_default { $$ = mk_node("TyParam", 4, $1, $3, $4, $5); }
+;
+
+maybe_bounds
+: %prec SHIFTPLUS
+  ':' bounds             { $$ = $2; }
+| %prec SHIFTPLUS %empty { $$ = mk_none(); }
+;
+
+bounds
+: bound            { $$ = mk_node("bounds", 1, $1); }
+| bounds '+' bound { $$ = ext_node($1, 1, $3); }
+;
+
+bound
+: lifetime
+| trait_ref
+;
+
+maybe_ltbounds
+: %prec SHIFTPLUS
+  ':' ltbounds       { $$ = $2; }
+| %empty             { $$ = mk_none(); }
+;
+
+ltbounds
+: lifetime              { $$ = mk_node("ltbounds", 1, $1); }
+| ltbounds '+' lifetime { $$ = ext_node($1, 1, $3); }
+;
+
+maybe_ty_default
+: '=' ty_sum { $$ = mk_node("TyDefault", 1, $2); }
+| %empty     { $$ = mk_none(); }
+;
+
+maybe_lifetimes
+: lifetimes
+| lifetimes ','
+| %empty { $$ = mk_none(); }
+;
+
+lifetimes
+: lifetime_and_bounds               { $$ = mk_node("Lifetimes", 1, $1); }
+| lifetimes ',' lifetime_and_bounds { $$ = ext_node($1, 1, $3); }
+;
+
+lifetime_and_bounds
+: LIFETIME maybe_ltbounds         { $$ = mk_node("lifetime", 2, mk_atom(yytext), $2); }
+| STATIC_LIFETIME                 { $$ = mk_atom("static_lifetime"); }
+;
+
+lifetime
+: LIFETIME         { $$ = mk_node("lifetime", 1, mk_atom(yytext)); }
+| STATIC_LIFETIME  { $$ = mk_atom("static_lifetime"); }
+;
+
+trait_ref
+: %prec IDENT path_generic_args_without_colons
+| %prec IDENT MOD_SEP path_generic_args_without_colons { $$ = $2; }
+;
+
+////////////////////////////////////////////////////////////////////////
+// Part 4: Blocks, statements, and expressions
+////////////////////////////////////////////////////////////////////////
+
+inner_attrs_and_block
+: '{' maybe_inner_attrs maybe_stmts '}'        { $$ = mk_node("ExprBlock", 2, $2, $3); }
+;
+
+block
+: '{' maybe_stmts '}'                          { $$ = mk_node("ExprBlock", 1, $2); }
+;
+
+maybe_stmts
+: stmts
+| stmts nonblock_expr { $$ = ext_node($1, 1, $2); }
+| nonblock_expr
+| %empty              { $$ = mk_none(); }
+;
+
+// There are two sub-grammars within a "stmts: exprs" derivation
+// depending on whether each stmt-expr is a block-expr form; this is to
+// handle the "semicolon rule" for stmt sequencing that permits
+// writing
+//
+//     if foo { bar } 10
+//
+// as a sequence of two stmts (one if-expr stmt, one lit-10-expr
+// stmt). Unfortunately by permitting juxtaposition of exprs in
+// sequence like that, the non-block expr grammar has to have a
+// second limited sub-grammar that excludes the prefix exprs that
+// are ambiguous with binops. That is to say:
+//
+//     {10} - 1
+//
+// should parse as (progn (progn 10) (- 1)) not (- (progn 10) 1), that
+// is to say, two statements rather than one, at least according to
+// the mainline rust parser.
+//
+// So we wind up with a 3-way split in exprs that occur in stmt lists:
+// block, nonblock-prefix, and nonblock-nonprefix.
+//
+// In non-stmts contexts, expr can relax this trichotomy.
+//
+// There are also two other expr subtypes: first, nonparen_expr
+// disallows exprs surrounded by parens (including tuple expressions),
+// this is neccesary for BOX (place) expressions, so a parens expr
+// following the BOX is always parsed as the place. There is also
+// expr_norange used in index_expr, which disallows '..' in
+// expressions as that has special meaning inside of brackets.
+
+stmts
+: stmt           { $$ = mk_node("stmts", 1, $1); }
+| stmts stmt     { $$ = ext_node($1, 1, $2); }
+;
+
+stmt
+: let
+|                 stmt_item
+|             PUB stmt_item { $$ = $2; }
+| outer_attrs     stmt_item { $$ = $2; }
+| outer_attrs PUB stmt_item { $$ = $3; }
+| full_block_expr
+| block
+| nonblock_expr ';'
+| ';'                   { $$ = mk_none(); }
+;
+
+maybe_exprs
+: exprs
+| exprs ','
+| %empty { $$ = mk_none(); }
+;
+
+maybe_expr
+: expr
+| %empty { $$ = mk_none(); }
+;
+
+exprs
+: expr                                                        { $$ = mk_node("exprs", 1, $1); }
+| exprs ',' expr                                              { $$ = ext_node($1, 1, $3); }
+;
+
+path_expr
+: path_generic_args_with_colons
+| MOD_SEP path_generic_args_with_colons      { $$ = $2; }
+| SELF MOD_SEP path_generic_args_with_colons { $$ = mk_node("SelfPath", 1, $3); }
+;
+
+// A path with a lifetime and type parameters with double colons before
+// the type parameters; e.g. `foo::bar::<'a>::Baz::<T>`
+//
+// These show up in expr context, in order to disambiguate from "less-than"
+// expressions.
+path_generic_args_with_colons
+: ident                                              { $$ = mk_node("components", 1, $1); }
+| path_generic_args_with_colons MOD_SEP ident        { $$ = ext_node($1, 1, $3); }
+| path_generic_args_with_colons MOD_SEP generic_args { $$ = ext_node($1, 1, $3); }
+;
+
+// the braces-delimited macro is a block_expr so it doesn't appear here
+macro_expr
+: path_expr '!' maybe_ident parens_delimited_token_trees   { $$ = mk_node("MacroExpr", 3, $1, $3, $4); }
+| path_expr '!' maybe_ident brackets_delimited_token_trees { $$ = mk_node("MacroExpr", 3, $1, $3, $4); }
+;
+
+nonblock_expr
+: lit                                                           { $$ = mk_node("ExprLit", 1, $1); }
+| %prec IDENT
+  path_expr                                                     { $$ = mk_node("ExprPath", 1, $1); }
+| SELF                                                          { $$ = mk_node("ExprPath", 1, mk_node("ident", 1, mk_atom("self"))); }
+| macro_expr                                                    { $$ = mk_node("ExprMac", 1, $1); }
+| path_expr '{' struct_expr_fields '}'                          { $$ = mk_node("ExprStruct", 2, $1, $3); }
+| nonblock_expr '.' path_generic_args_with_colons               { $$ = mk_node("ExprField", 2, $1, $3); }
+| nonblock_expr '.' LIT_INTEGER                                 { $$ = mk_node("ExprTupleIndex", 1, $1); }
+| nonblock_expr '[' index_expr ']'                              { $$ = mk_node("ExprIndex", 2, $1, $3); }
+| nonblock_expr '(' maybe_exprs ')'                             { $$ = mk_node("ExprCall", 2, $1, $3); }
+| '[' vec_expr ']'                                              { $$ = mk_node("ExprVec", 1, $2); }
+| '(' maybe_exprs ')'                                           { $$ = mk_node("ExprParen", 1, $2); }
+| CONTINUE                                                      { $$ = mk_node("ExprAgain", 0); }
+| CONTINUE lifetime                                             { $$ = mk_node("ExprAgain", 1, $2); }
+| RETURN                                                        { $$ = mk_node("ExprRet", 0); }
+| RETURN expr                                                   { $$ = mk_node("ExprRet", 1, $2); }
+| BREAK                                                         { $$ = mk_node("ExprBreak", 0); }
+| BREAK lifetime                                                { $$ = mk_node("ExprBreak", 1, $2); }
+| nonblock_expr '=' expr                                        { $$ = mk_node("ExprAssign", 2, $1, $3); }
+| nonblock_expr SHLEQ expr                                      { $$ = mk_node("ExprAssignShl", 2, $1, $3); }
+| nonblock_expr SHREQ expr                                      { $$ = mk_node("ExprAssignShr", 2, $1, $3); }
+| nonblock_expr MINUSEQ expr                                    { $$ = mk_node("ExprAssignSub", 2, $1, $3); }
+| nonblock_expr ANDEQ expr                                      { $$ = mk_node("ExprAssignBitAnd", 2, $1, $3); }
+| nonblock_expr OREQ expr                                       { $$ = mk_node("ExprAssignBitOr", 2, $1, $3); }
+| nonblock_expr PLUSEQ expr                                     { $$ = mk_node("ExprAssignAdd", 2, $1, $3); }
+| nonblock_expr STAREQ expr                                     { $$ = mk_node("ExprAssignMul", 2, $1, $3); }
+| nonblock_expr SLASHEQ expr                                    { $$ = mk_node("ExprAssignDiv", 2, $1, $3); }
+| nonblock_expr CARETEQ expr                                    { $$ = mk_node("ExprAssignBitXor", 2, $1, $3); }
+| nonblock_expr PERCENTEQ expr                                  { $$ = mk_node("ExprAssignRem", 2, $1, $3); }
+| nonblock_expr OROR expr                                       { $$ = mk_node("ExprBinary", 3, mk_atom("BiOr"), $1, $3); }
+| nonblock_expr ANDAND expr                                     { $$ = mk_node("ExprBinary", 3, mk_atom("BiAnd"), $1, $3); }
+| nonblock_expr EQEQ expr                                       { $$ = mk_node("ExprBinary", 3, mk_atom("BiEq"), $1, $3); }
+| nonblock_expr NE expr                                         { $$ = mk_node("ExprBinary", 3, mk_atom("BiNe"), $1, $3); }
+| nonblock_expr '<' expr                                        { $$ = mk_node("ExprBinary", 3, mk_atom("BiLt"), $1, $3); }
+| nonblock_expr '>' expr                                        { $$ = mk_node("ExprBinary", 3, mk_atom("BiGt"), $1, $3); }
+| nonblock_expr LE expr                                         { $$ = mk_node("ExprBinary", 3, mk_atom("BiLe"), $1, $3); }
+| nonblock_expr GE expr                                         { $$ = mk_node("ExprBinary", 3, mk_atom("BiGe"), $1, $3); }
+| nonblock_expr '|' expr                                        { $$ = mk_node("ExprBinary", 3, mk_atom("BiBitOr"), $1, $3); }
+| nonblock_expr '^' expr                                        { $$ = mk_node("ExprBinary", 3, mk_atom("BiBitXor"), $1, $3); }
+| nonblock_expr '&' expr                                        { $$ = mk_node("ExprBinary", 3, mk_atom("BiBitAnd"), $1, $3); }
+| nonblock_expr SHL expr                                        { $$ = mk_node("ExprBinary", 3, mk_atom("BiShl"), $1, $3); }
+| nonblock_expr SHR expr                                        { $$ = mk_node("ExprBinary", 3, mk_atom("BiShr"), $1, $3); }
+| nonblock_expr '+' expr                                        { $$ = mk_node("ExprBinary", 3, mk_atom("BiAdd"), $1, $3); }
+| nonblock_expr '-' expr                                        { $$ = mk_node("ExprBinary", 3, mk_atom("BiSub"), $1, $3); }
+| nonblock_expr '*' expr                                        { $$ = mk_node("ExprBinary", 3, mk_atom("BiMul"), $1, $3); }
+| nonblock_expr '/' expr                                        { $$ = mk_node("ExprBinary", 3, mk_atom("BiDiv"), $1, $3); }
+| nonblock_expr '%' expr                                        { $$ = mk_node("ExprBinary", 3, mk_atom("BiRem"), $1, $3); }
+| nonblock_expr DOTDOT                                          { $$ = mk_node("ExprRange", 2, $1, mk_none()); }
+| nonblock_expr DOTDOT expr                                     { $$ = mk_node("ExprRange", 2, $1, $3); }
+|               DOTDOT expr                                     { $$ = mk_node("ExprRange", 2, mk_none(), $2); }
+| nonblock_expr AS ty                                           { $$ = mk_node("ExprCast", 2, $1, $3); }
+| BOX nonparen_expr                                             { $$ = mk_node("ExprBox", 1, $2); }
+| %prec BOXPLACE BOX '(' maybe_expr ')' nonblock_expr           { $$ = mk_node("ExprBox", 2, $3, $5); }
+| nonblock_prefix_expr
+;
+
+expr
+: lit                                                 { $$ = mk_node("ExprLit", 1, $1); }
+| %prec IDENT
+  path_expr                                           { $$ = mk_node("ExprPath", 1, $1); }
+| SELF                                                { $$ = mk_node("ExprPath", 1, mk_node("ident", 1, mk_atom("self"))); }
+| macro_expr                                          { $$ = mk_node("ExprMac", 1, $1); }
+| path_expr '{' struct_expr_fields '}'                { $$ = mk_node("ExprStruct", 2, $1, $3); }
+| expr '.' path_generic_args_with_colons              { $$ = mk_node("ExprField", 2, $1, $3); }
+| expr '.' LIT_INTEGER                                { $$ = mk_node("ExprTupleIndex", 1, $1); }
+| expr '[' index_expr ']'                             { $$ = mk_node("ExprIndex", 2, $1, $3); }
+| expr '(' maybe_exprs ')'                            { $$ = mk_node("ExprCall", 2, $1, $3); }
+| '(' maybe_exprs ')'                                 { $$ = mk_node("ExprParen", 1, $2); }
+| '[' vec_expr ']'                                    { $$ = mk_node("ExprVec", 1, $2); }
+| CONTINUE                                            { $$ = mk_node("ExprAgain", 0); }
+| CONTINUE ident                                      { $$ = mk_node("ExprAgain", 1, $2); }
+| RETURN                                              { $$ = mk_node("ExprRet", 0); }
+| RETURN expr                                         { $$ = mk_node("ExprRet", 1, $2); }
+| BREAK                                               { $$ = mk_node("ExprBreak", 0); }
+| BREAK ident                                         { $$ = mk_node("ExprBreak", 1, $2); }
+| expr '=' expr                                       { $$ = mk_node("ExprAssign", 2, $1, $3); }
+| expr SHLEQ expr                                     { $$ = mk_node("ExprAssignShl", 2, $1, $3); }
+| expr SHREQ expr                                     { $$ = mk_node("ExprAssignShr", 2, $1, $3); }
+| expr MINUSEQ expr                                   { $$ = mk_node("ExprAssignSub", 2, $1, $3); }
+| expr ANDEQ expr                                     { $$ = mk_node("ExprAssignBitAnd", 2, $1, $3); }
+| expr OREQ expr                                      { $$ = mk_node("ExprAssignBitOr", 2, $1, $3); }
+| expr PLUSEQ expr                                    { $$ = mk_node("ExprAssignAdd", 2, $1, $3); }
+| expr STAREQ expr                                    { $$ = mk_node("ExprAssignMul", 2, $1, $3); }
+| expr SLASHEQ expr                                   { $$ = mk_node("ExprAssignDiv", 2, $1, $3); }
+| expr CARETEQ expr                                   { $$ = mk_node("ExprAssignBitXor", 2, $1, $3); }
+| expr PERCENTEQ expr                                 { $$ = mk_node("ExprAssignRem", 2, $1, $3); }
+| expr OROR expr                                      { $$ = mk_node("ExprBinary", 3, mk_atom("BiOr"), $1, $3); }
+| expr ANDAND expr                                    { $$ = mk_node("ExprBinary", 3, mk_atom("BiAnd"), $1, $3); }
+| expr EQEQ expr                                      { $$ = mk_node("ExprBinary", 3, mk_atom("BiEq"), $1, $3); }
+| expr NE expr                                        { $$ = mk_node("ExprBinary", 3, mk_atom("BiNe"), $1, $3); }
+| expr '<' expr                                       { $$ = mk_node("ExprBinary", 3, mk_atom("BiLt"), $1, $3); }
+| expr '>' expr                                       { $$ = mk_node("ExprBinary", 3, mk_atom("BiGt"), $1, $3); }
+| expr LE expr                                        { $$ = mk_node("ExprBinary", 3, mk_atom("BiLe"), $1, $3); }
+| expr GE expr                                        { $$ = mk_node("ExprBinary", 3, mk_atom("BiGe"), $1, $3); }
+| expr '|' expr                                       { $$ = mk_node("ExprBinary", 3, mk_atom("BiBitOr"), $1, $3); }
+| expr '^' expr                                       { $$ = mk_node("ExprBinary", 3, mk_atom("BiBitXor"), $1, $3); }
+| expr '&' expr                                       { $$ = mk_node("ExprBinary", 3, mk_atom("BiBitAnd"), $1, $3); }
+| expr SHL expr                                       { $$ = mk_node("ExprBinary", 3, mk_atom("BiShl"), $1, $3); }
+| expr SHR expr                                       { $$ = mk_node("ExprBinary", 3, mk_atom("BiShr"), $1, $3); }
+| expr '+' expr                                       { $$ = mk_node("ExprBinary", 3, mk_atom("BiAdd"), $1, $3); }
+| expr '-' expr                                       { $$ = mk_node("ExprBinary", 3, mk_atom("BiSub"), $1, $3); }
+| expr '*' expr                                       { $$ = mk_node("ExprBinary", 3, mk_atom("BiMul"), $1, $3); }
+| expr '/' expr                                       { $$ = mk_node("ExprBinary", 3, mk_atom("BiDiv"), $1, $3); }
+| expr '%' expr                                       { $$ = mk_node("ExprBinary", 3, mk_atom("BiRem"), $1, $3); }
+| expr DOTDOT                                         { $$ = mk_node("ExprRange", 2, $1, mk_none()); }
+| expr DOTDOT expr                                    { $$ = mk_node("ExprRange", 2, $1, $3); }
+|      DOTDOT expr                                    { $$ = mk_node("ExprRange", 2, mk_none(), $2); }
+| expr AS ty                                          { $$ = mk_node("ExprCast", 2, $1, $3); }
+| BOX nonparen_expr                                   { $$ = mk_node("ExprBox", 1, $2); }
+| %prec BOXPLACE BOX '(' maybe_expr ')' expr          { $$ = mk_node("ExprBox", 2, $3, $5); }
+| block_expr
+| block
+| nonblock_prefix_expr
+;
+
+nonparen_expr
+: lit                                                 { $$ = mk_node("ExprLit", 1, $1); }
+| %prec IDENT
+  path_expr                                           { $$ = mk_node("ExprPath", 1, $1); }
+| SELF                                                { $$ = mk_node("ExprPath", 1, mk_node("ident", 1, mk_atom("self"))); }
+| macro_expr                                          { $$ = mk_node("ExprMac", 1, $1); }
+| path_expr '{' struct_expr_fields '}'                { $$ = mk_node("ExprStruct", 2, $1, $3); }
+| nonparen_expr '.' path_generic_args_with_colons     { $$ = mk_node("ExprField", 2, $1, $3); }
+| nonparen_expr '.' LIT_INTEGER                       { $$ = mk_node("ExprTupleIndex", 1, $1); }
+| nonparen_expr '[' index_expr ']'                    { $$ = mk_node("ExprIndex", 2, $1, $3); }
+| nonparen_expr '(' maybe_exprs ')'                   { $$ = mk_node("ExprCall", 2, $1, $3); }
+| '[' vec_expr ']'                                    { $$ = mk_node("ExprVec", 1, $2); }
+| CONTINUE                                            { $$ = mk_node("ExprAgain", 0); }
+| CONTINUE ident                                      { $$ = mk_node("ExprAgain", 1, $2); }
+| RETURN                                              { $$ = mk_node("ExprRet", 0); }
+| RETURN expr                                         { $$ = mk_node("ExprRet", 1, $2); }
+| BREAK                                               { $$ = mk_node("ExprBreak", 0); }
+| BREAK ident                                         { $$ = mk_node("ExprBreak", 1, $2); }
+| nonparen_expr '=' nonparen_expr                     { $$ = mk_node("ExprAssign", 2, $1, $3); }
+| nonparen_expr SHLEQ nonparen_expr                   { $$ = mk_node("ExprAssignShl", 2, $1, $3); }
+| nonparen_expr SHREQ nonparen_expr                   { $$ = mk_node("ExprAssignShr", 2, $1, $3); }
+| nonparen_expr MINUSEQ nonparen_expr                 { $$ = mk_node("ExprAssignSub", 2, $1, $3); }
+| nonparen_expr ANDEQ nonparen_expr                   { $$ = mk_node("ExprAssignBitAnd", 2, $1, $3); }
+| nonparen_expr OREQ nonparen_expr                    { $$ = mk_node("ExprAssignBitOr", 2, $1, $3); }
+| nonparen_expr PLUSEQ nonparen_expr                  { $$ = mk_node("ExprAssignAdd", 2, $1, $3); }
+| nonparen_expr STAREQ nonparen_expr                  { $$ = mk_node("ExprAssignMul", 2, $1, $3); }
+| nonparen_expr SLASHEQ nonparen_expr                 { $$ = mk_node("ExprAssignDiv", 2, $1, $3); }
+| nonparen_expr CARETEQ nonparen_expr                 { $$ = mk_node("ExprAssignBitXor", 2, $1, $3); }
+| nonparen_expr PERCENTEQ nonparen_expr               { $$ = mk_node("ExprAssignRem", 2, $1, $3); }
+| nonparen_expr OROR nonparen_expr                    { $$ = mk_node("ExprBinary", 3, mk_atom("BiOr"), $1, $3); }
+| nonparen_expr ANDAND nonparen_expr                  { $$ = mk_node("ExprBinary", 3, mk_atom("BiAnd"), $1, $3); }
+| nonparen_expr EQEQ nonparen_expr                    { $$ = mk_node("ExprBinary", 3, mk_atom("BiEq"), $1, $3); }
+| nonparen_expr NE nonparen_expr                      { $$ = mk_node("ExprBinary", 3, mk_atom("BiNe"), $1, $3); }
+| nonparen_expr '<' nonparen_expr                     { $$ = mk_node("ExprBinary", 3, mk_atom("BiLt"), $1, $3); }
+| nonparen_expr '>' nonparen_expr                     { $$ = mk_node("ExprBinary", 3, mk_atom("BiGt"), $1, $3); }
+| nonparen_expr LE nonparen_expr                      { $$ = mk_node("ExprBinary", 3, mk_atom("BiLe"), $1, $3); }
+| nonparen_expr GE nonparen_expr                      { $$ = mk_node("ExprBinary", 3, mk_atom("BiGe"), $1, $3); }
+| nonparen_expr '|' nonparen_expr                     { $$ = mk_node("ExprBinary", 3, mk_atom("BiBitOr"), $1, $3); }
+| nonparen_expr '^' nonparen_expr                     { $$ = mk_node("ExprBinary", 3, mk_atom("BiBitXor"), $1, $3); }
+| nonparen_expr '&' nonparen_expr                     { $$ = mk_node("ExprBinary", 3, mk_atom("BiBitAnd"), $1, $3); }
+| nonparen_expr SHL nonparen_expr                     { $$ = mk_node("ExprBinary", 3, mk_atom("BiShl"), $1, $3); }
+| nonparen_expr SHR nonparen_expr                     { $$ = mk_node("ExprBinary", 3, mk_atom("BiShr"), $1, $3); }
+| nonparen_expr '+' nonparen_expr                     { $$ = mk_node("ExprBinary", 3, mk_atom("BiAdd"), $1, $3); }
+| nonparen_expr '-' nonparen_expr                     { $$ = mk_node("ExprBinary", 3, mk_atom("BiSub"), $1, $3); }
+| nonparen_expr '*' nonparen_expr                     { $$ = mk_node("ExprBinary", 3, mk_atom("BiMul"), $1, $3); }
+| nonparen_expr '/' nonparen_expr                     { $$ = mk_node("ExprBinary", 3, mk_atom("BiDiv"), $1, $3); }
+| nonparen_expr '%' nonparen_expr                     { $$ = mk_node("ExprBinary", 3, mk_atom("BiRem"), $1, $3); }
+| nonparen_expr DOTDOT                                { $$ = mk_node("ExprRange", 2, $1, mk_none()); }
+| nonparen_expr DOTDOT nonparen_expr                  { $$ = mk_node("ExprRange", 2, $1, $3); }
+|               DOTDOT nonparen_expr                  { $$ = mk_node("ExprRange", 2, mk_none(), $2); }
+| nonparen_expr AS ty                                 { $$ = mk_node("ExprCast", 2, $1, $3); }
+| BOX nonparen_expr                                   { $$ = mk_node("ExprBox", 1, $2); }
+| %prec BOXPLACE BOX '(' maybe_expr ')' expr          { $$ = mk_node("ExprBox", 1, $3, $5); }
+| block_expr
+| block
+| nonblock_prefix_expr
+;
+
+expr_norange
+: lit                                                 { $$ = mk_node("ExprLit", 1, $1); }
+| %prec IDENT
+  path_expr                                           { $$ = mk_node("ExprPath", 1, $1); }
+| SELF                                                { $$ = mk_node("ExprPath", 1, mk_node("ident", 1, mk_atom("self"))); }
+| macro_expr                                          { $$ = mk_node("ExprMac", 1, $1); }
+| path_expr '{' struct_expr_fields '}'                { $$ = mk_node("ExprStruct", 2, $1, $3); }
+| expr_norange '.' path_generic_args_with_colons      { $$ = mk_node("ExprField", 2, $1, $3); }
+| expr_norange '.' LIT_INTEGER                        { $$ = mk_node("ExprTupleIndex", 1, $1); }
+| expr_norange '[' index_expr ']'                     { $$ = mk_node("ExprIndex", 2, $1, $3); }
+| expr_norange '(' maybe_exprs ')'                    { $$ = mk_node("ExprCall", 2, $1, $3); }
+| '(' maybe_exprs ')'                                 { $$ = mk_node("ExprParen", 1, $2); }
+| '[' vec_expr ']'                                    { $$ = mk_node("ExprVec", 1, $2); }
+| CONTINUE                                            { $$ = mk_node("ExprAgain", 0); }
+| CONTINUE ident                                      { $$ = mk_node("ExprAgain", 1, $2); }
+| RETURN                                              { $$ = mk_node("ExprRet", 0); }
+| RETURN expr                                         { $$ = mk_node("ExprRet", 1, $2); }
+| BREAK                                               { $$ = mk_node("ExprBreak", 0); }
+| BREAK ident                                         { $$ = mk_node("ExprBreak", 1, $2); }
+| expr_norange '=' expr_norange                       { $$ = mk_node("ExprAssign", 2, $1, $3); }
+| expr_norange SHLEQ expr_norange                     { $$ = mk_node("ExprAssignShl", 2, $1, $3); }
+| expr_norange SHREQ expr_norange                     { $$ = mk_node("ExprAssignShr", 2, $1, $3); }
+| expr_norange MINUSEQ expr_norange                   { $$ = mk_node("ExprAssignSub", 2, $1, $3); }
+| expr_norange ANDEQ expr_norange                     { $$ = mk_node("ExprAssignBitAnd", 2, $1, $3); }
+| expr_norange OREQ expr_norange                      { $$ = mk_node("ExprAssignBitOr", 2, $1, $3); }
+| expr_norange PLUSEQ expr_norange                    { $$ = mk_node("ExprAssignAdd", 2, $1, $3); }
+| expr_norange STAREQ expr_norange                    { $$ = mk_node("ExprAssignMul", 2, $1, $3); }
+| expr_norange SLASHEQ expr_norange                   { $$ = mk_node("ExprAssignDiv", 2, $1, $3); }
+| expr_norange CARETEQ expr_norange                   { $$ = mk_node("ExprAssignBitXor", 2, $1, $3); }
+| expr_norange PERCENTEQ expr_norange                 { $$ = mk_node("ExprAssignRem", 2, $1, $3); }
+| expr_norange OROR expr_norange                      { $$ = mk_node("ExprBinary", 3, mk_atom("BiOr"), $1, $3); }
+| expr_norange ANDAND expr_norange                    { $$ = mk_node("ExprBinary", 3, mk_atom("BiAnd"), $1, $3); }
+| expr_norange EQEQ expr_norange                      { $$ = mk_node("ExprBinary", 3, mk_atom("BiEq"), $1, $3); }
+| expr_norange NE expr_norange                        { $$ = mk_node("ExprBinary", 3, mk_atom("BiNe"), $1, $3); }
+| expr_norange '<' expr_norange                       { $$ = mk_node("ExprBinary", 3, mk_atom("BiLt"), $1, $3); }
+| expr_norange '>' expr_norange                       { $$ = mk_node("ExprBinary", 3, mk_atom("BiGt"), $1, $3); }
+| expr_norange LE expr_norange                        { $$ = mk_node("ExprBinary", 3, mk_atom("BiLe"), $1, $3); }
+| expr_norange GE expr_norange                        { $$ = mk_node("ExprBinary", 3, mk_atom("BiGe"), $1, $3); }
+| expr_norange '|' expr_norange                       { $$ = mk_node("ExprBinary", 3, mk_atom("BiBitOr"), $1, $3); }
+| expr_norange '^' expr_norange                       { $$ = mk_node("ExprBinary", 3, mk_atom("BiBitXor"), $1, $3); }
+| expr_norange '&' expr_norange                       { $$ = mk_node("ExprBinary", 3, mk_atom("BiBitAnd"), $1, $3); }
+| expr_norange SHL expr_norange                       { $$ = mk_node("ExprBinary", 3, mk_atom("BiShl"), $1, $3); }
+| expr_norange SHR expr_norange                       { $$ = mk_node("ExprBinary", 3, mk_atom("BiShr"), $1, $3); }
+| expr_norange '+' expr_norange                       { $$ = mk_node("ExprBinary", 3, mk_atom("BiAdd"), $1, $3); }
+| expr_norange '-' expr_norange                       { $$ = mk_node("ExprBinary", 3, mk_atom("BiSub"), $1, $3); }
+| expr_norange '*' expr_norange                       { $$ = mk_node("ExprBinary", 3, mk_atom("BiMul"), $1, $3); }
+| expr_norange '/' expr_norange                       { $$ = mk_node("ExprBinary", 3, mk_atom("BiDiv"), $1, $3); }
+| expr_norange '%' expr_norange                       { $$ = mk_node("ExprBinary", 3, mk_atom("BiRem"), $1, $3); }
+| expr_norange AS ty                                  { $$ = mk_node("Expr_NorangeCast", 2, $1, $3); }
+| BOX nonparen_expr                                   { $$ = mk_node("ExprBox", 1, $2); }
+| %prec BOXPLACE BOX '(' maybe_expr ')' expr_norange  { $$ = mk_node("ExprBox", 2, $3, $5); }
+| block_expr
+| block
+| nonblock_prefix_expr
+;
+
+expr_nostruct
+: lit                                                 { $$ = mk_node("ExprLit", 1, $1); }
+| %prec IDENT
+  path_expr                                           { $$ = mk_node("ExprPath", 1, $1); }
+| SELF                                                { $$ = mk_node("ExprPath", 1, mk_node("ident", 1, mk_atom("self"))); }
+| macro_expr                                          { $$ = mk_node("ExprMac", 1, $1); }
+| expr_nostruct '.' path_generic_args_with_colons     { $$ = mk_node("ExprField", 2, $1, $3); }
+| expr_nostruct '.' LIT_INTEGER                       { $$ = mk_node("ExprTupleIndex", 1, $1); }
+| expr_nostruct '[' index_expr ']'                    { $$ = mk_node("ExprIndex", 2, $1, $3); }
+| expr_nostruct '(' maybe_exprs ')'                   { $$ = mk_node("ExprCall", 2, $1, $3); }
+| '[' vec_expr ']'                                    { $$ = mk_node("ExprVec", 1, $2); }
+| '(' maybe_exprs ')'                                 { $$ = mk_node("ExprParen", 1, $2); }
+| CONTINUE                                            { $$ = mk_node("ExprAgain", 0); }
+| CONTINUE ident                                      { $$ = mk_node("ExprAgain", 1, $2); }
+| RETURN                                              { $$ = mk_node("ExprRet", 0); }
+| RETURN expr                                         { $$ = mk_node("ExprRet", 1, $2); }
+| BREAK                                               { $$ = mk_node("ExprBreak", 0); }
+| BREAK ident                                         { $$ = mk_node("ExprBreak", 1, $2); }
+| expr_nostruct '=' expr_nostruct                     { $$ = mk_node("ExprAssign", 2, $1, $3); }
+| expr_nostruct SHLEQ expr_nostruct                   { $$ = mk_node("ExprAssignShl", 2, $1, $3); }
+| expr_nostruct SHREQ expr_nostruct                   { $$ = mk_node("ExprAssignShr", 2, $1, $3); }
+| expr_nostruct MINUSEQ expr_nostruct                 { $$ = mk_node("ExprAssignSub", 2, $1, $3); }
+| expr_nostruct ANDEQ expr_nostruct                   { $$ = mk_node("ExprAssignBitAnd", 2, $1, $3); }
+| expr_nostruct OREQ expr_nostruct                    { $$ = mk_node("ExprAssignBitOr", 2, $1, $3); }
+| expr_nostruct PLUSEQ expr_nostruct                  { $$ = mk_node("ExprAssignAdd", 2, $1, $3); }
+| expr_nostruct STAREQ expr_nostruct                  { $$ = mk_node("ExprAssignMul", 2, $1, $3); }
+| expr_nostruct SLASHEQ expr_nostruct                 { $$ = mk_node("ExprAssignDiv", 2, $1, $3); }
+| expr_nostruct CARETEQ expr_nostruct                 { $$ = mk_node("ExprAssignBitXor", 2, $1, $3); }
+| expr_nostruct PERCENTEQ expr_nostruct               { $$ = mk_node("ExprAssignRem", 2, $1, $3); }
+| expr_nostruct OROR expr_nostruct                    { $$ = mk_node("ExprBinary", 3, mk_atom("BiOr"), $1, $3); }
+| expr_nostruct ANDAND expr_nostruct                  { $$ = mk_node("ExprBinary", 3, mk_atom("BiAnd"), $1, $3); }
+| expr_nostruct EQEQ expr_nostruct                    { $$ = mk_node("ExprBinary", 3, mk_atom("BiEq"), $1, $3); }
+| expr_nostruct NE expr_nostruct                      { $$ = mk_node("ExprBinary", 3, mk_atom("BiNe"), $1, $3); }
+| expr_nostruct '<' expr_nostruct                     { $$ = mk_node("ExprBinary", 3, mk_atom("BiLt"), $1, $3); }
+| expr_nostruct '>' expr_nostruct                     { $$ = mk_node("ExprBinary", 3, mk_atom("BiGt"), $1, $3); }
+| expr_nostruct LE expr_nostruct                      { $$ = mk_node("ExprBinary", 3, mk_atom("BiLe"), $1, $3); }
+| expr_nostruct GE expr_nostruct                      { $$ = mk_node("ExprBinary", 3, mk_atom("BiGe"), $1, $3); }
+| expr_nostruct '|' expr_nostruct                     { $$ = mk_node("ExprBinary", 3, mk_atom("BiBitOr"), $1, $3); }
+| expr_nostruct '^' expr_nostruct                     { $$ = mk_node("ExprBinary", 3, mk_atom("BiBitXor"), $1, $3); }
+| expr_nostruct '&' expr_nostruct                     { $$ = mk_node("ExprBinary", 3, mk_atom("BiBitAnd"), $1, $3); }
+| expr_nostruct SHL expr_nostruct                     { $$ = mk_node("ExprBinary", 3, mk_atom("BiShl"), $1, $3); }
+| expr_nostruct SHR expr_nostruct                     { $$ = mk_node("ExprBinary", 3, mk_atom("BiShr"), $1, $3); }
+| expr_nostruct '+' expr_nostruct                     { $$ = mk_node("ExprBinary", 3, mk_atom("BiAdd"), $1, $3); }
+| expr_nostruct '-' expr_nostruct                     { $$ = mk_node("ExprBinary", 3, mk_atom("BiSub"), $1, $3); }
+| expr_nostruct '*' expr_nostruct                     { $$ = mk_node("ExprBinary", 3, mk_atom("BiMul"), $1, $3); }
+| expr_nostruct '/' expr_nostruct                     { $$ = mk_node("ExprBinary", 3, mk_atom("BiDiv"), $1, $3); }
+| expr_nostruct '%' expr_nostruct                     { $$ = mk_node("ExprBinary", 3, mk_atom("BiRem"), $1, $3); }
+| expr_nostruct DOTDOT                                { $$ = mk_node("ExprRange", 2, $1, mk_none()); }
+| expr_nostruct DOTDOT expr_nostruct                  { $$ = mk_node("ExprRange", 2, $1, $3); }
+|               DOTDOT expr_nostruct                  { $$ = mk_node("ExprRange", 2, mk_none(), $2); }
+| expr_nostruct AS ty                                 { $$ = mk_node("ExprCast", 2, $1, $3); }
+| BOX nonparen_expr                                   { $$ = mk_node("ExprBox", 1, $2); }
+| %prec BOXPLACE BOX '(' maybe_expr ')' expr_nostruct { $$ = mk_node("ExprBox", 1, $3, $5); }
+| block_expr
+| block
+| nonblock_prefix_expr_nostruct
+;
+
+nonblock_prefix_expr_nostruct
+: '-' expr_nostruct                         { $$ = mk_node("ExprUnary", 2, mk_atom("UnNeg"), $2); }
+| '!' expr_nostruct                         { $$ = mk_node("ExprUnary", 2, mk_atom("UnNot"), $2); }
+| '*' expr_nostruct                         { $$ = mk_node("ExprUnary", 2, mk_atom("UnDeref"), $2); }
+| '&' maybe_mut expr_nostruct               { $$ = mk_node("ExprAddrOf", 2, $2, $3); }
+| ANDAND maybe_mut expr_nostruct            { $$ = mk_node("ExprAddrOf", 1, mk_node("ExprAddrOf", 2, $2, $3)); }
+| lambda_expr_nostruct
+| MOVE lambda_expr_nostruct                 { $$ = $2; }
+| proc_expr_nostruct
+;
+
+nonblock_prefix_expr
+: '-' expr                         { $$ = mk_node("ExprUnary", 2, mk_atom("UnNeg"), $2); }
+| '!' expr                         { $$ = mk_node("ExprUnary", 2, mk_atom("UnNot"), $2); }
+| '*' expr                         { $$ = mk_node("ExprUnary", 2, mk_atom("UnDeref"), $2); }
+| '&' maybe_mut expr               { $$ = mk_node("ExprAddrOf", 2, $2, $3); }
+| ANDAND maybe_mut expr            { $$ = mk_node("ExprAddrOf", 1, mk_node("ExprAddrOf", 2, $2, $3)); }
+| lambda_expr
+| MOVE lambda_expr                 { $$ = $2; }
+| proc_expr
+;
+
+lambda_expr
+: %prec LAMBDA
+  OROR ret_ty expr                                        { $$ = mk_node("ExprFnBlock", 3, mk_none(), $2, $3); }
+| %prec LAMBDA
+  '|' maybe_unboxed_closure_kind '|' ret_ty expr          { $$ = mk_node("ExprFnBlock", 3, mk_none(), $4, $5); }
+| %prec LAMBDA
+  '|' inferrable_params '|' ret_ty expr                   { $$ = mk_node("ExprFnBlock", 3, $2, $4, $5); }
+| %prec LAMBDA
+  '|' '&' maybe_mut ':' inferrable_params '|' ret_ty expr { $$ = mk_node("ExprFnBlock", 3, $5, $7, $8); }
+| %prec LAMBDA
+  '|' ':' inferrable_params '|' ret_ty expr               { $$ = mk_node("ExprFnBlock", 3, $3, $5, $6); }
+;
+
+lambda_expr_nostruct
+: %prec LAMBDA
+  OROR expr_nostruct                                        { $$ = mk_node("ExprFnBlock", 2, mk_none(), $2); }
+| %prec LAMBDA
+  '|' maybe_unboxed_closure_kind '|'  expr_nostruct         { $$ = mk_node("ExprFnBlock", 2, mk_none(), $4); }
+| %prec LAMBDA
+  '|' inferrable_params '|' expr_nostruct                   { $$ = mk_node("ExprFnBlock", 2, $2, $4); }
+| %prec LAMBDA
+  '|' '&' maybe_mut ':' inferrable_params '|' expr_nostruct { $$ = mk_node("ExprFnBlock", 2, $5, $7); }
+| %prec LAMBDA
+  '|' ':' inferrable_params '|' expr_nostruct               { $$ = mk_node("ExprFnBlock", 2, $3, $5); }
+
+;
+
+proc_expr
+: %prec LAMBDA
+  PROC '(' ')' expr                         { $$ = mk_node("ExprProc", 2, mk_none(), $4); }
+| %prec LAMBDA
+  PROC '(' inferrable_params ')' expr       { $$ = mk_node("ExprProc", 2, $3, $5); }
+;
+
+proc_expr_nostruct
+: %prec LAMBDA
+  PROC '(' ')' expr_nostruct                     { $$ = mk_node("ExprProc", 2, mk_none(), $4); }
+| %prec LAMBDA
+  PROC '(' inferrable_params ')' expr_nostruct   { $$ = mk_node("ExprProc", 2, $3, $5); }
+;
+
+vec_expr
+: maybe_exprs
+| exprs ';' expr { $$ = mk_node("VecRepeat", 2, $1, $3); }
+;
+
+index_expr
+: expr_norange                     { $$ = mk_node("Index", 1, $1); }
+| expr_norange DOTDOT              { $$ = mk_node("SliceToEnd", 1, $1); }
+|              DOTDOT expr_norange { $$ = mk_node("SliceFromBeginning", 1, $2); }
+| expr_norange DOTDOT expr_norange { $$ = mk_node("Slice", 2, $1, $3); }
+| %empty                           { $$ = mk_none(); }
+;
+
+struct_expr_fields
+: field_inits
+| field_inits ','
+| maybe_field_inits default_field_init { $$ = ext_node($1, 1, $2); }
+;
+
+maybe_field_inits
+: field_inits
+| field_inits ','
+| %empty { $$ = mk_none(); }
+;
+
+field_inits
+: field_init                 { $$ = mk_node("FieldInits", 1, $1); }
+| field_inits ',' field_init { $$ = ext_node($1, 1, $3); }
+;
+
+field_init
+: ident ':' expr   { $$ = mk_node("FieldInit", 2, $1, $3); }
+;
+
+default_field_init
+: DOTDOT expr   { $$ = mk_node("DefaultFieldInit", 1, $2); }
+;
+
+block_expr
+: expr_match
+| expr_if
+| expr_if_let
+| expr_while
+| expr_while_let
+| expr_loop
+| expr_for
+| UNSAFE block                                           { $$ = mk_node("UnsafeBlock", 1, $2); }
+| path_expr '!' maybe_ident braces_delimited_token_trees { $$ = mk_node("Macro", 3, $1, $3, $4); }
+;
+
+full_block_expr
+: block_expr
+| full_block_expr '.' path_generic_args_with_colons { $$ = mk_node("ExprField", 2, $1, $3); }
+| full_block_expr '.' LIT_INTEGER                   { $$ = mk_node("ExprTupleIndex", 1, $1); }
+;
+
+expr_match
+: MATCH expr_nostruct '{' '}'                                     { $$ = mk_node("ExprMatch", 1, $2); }
+| MATCH expr_nostruct '{' match_clauses                       '}' { $$ = mk_node("ExprMatch", 2, $2, $4); }
+| MATCH expr_nostruct '{' match_clauses nonblock_match_clause '}' { $$ = mk_node("ExprMatch", 2, $2, ext_node($4, 1, $5)); }
+| MATCH expr_nostruct '{'               nonblock_match_clause '}' { $$ = mk_node("ExprMatch", 2, $2, mk_node("Arms", 1, $4)); }
+;
+
+match_clauses
+: match_clause               { $$ = mk_node("Arms", 1, $1); }
+| match_clauses match_clause { $$ = ext_node($1, 1, $2); }
+;
+
+match_clause
+: nonblock_match_clause ','
+| block_match_clause
+| block_match_clause ','
+;
+
+nonblock_match_clause
+: maybe_outer_attrs pats_or maybe_guard FAT_ARROW nonblock_expr   { $$ = mk_node("Arm", 4, $1, $2, $3, $5); }
+| maybe_outer_attrs pats_or maybe_guard FAT_ARROW full_block_expr { $$ = mk_node("Arm", 4, $1, $2, $3, $5); }
+;
+
+block_match_clause
+: maybe_outer_attrs pats_or maybe_guard FAT_ARROW block { $$ = mk_node("Arm", 4, $1, $2, $3, $5); }
+;
+
+maybe_guard
+: IF expr_nostruct           { $$ = $2; }
+| %empty                     { $$ = mk_none(); }
+;
+
+expr_if
+: IF expr_nostruct block                              { $$ = mk_node("ExprIf", 2, $2, $3); }
+| IF expr_nostruct block ELSE block_or_if             { $$ = mk_node("ExprIf", 3, $2, $3, $5); }
+;
+
+expr_if_let
+: IF LET pat '=' expr_nostruct block                  { $$ = mk_node("ExprIfLet", 3, $3, $5, $6); }
+| IF LET pat '=' expr_nostruct block ELSE block_or_if { $$ = mk_node("ExprIfLet", 4, $3, $5, $6, $8); }
+;
+
+block_or_if
+: block
+| expr_if
+| expr_if_let
+;
+
+expr_while
+: maybe_label WHILE expr_nostruct block               { $$ = mk_node("ExprWhile", 3, $1, $3, $4); }
+;
+
+expr_while_let
+: maybe_label WHILE LET pat '=' expr_nostruct block   { $$ = mk_node("ExprWhileLet", 4, $1, $4, $6, $7); }
+;
+
+expr_loop
+: maybe_label LOOP block                              { $$ = mk_node("ExprLoop", 2, $1, $3); }
+;
+
+expr_for
+: maybe_label FOR pat IN expr_nostruct block          { $$ = mk_node("ExprForLoop", 4, $1, $3, $5, $6); }
+;
+
+maybe_label
+: lifetime ':'
+| %empty { $$ = mk_none(); }
+;
+
+let
+: LET pat maybe_ty_ascription maybe_init_expr ';' { $$ = mk_node("DeclLocal", 3, $2, $3, $4); }
+;
+
+////////////////////////////////////////////////////////////////////////
+// Part 5: Macros and misc. rules
+////////////////////////////////////////////////////////////////////////
+
+lit
+: LIT_BYTE                   { $$ = mk_node("LitByte", 1, mk_atom(yytext)); }
+| LIT_CHAR                   { $$ = mk_node("LitChar", 1, mk_atom(yytext)); }
+| LIT_INTEGER                { $$ = mk_node("LitInteger", 1, mk_atom(yytext)); }
+| LIT_FLOAT                  { $$ = mk_node("LitFloat", 1, mk_atom(yytext)); }
+| TRUE                       { $$ = mk_node("LitBool", 1, mk_atom(yytext)); }
+| FALSE                      { $$ = mk_node("LitBool", 1, mk_atom(yytext)); }
+| str
+;
+
+str
+: LIT_STR                    { $$ = mk_node("LitStr", 1, mk_atom(yytext), mk_atom("CookedStr")); }
+| LIT_STR_RAW                { $$ = mk_node("LitStr", 1, mk_atom(yytext), mk_atom("RawStr")); }
+| LIT_BINARY                 { $$ = mk_node("LitBinary", 1, mk_atom(yytext), mk_atom("BinaryStr")); }
+| LIT_BINARY_RAW             { $$ = mk_node("LitBinary", 1, mk_atom(yytext), mk_atom("RawBinaryStr")); }
+;
+
+maybe_ident
+: %empty { $$ = mk_none(); }
+| ident
+;
+
+ident
+: IDENT                      { $$ = mk_node("ident", 1, mk_atom(yytext)); }
+;
+
+unpaired_token
+: SHL                        { $$ = mk_atom(yytext); }
+| SHR                        { $$ = mk_atom(yytext); }
+| LE                         { $$ = mk_atom(yytext); }
+| EQEQ                       { $$ = mk_atom(yytext); }
+| NE                         { $$ = mk_atom(yytext); }
+| GE                         { $$ = mk_atom(yytext); }
+| ANDAND                     { $$ = mk_atom(yytext); }
+| OROR                       { $$ = mk_atom(yytext); }
+| SHLEQ                      { $$ = mk_atom(yytext); }
+| SHREQ                      { $$ = mk_atom(yytext); }
+| MINUSEQ                    { $$ = mk_atom(yytext); }
+| ANDEQ                      { $$ = mk_atom(yytext); }
+| OREQ                       { $$ = mk_atom(yytext); }
+| PLUSEQ                     { $$ = mk_atom(yytext); }
+| STAREQ                     { $$ = mk_atom(yytext); }
+| SLASHEQ                    { $$ = mk_atom(yytext); }
+| CARETEQ                    { $$ = mk_atom(yytext); }
+| PERCENTEQ                  { $$ = mk_atom(yytext); }
+| DOTDOT                     { $$ = mk_atom(yytext); }
+| DOTDOTDOT                  { $$ = mk_atom(yytext); }
+| MOD_SEP                    { $$ = mk_atom(yytext); }
+| RARROW                     { $$ = mk_atom(yytext); }
+| FAT_ARROW                  { $$ = mk_atom(yytext); }
+| LIT_BYTE                   { $$ = mk_atom(yytext); }
+| LIT_CHAR                   { $$ = mk_atom(yytext); }
+| LIT_INTEGER                { $$ = mk_atom(yytext); }
+| LIT_FLOAT                  { $$ = mk_atom(yytext); }
+| LIT_STR                    { $$ = mk_atom(yytext); }
+| LIT_STR_RAW                { $$ = mk_atom(yytext); }
+| LIT_BINARY                 { $$ = mk_atom(yytext); }
+| LIT_BINARY_RAW             { $$ = mk_atom(yytext); }
+| IDENT                      { $$ = mk_atom(yytext); }
+| UNDERSCORE                 { $$ = mk_atom(yytext); }
+| LIFETIME                   { $$ = mk_atom(yytext); }
+| SELF                       { $$ = mk_atom(yytext); }
+| STATIC                     { $$ = mk_atom(yytext); }
+| AS                         { $$ = mk_atom(yytext); }
+| BREAK                      { $$ = mk_atom(yytext); }
+| CRATE                      { $$ = mk_atom(yytext); }
+| ELSE                       { $$ = mk_atom(yytext); }
+| ENUM                       { $$ = mk_atom(yytext); }
+| EXTERN                     { $$ = mk_atom(yytext); }
+| FALSE                      { $$ = mk_atom(yytext); }
+| FN                         { $$ = mk_atom(yytext); }
+| FOR                        { $$ = mk_atom(yytext); }
+| IF                         { $$ = mk_atom(yytext); }
+| IMPL                       { $$ = mk_atom(yytext); }
+| IN                         { $$ = mk_atom(yytext); }
+| LET                        { $$ = mk_atom(yytext); }
+| LOOP                       { $$ = mk_atom(yytext); }
+| MATCH                      { $$ = mk_atom(yytext); }
+| MOD                        { $$ = mk_atom(yytext); }
+| MOVE                       { $$ = mk_atom(yytext); }
+| MUT                        { $$ = mk_atom(yytext); }
+| PRIV                       { $$ = mk_atom(yytext); }
+| PUB                        { $$ = mk_atom(yytext); }
+| REF                        { $$ = mk_atom(yytext); }
+| RETURN                     { $$ = mk_atom(yytext); }
+| STRUCT                     { $$ = mk_atom(yytext); }
+| TRUE                       { $$ = mk_atom(yytext); }
+| TRAIT                      { $$ = mk_atom(yytext); }
+| TYPE                       { $$ = mk_atom(yytext); }
+| UNSAFE                     { $$ = mk_atom(yytext); }
+| USE                        { $$ = mk_atom(yytext); }
+| WHILE                      { $$ = mk_atom(yytext); }
+| CONTINUE                   { $$ = mk_atom(yytext); }
+| PROC                       { $$ = mk_atom(yytext); }
+| BOX                        { $$ = mk_atom(yytext); }
+| CONST                      { $$ = mk_atom(yytext); }
+| WHERE                      { $$ = mk_atom(yytext); }
+| TYPEOF                     { $$ = mk_atom(yytext); }
+| INNER_DOC_COMMENT          { $$ = mk_atom(yytext); }
+| OUTER_DOC_COMMENT          { $$ = mk_atom(yytext); }
+| SHEBANG                    { $$ = mk_atom(yytext); }
+| STATIC_LIFETIME            { $$ = mk_atom(yytext); }
+| ';'                        { $$ = mk_atom(yytext); }
+| ','                        { $$ = mk_atom(yytext); }
+| '.'                        { $$ = mk_atom(yytext); }
+| '@'                        { $$ = mk_atom(yytext); }
+| '#'                        { $$ = mk_atom(yytext); }
+| '~'                        { $$ = mk_atom(yytext); }
+| ':'                        { $$ = mk_atom(yytext); }
+| '$'                        { $$ = mk_atom(yytext); }
+| '='                        { $$ = mk_atom(yytext); }
+| '?'                        { $$ = mk_atom(yytext); }
+| '!'                        { $$ = mk_atom(yytext); }
+| '<'                        { $$ = mk_atom(yytext); }
+| '>'                        { $$ = mk_atom(yytext); }
+| '-'                        { $$ = mk_atom(yytext); }
+| '&'                        { $$ = mk_atom(yytext); }
+| '|'                        { $$ = mk_atom(yytext); }
+| '+'                        { $$ = mk_atom(yytext); }
+| '*'                        { $$ = mk_atom(yytext); }
+| '/'                        { $$ = mk_atom(yytext); }
+| '^'                        { $$ = mk_atom(yytext); }
+| '%'                        { $$ = mk_atom(yytext); }
+;
+
+token_trees
+: %empty                     { $$ = mk_node("TokenTrees", 0); }
+| token_trees token_tree     { $$ = ext_node($1, 1, $2); }
+;
+
+token_tree
+: delimited_token_trees
+| unpaired_token         { $$ = mk_node("TTTok", 1, $1); }
+;
+
+delimited_token_trees
+: parens_delimited_token_trees
+| braces_delimited_token_trees
+| brackets_delimited_token_trees
+;
+
+parens_delimited_token_trees
+: '(' token_trees ')'
+{
+  $$ = mk_node("TTDelim", 3,
+               mk_node("TTTok", 1, mk_atom("(")),
+               $2,
+               mk_node("TTTok", 1, mk_atom(")")));
+}
+;
+
+braces_delimited_token_trees
+: '{' token_trees '}'
+{
+  $$ = mk_node("TTDelim", 3,
+               mk_node("TTTok", 1, mk_atom("{")),
+               $2,
+               mk_node("TTTok", 1, mk_atom("}")));
+}
+;
+
+brackets_delimited_token_trees
+: '[' token_trees ']'
+{
+  $$ = mk_node("TTDelim", 3,
+               mk_node("TTTok", 1, mk_atom("[")),
+               $2,
+               mk_node("TTTok", 1, mk_atom("]")));
+}
+;
diff --git a/src/grammar/testparser.py b/src/grammar/testparser.py
new file mode 100755
index 00000000000..38e57be288b
--- /dev/null
+++ b/src/grammar/testparser.py
@@ -0,0 +1,65 @@
+#!/usr/bin/env python
+#
+# Copyright 2015 The Rust Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution and at
+# http://rust-lang.org/COPYRIGHT.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+import sys
+
+import os
+import subprocess
+import argparse
+
+# usage: testparser.py [-h] [-p PARSER [PARSER ...]] -s SOURCE_DIR
+
+# Parsers should read from stdin and return exit status 0 for a
+# successful parse, and nonzero for an unsuccessful parse
+
+parser = argparse.ArgumentParser()
+parser.add_argument('-p', '--parser', nargs='+')
+parser.add_argument('-s', '--source-dir', nargs=1, required=True)
+args = parser.parse_args(sys.argv[1:])
+
+total = 0
+ok = {}
+bad = {}
+for parser in args.parser:
+    ok[parser] = 0
+    bad[parser] = []
+devnull = open(os.devnull, 'w')
+print "\n"
+
+for base, dirs, files in os.walk(args.source_dir[0]):
+    for f in filter(lambda p: p.endswith('.rs'), files):
+        p = os.path.join(base, f)
+        compile_fail = 'compile-fail' in p
+        ignore = any('ignore-test' in line or 'ignore-lexer-test' in line
+                     for line in open(p).readlines())
+        if compile_fail or ignore:
+            continue
+        total += 1
+        for parser in args.parser:
+            if subprocess.call(parser, stdin=open(p), stderr=subprocess.STDOUT, stdout=devnull) == 0:
+                ok[parser] += 1
+            else:
+                bad[parser].append(p)
+        parser_stats = ', '.join(['{}: {}'.format(parser, ok[parser]) for parser in args.parser])
+        sys.stdout.write("\033[K\r total: {}, {}, scanned {}"
+                         .format(total, os.path.relpath(parser_stats), os.path.relpath(p)))
+
+devnull.close()
+
+print "\n"
+
+for parser in args.parser:
+    filename = os.path.basename(parser) + '.bad'
+    print("writing {} files that failed to parse with {} to {}".format(len(bad[parser]), parser, filename))
+    with open(filename, "w") as f:
+          for p in bad[parser]:
+              f.write(p)
+              f.write("\n")
diff --git a/src/grammar/tokens.h b/src/grammar/tokens.h
new file mode 100644
index 00000000000..4457e3f8b5a
--- /dev/null
+++ b/src/grammar/tokens.h
@@ -0,0 +1,91 @@
+// Copyright 2015 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+enum Token {
+  SHL = 257, // Parser generators reserve 0-256 for char literals
+  SHR,
+  LE,
+  EQEQ,
+  NE,
+  GE,
+  ANDAND,
+  OROR,
+  SHLEQ,
+  SHREQ,
+  MINUSEQ,
+  ANDEQ,
+  OREQ,
+  PLUSEQ,
+  STAREQ,
+  SLASHEQ,
+  CARETEQ,
+  PERCENTEQ,
+  DOTDOT,
+  DOTDOTDOT,
+  MOD_SEP,
+  RARROW,
+  FAT_ARROW,
+  LIT_BYTE,
+  LIT_CHAR,
+  LIT_INTEGER,
+  LIT_FLOAT,
+  LIT_STR,
+  LIT_STR_RAW,
+  LIT_BINARY,
+  LIT_BINARY_RAW,
+  IDENT,
+  UNDERSCORE,
+  LIFETIME,
+
+  // keywords
+  SELF,
+  STATIC,
+  AS,
+  BREAK,
+  CRATE,
+  ELSE,
+  ENUM,
+  EXTERN,
+  FALSE,
+  FN,
+  FOR,
+  IF,
+  IMPL,
+  IN,
+  LET,
+  LOOP,
+  MATCH,
+  MOD,
+  MOVE,
+  MUT,
+  PRIV,
+  PUB,
+  REF,
+  RETURN,
+  STRUCT,
+  TRUE,
+  TRAIT,
+  TYPE,
+  UNSAFE,
+  USE,
+  WHILE,
+  CONTINUE,
+  PROC,
+  BOX,
+  CONST,
+  WHERE,
+  TYPEOF,
+  INNER_DOC_COMMENT,
+  OUTER_DOC_COMMENT,
+
+  SHEBANG,
+  SHEBANG_LINE,
+  STATIC_LIFETIME
+};