Model lexer: Fix remaining issues

57 files changed, 102 insertions, 179 deletions

diff --git a/src/grammar/README.md b/src/grammar/README.md
index 1f7923e1caf..6e0cf17a880 100644
--- a/src/grammar/README.md
+++ b/src/grammar/README.md
@@ -12,7 +12,7 @@ javac *.java
 rustc -O verify.rs
 for file in ../*/**.rs; do
     echo $file;
-    grun RustLexer tokens -tokens < $file | ./verify $file RustLexer.tokens || break
+    grun RustLexer tokens -tokens < "$file" | ./verify "$file" RustLexer.tokens || break
 done
 ```
 
diff --git a/src/grammar/RustLexer.g4 b/src/grammar/RustLexer.g4
index 8739d135b4f..3d8f3aeb28f 100644
--- a/src/grammar/RustLexer.g4
+++ b/src/grammar/RustLexer.g4
@@ -1,5 +1,12 @@
 lexer grammar RustLexer;
 
+@lexer::members {
+  public boolean is_at(int pos) {
+    return _input.index() == pos;
+  }
+}
+
+
 tokens {
     EQ, LT, LE, EQEQ, NE, GE, GT, ANDAND, OROR, NOT, TILDE, PLUT,
     MINUS, STAR, SLASH, PERCENT, CARET, AND, OR, SHL, SHR, BINOP,
@@ -8,7 +15,7 @@ tokens {
     LBRACE, RBRACE, POUND, DOLLAR, UNDERSCORE, LIT_CHAR,
     LIT_INTEGER, LIT_FLOAT, LIT_STR, LIT_STR_RAW, LIT_BINARY,
     LIT_BINARY_RAW, IDENT, LIFETIME, WHITESPACE, DOC_COMMENT,
-    COMMENT
+    COMMENT, SHEBANG
 }
 
 import xidstart , xidcontinue;
@@ -86,85 +93,54 @@ fragment CHAR_ESCAPE
   | [xX] HEXIT HEXIT
   | 'u' HEXIT HEXIT HEXIT HEXIT
   | 'U' HEXIT HEXIT HEXIT HEXIT HEXIT HEXIT HEXIT HEXIT
+  | 'u{' HEXIT '}'
+  | 'u{' HEXIT HEXIT '}'
+  | 'u{' HEXIT HEXIT HEXIT '}'
+  | 'u{' HEXIT HEXIT HEXIT HEXIT '}'
+  | 'u{' HEXIT HEXIT HEXIT HEXIT HEXIT '}'
+  | 'u{' HEXIT HEXIT HEXIT HEXIT HEXIT HEXIT '}'
   ;
 
 fragment SUFFIX
   : IDENT
   ;
 
+fragment INTEGER_SUFFIX
+  : { _input.LA(1) != 'e' && _input.LA(1) != 'E' }? SUFFIX
+  ;
+
 LIT_CHAR
-  : '\'' ( '\\' CHAR_ESCAPE | ~[\\'\n\t\r] | '\ud800' .. '\udbff' '\udc00' .. '\udfff' ) '\'' SUFFIX?
+  : '\'' ( '\\' CHAR_ESCAPE
+         | ~[\\'\n\t\r]
+         | '\ud800' .. '\udbff' '\udc00' .. '\udfff'
+         )
+    '\'' SUFFIX?
   ;
 
 LIT_BYTE
-  : 'b\'' ( '\\' ( [xX] HEXIT HEXIT | [nrt\\'"0] ) | ~[\\'\n\t\r] ) '\'' SUFFIX?
+  : 'b\'' ( '\\' ( [xX] HEXIT HEXIT
+                 | [nrt\\'"0] )
+          | ~[\\'\n\t\r] '\udc00'..'\udfff'?
+          )
+    '\'' SUFFIX?
   ;
 
 LIT_INTEGER
-  : [0-9][0-9_]* SUFFIX?
-  | '0b' [01][01_]* SUFFIX?
-  | '0o' [0-7][0-7_]* SUFFIX?
-  | '0x' [0-9a-fA-F][0-9a-fA-F_]* SUFFIX?
+
+  : [0-9][0-9_]* INTEGER_SUFFIX?
+  | '0b' [01_]+ INTEGER_SUFFIX?
+  | '0o' [0-7_]+ INTEGER_SUFFIX?
+  | '0x' [0-9a-fA-F_]+ INTEGER_SUFFIX?
   ;
 
 LIT_FLOAT
   : [0-9][0-9_]* ('.' {
-        /* dot followed by another dot is a range, no float */
+        /* dot followed by another dot is a range, not a float */
         _input.LA(1) != '.' &&
-        /* dot followed by an identifier is an integer with a function call, no float */
+        /* dot followed by an identifier is an integer with a function call, not a float */
         _input.LA(1) != '_' &&
-        _input.LA(1) != 'a' &&
-        _input.LA(1) != 'b' &&
-        _input.LA(1) != 'c' &&
-        _input.LA(1) != 'd' &&
-        _input.LA(1) != 'e' &&
-        _input.LA(1) != 'f' &&
-        _input.LA(1) != 'g' &&
-        _input.LA(1) != 'h' &&
-        _input.LA(1) != 'i' &&
-        _input.LA(1) != 'j' &&
-        _input.LA(1) != 'k' &&
-        _input.LA(1) != 'l' &&
-        _input.LA(1) != 'm' &&
-        _input.LA(1) != 'n' &&
-        _input.LA(1) != 'o' &&
-        _input.LA(1) != 'p' &&
-        _input.LA(1) != 'q' &&
-        _input.LA(1) != 'r' &&
-        _input.LA(1) != 's' &&
-        _input.LA(1) != 't' &&
-        _input.LA(1) != 'u' &&
-        _input.LA(1) != 'v' &&
-        _input.LA(1) != 'w' &&
-        _input.LA(1) != 'x' &&
-        _input.LA(1) != 'y' &&
-        _input.LA(1) != 'z' &&
-        _input.LA(1) != 'A' &&
-        _input.LA(1) != 'B' &&
-        _input.LA(1) != 'C' &&
-        _input.LA(1) != 'D' &&
-        _input.LA(1) != 'E' &&
-        _input.LA(1) != 'F' &&
-        _input.LA(1) != 'G' &&
-        _input.LA(1) != 'H' &&
-        _input.LA(1) != 'I' &&
-        _input.LA(1) != 'J' &&
-        _input.LA(1) != 'K' &&
-        _input.LA(1) != 'L' &&
-        _input.LA(1) != 'M' &&
-        _input.LA(1) != 'N' &&
-        _input.LA(1) != 'O' &&
-        _input.LA(1) != 'P' &&
-        _input.LA(1) != 'Q' &&
-        _input.LA(1) != 'R' &&
-        _input.LA(1) != 'S' &&
-        _input.LA(1) != 'T' &&
-        _input.LA(1) != 'U' &&
-        _input.LA(1) != 'V' &&
-        _input.LA(1) != 'W' &&
-        _input.LA(1) != 'X' &&
-        _input.LA(1) != 'Y' &&
-        _input.LA(1) != 'Z'
+        !(_input.LA(1) >= 'a' && _input.LA(1) <= 'z') &&
+        !(_input.LA(1) >= 'A' && _input.LA(1) <= 'Z')
   }? | ('.' [0-9][0-9_]*)? ([eE] [-+]? [0-9][0-9_]*)? SUFFIX?)
   ;
 
@@ -172,8 +148,8 @@ LIT_STR
   : '"' ('\\\n' | '\\\r\n' | '\\' CHAR_ESCAPE | .)*? '"' SUFFIX?
   ;
 
-LIT_BINARY : 'b' LIT_STR SUFFIX?;
-LIT_BINARY_RAW : 'rb' LIT_STR_RAW SUFFIX?;
+LIT_BINARY : 'b' LIT_STR ;
+LIT_BINARY_RAW : 'b' LIT_STR_RAW ;
 
 /* this is a bit messy */
 
@@ -201,13 +177,19 @@ LIFETIME : '\'' IDENT ;
 
 WHITESPACE : [ \r\n\t]+ ;
 
-UNDOC_COMMENT     : '////' ~[\r\n]* -> type(COMMENT) ;
+UNDOC_COMMENT     : '////' ~[\n]* -> type(COMMENT) ;
 YESDOC_COMMENT    : '///' ~[\r\n]* -> type(DOC_COMMENT) ;
 OUTER_DOC_COMMENT : '//!' ~[\r\n]* -> type(DOC_COMMENT) ;
-LINE_COMMENT      : '//' ~[\r\n]* -> type(COMMENT) ;
+LINE_COMMENT      : '//' ( ~[/\n] ~[\n]* )? -> type(COMMENT) ;
 
 DOC_BLOCK_COMMENT
   : ('/**' ~[*] | '/*!') (DOC_BLOCK_COMMENT | .)*? '*/' -> type(DOC_COMMENT)
   ;
 
 BLOCK_COMMENT : '/*' (BLOCK_COMMENT | .)*? '*/' -> type(COMMENT) ;
+
+/* these appear at the beginning of a file */
+
+SHEBANG : '#!' { is_at(2) && _input.LA(1) != '[' }? ~[\r\n]* -> type(SHEBANG) ;
+
+UTF8_BOM : '\ufeff' { is_at(1) }? -> skip ;
diff --git a/src/grammar/check.sh b/src/grammar/check.sh
index b0628303b66..560b6b72471 100755
--- a/src/grammar/check.sh
+++ b/src/grammar/check.sh
@@ -18,13 +18,13 @@ failed=0
 skipped=0
 
 check() {
-    grep --silent "// ignore-lexer-test" $1;
+    grep --silent "// ignore-lexer-test" "$1";
 
     # if it's *not* found...
     if [ $? -eq 1 ]; then
         cd $2 # This `cd` is so java will pick up RustLexer.class. I couldn't
-        # figure out how to wrangle the CLASSPATH, just adding build/grammr didn't
-        # seem to have anny effect.
+        # figure out how to wrangle the CLASSPATH, just adding build/grammar
+        # didn't seem to have any effect.
         if $3 RustLexer tokens -tokens < $1 | $4 $1 $5; then
             echo "pass: $1"
             passed=`expr $passed + 1`
@@ -39,7 +39,7 @@ check() {
 }
 
 for file in $(find $1 -iname '*.rs' ! -path '*/test/compile-fail*'); do
-    check $file $2 $3 $4 $5
+    check "$file" $2 $3 $4 $5
 done
 
 printf "\ntest result: "
diff --git a/src/grammar/verify.rs b/src/grammar/verify.rs
index 8bf501c7f3f..dec797747c2 100644
--- a/src/grammar/verify.rs
+++ b/src/grammar/verify.rs
@@ -8,9 +8,7 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-#![feature(plugin)]
-
-#![allow(unstable)]
+#![feature(plugin, rustc_private, str_char, collections)]
 
 extern crate syntax;
 extern crate rustc;
@@ -19,7 +17,10 @@ extern crate rustc;
 extern crate log;
 
 use std::collections::HashMap;
-use std::io::File;
+use std::env;
+use std::fs::File;
+use std::io::{BufRead, Read};
+use std::path::Path;
 
 use syntax::parse;
 use syntax::parse::lexer;
@@ -27,6 +28,7 @@ use rustc::session::{self, config};
 
 use syntax::ast;
 use syntax::ast::Name;
+use syntax::codemap;
 use syntax::codemap::Pos;
 use syntax::parse::token;
 use syntax::parse::lexer::TokenAndSpan;
@@ -108,6 +110,7 @@ fn parse_token_list(file: &str) -> HashMap<String, token::Token> {
             "LIT_BINARY"        => token::Literal(token::Binary(Name(0)), None),
             "LIT_BINARY_RAW"    => token::Literal(token::BinaryRaw(Name(0), 0), None),
             "QUESTION"          => token::Question,
+            "SHEBANG"           => token::Shebang(Name(0)),
             _                   => continue,
         };
 
@@ -166,24 +169,26 @@ fn count(lit: &str) -> usize {
     lit.chars().take_while(|c| *c == '#').count()
 }
 
-fn parse_antlr_token(s: &str, tokens: &HashMap<String, token::Token>, surrogate_pairs_pos: &[usize])
+fn parse_antlr_token(s: &str, tokens: &HashMap<String, token::Token>, surrogate_pairs_pos: &[usize],
+                     has_bom: bool)
                      -> TokenAndSpan {
     // old regex:
     // \[@(?P<seq>\d+),(?P<start>\d+):(?P<end>\d+)='(?P<content>.+?)',<(?P<toknum>-?\d+)>,\d+:\d+]
-    let start = s.find_str("[@").unwrap();
-    let comma = start + s[start..].find_str(",").unwrap();
-    let colon = comma + s[comma..].find_str(":").unwrap();
-    let content_start = colon + s[colon..].find_str("='").unwrap();
-    let content_end = content_start + s[content_start..].find_str("',<").unwrap();
-    let toknum_end = content_end + s[content_end..].find_str(">,").unwrap();
+    let start = s.find("[@").unwrap();
+    let comma = start + s[start..].find(",").unwrap();
+    let colon = comma + s[comma..].find(":").unwrap();
+    let content_start = colon + s[colon..].find("='").unwrap();
+    // Use rfind instead of find, because we don't want to stop at the content
+    let content_end = content_start + s[content_start..].rfind("',<").unwrap();
+    let toknum_end = content_end + s[content_end..].find(">,").unwrap();
 
     let start = &s[comma + 1 .. colon];
     let end = &s[colon + 1 .. content_start];
     let content = &s[content_start + 2 .. content_end];
     let toknum = &s[content_end + 3 .. toknum_end];
 
-    let proto_tok = tokens.get(toknum).expect(format!("didn't find token {:?} in the map",
-                                                              toknum));
+    let not_found = format!("didn't find token {:?} in the map", toknum);
+    let proto_tok = tokens.get(toknum).expect(&not_found[..]);
 
     let nm = parse::token::intern(content);
 
@@ -209,24 +214,25 @@ fn parse_antlr_token(s: &str, tokens: &HashMap<String, token::Token>, surrogate_
         ref t => t.clone()
     };
 
-    let offset = if real_tok == token::Eof
- {
+    let start_offset = if real_tok == token::Eof {
         1
     } else {
         0
     };
 
-    let mut lo = start.parse::<u32>().unwrap() - offset;
-    let mut hi = end.parse::<u32>().unwrap() + 1;
+    let offset = if has_bom { 1 } else { 0 };
+
+    let mut lo = start.parse::<u32>().unwrap() - start_offset - offset;
+    let mut hi = end.parse::<u32>().unwrap() + 1 - offset;
 
     // Adjust the span: For each surrogate pair already encountered, subtract one position.
     lo -= surrogate_pairs_pos.binary_search(&(lo as usize)).unwrap_or_else(|x| x) as u32;
     hi -= surrogate_pairs_pos.binary_search(&(hi as usize)).unwrap_or_else(|x| x) as u32;
 
-    let sp = syntax::codemap::Span {
-        lo: syntax::codemap::BytePos(lo),
-        hi: syntax::codemap::BytePos(hi),
-        expn_id: syntax::codemap::NO_EXPANSION
+    let sp = codemap::Span {
+        lo: codemap::BytePos(lo),
+        hi: codemap::BytePos(hi),
+        expn_id: codemap::NO_EXPANSION
     };
 
     TokenAndSpan {
@@ -245,10 +251,10 @@ fn tok_cmp(a: &token::Token, b: &token::Token) -> bool {
     }
 }
 
-fn span_cmp(antlr_sp: syntax::codemap::Span, rust_sp: syntax::codemap::Span, cm: &syntax::codemap::CodeMap) -> bool {
+fn span_cmp(antlr_sp: codemap::Span, rust_sp: codemap::Span, cm: &codemap::CodeMap) -> bool {
     antlr_sp.expn_id == rust_sp.expn_id &&
-        antlr_sp.lo.to_uint() == cm.bytepos_to_file_charpos(rust_sp.lo).to_uint() &&
-        antlr_sp.hi.to_uint() == cm.bytepos_to_file_charpos(rust_sp.hi).to_uint()
+        antlr_sp.lo.to_usize() == cm.bytepos_to_file_charpos(rust_sp.lo).to_usize() &&
+        antlr_sp.hi.to_usize() == cm.bytepos_to_file_charpos(rust_sp.hi).to_usize()
 }
 
 fn main() {
@@ -257,10 +263,15 @@ fn main() {
         r.next_token()
     }
 
-    let args = std::os::args();
+    let mut args = env::args().skip(1);
+    let filename = args.next().unwrap();
+    if filename.find("parse-fail").is_some() {
+        return;
+    }
 
     // Rust's lexer
-    let code = File::open(&Path::new(args[1])).unwrap().read_to_string().unwrap();
+    let mut code = String::new();
+    File::open(&Path::new(&filename)).unwrap().read_to_string(&mut code).unwrap();
 
     let surrogate_pairs_pos: Vec<usize> = code.chars().enumerate()
                                                      .filter(|&(_, c)| c as usize > 0xFFFF)
@@ -269,6 +280,8 @@ fn main() {
                                                      .map(|(x, n)| x + n)
                                                      .collect();
 
+    let has_bom = code.starts_with("\u{feff}");
+
     debug!("Pairs: {:?}", surrogate_pairs_pos);
 
     let options = config::basic_options();
@@ -281,15 +294,18 @@ fn main() {
     let ref cm = lexer.span_diagnostic.cm;
 
     // ANTLR
-    let mut token_file = File::open(&Path::new(args[2]));
-    let token_map = parse_token_list(token_file.read_to_string().unwrap());
+    let mut token_file = File::open(&Path::new(&args.next().unwrap())).unwrap();
+    let mut token_list = String::new();
+    token_file.read_to_string(&mut token_list).unwrap();
+    let token_map = parse_token_list(&token_list[..]);
 
-    let mut stdin = std::io::stdin();
-    let mut lock = stdin.lock();
+    let stdin = std::io::stdin();
+    let lock = stdin.lock();
     let lines = lock.lines();
-    let mut antlr_tokens = lines.map(|l| parse_antlr_token(l.unwrap().trim(),
-                                                           &token_map,
-                                                           &surrogate_pairs_pos[]));
+    let antlr_tokens = lines.map(|l| parse_antlr_token(l.unwrap().trim(),
+                                                       &token_map,
+                                                       &surrogate_pairs_pos[..],
+                                                       has_bom));
 
     for antlr_tok in antlr_tokens {
         let rustc_tok = next(&mut lexer);
@@ -314,7 +330,7 @@ fn main() {
                         }
                         _ => panic!("{:?} is not {:?}", antlr_tok, rustc_tok)
                     },)*
-                    ref c => assert!(c == &antlr_tok.tok, "{:?} is not {:?}", rustc_tok, antlr_tok)
+                    ref c => assert!(c == &antlr_tok.tok, "{:?} is not {:?}", antlr_tok, rustc_tok)
                 }
             )
         }
diff --git a/src/libcollections/fmt.rs b/src/libcollections/fmt.rs
index 5f0d9012d1a..40b64b5c3b4 100644
--- a/src/libcollections/fmt.rs
+++ b/src/libcollections/fmt.rs
@@ -7,8 +7,6 @@
 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
-//
-// ignore-lexer-test FIXME #15679
 
 //! Utilities for formatting and printing strings
 //!
diff --git a/src/libcollections/str.rs b/src/libcollections/str.rs
index 0f902e258b9..266cda9a237 100644
--- a/src/libcollections/str.rs
+++ b/src/libcollections/str.rs
@@ -7,8 +7,6 @@
 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
-//
-// ignore-lexer-test FIXME #15679
 
 //! Unicode string manipulation (the `str` type).
 //!
diff --git a/src/libcollections/string.rs b/src/libcollections/string.rs
index 74af5783fa8..3422bfe5423 100644
--- a/src/libcollections/string.rs
+++ b/src/libcollections/string.rs
@@ -7,8 +7,6 @@
 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
-//
-// ignore-lexer-test FIXME #15679
 
 //! An owned, growable string that enforces that its contents are valid UTF-8.
 
diff --git a/src/libcore/hash/sip.rs b/src/libcore/hash/sip.rs
index 6820a7025fc..65f790d5d43 100644
--- a/src/libcore/hash/sip.rs
+++ b/src/libcore/hash/sip.rs
@@ -7,8 +7,6 @@
 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
-//
-// ignore-lexer-test FIXME #15883
 
 //! An implementation of SipHash 2-4.
 
diff --git a/src/libcore/num/mod.rs b/src/libcore/num/mod.rs
index a056e585fee..bcfdcfcd5e6 100644
--- a/src/libcore/num/mod.rs
+++ b/src/libcore/num/mod.rs
@@ -7,8 +7,6 @@
 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
-//
-// ignore-lexer-test FIXME #15679
 
 //! Numeric traits and functions for the built-in numeric types.
 
diff --git a/src/libcore/str/mod.rs b/src/libcore/str/mod.rs
index 2d6ef39361e..34810b4864e 100644
--- a/src/libcore/str/mod.rs
+++ b/src/libcore/str/mod.rs
@@ -7,8 +7,6 @@
 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
-//
-// ignore-lexer-test FIXME #15679
 
 //! String manipulation
 //!
diff --git a/src/libcoretest/char.rs b/src/libcoretest/char.rs
index 4939277aa59..b73807aa317 100644
--- a/src/libcoretest/char.rs
+++ b/src/libcoretest/char.rs
@@ -7,8 +7,6 @@
 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
-//
-// ignore-lexer-test FIXME #15679
 
 #[test]
 fn test_is_lowercase() {
diff --git a/src/libgetopts/lib.rs b/src/libgetopts/lib.rs
index 02c4a233996..197199e743f 100644
--- a/src/libgetopts/lib.rs
+++ b/src/libgetopts/lib.rs
@@ -7,8 +7,6 @@
 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
-//
-// ignore-lexer-test FIXME #15677
 
 //! Simple getopt alternative.
 //!
diff --git a/src/librand/distributions/gamma.rs b/src/librand/distributions/gamma.rs
index 1125d096536..f37093c6db8 100644
--- a/src/librand/distributions/gamma.rs
+++ b/src/librand/distributions/gamma.rs
@@ -7,8 +7,6 @@
 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
-//
-// ignore-lexer-test FIXME #15679
 
 //! The Gamma and derived distributions.
 
diff --git a/src/librustc_unicode/u_str.rs b/src/librustc_unicode/u_str.rs
index 09a5feb5fef..c63c586b6a9 100644
--- a/src/librustc_unicode/u_str.rs
+++ b/src/librustc_unicode/u_str.rs
@@ -7,8 +7,6 @@
 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
-//
-// ignore-lexer-test FIXME #15679
 
 //! Unicode-intensive string manipulations.
 //!
diff --git a/src/libserialize/hex.rs b/src/libserialize/hex.rs
index 0676edf8169..87f1dca2cae 100644
--- a/src/libserialize/hex.rs
+++ b/src/libserialize/hex.rs
@@ -7,8 +7,6 @@
 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
-//
-// ignore-lexer-test FIXME #15679
 
 //! Hex binary-to-text encoding
 
diff --git a/src/libstd/ascii.rs b/src/libstd/ascii.rs
index a2ba8c4c1ba..ccc56960b02 100644
--- a/src/libstd/ascii.rs
+++ b/src/libstd/ascii.rs
@@ -7,8 +7,6 @@
 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
-//
-// ignore-lexer-test FIXME #15679
 
 //! Operations on ASCII strings and characters
 
diff --git a/src/libstd/collections/hash/map.rs b/src/libstd/collections/hash/map.rs
index 4ac15b7991b..a5bbbee790a 100644
--- a/src/libstd/collections/hash/map.rs
+++ b/src/libstd/collections/hash/map.rs
@@ -7,8 +7,6 @@
 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
-//
-// ignore-lexer-test FIXME #15883
 
 use self::Entry::*;
 use self::SearchResult::*;
diff --git a/src/libstd/collections/hash/set.rs b/src/libstd/collections/hash/set.rs
index 62c03389b24..82109900bf2 100644
--- a/src/libstd/collections/hash/set.rs
+++ b/src/libstd/collections/hash/set.rs
@@ -7,8 +7,6 @@
 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
-//
-// ignore-lexer-test FIXME #15883
 
 use borrow::Borrow;
 use clone::Clone;
diff --git a/src/libstd/collections/hash/table.rs b/src/libstd/collections/hash/table.rs
index dec6d1e2209..65ebf8515e6 100644
--- a/src/libstd/collections/hash/table.rs
+++ b/src/libstd/collections/hash/table.rs
@@ -7,8 +7,6 @@
 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
-//
-// ignore-lexer-test FIXME #15883
 
 use self::BucketState::*;
 
diff --git a/src/libstd/io/buffered.rs b/src/libstd/io/buffered.rs
index bd44a9547b4..67cac42c35e 100644
--- a/src/libstd/io/buffered.rs
+++ b/src/libstd/io/buffered.rs
@@ -7,8 +7,6 @@
 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
-//
-// ignore-lexer-test FIXME #15883
 
 //! Buffering wrappers for I/O traits
 
diff --git a/src/libstd/num/strconv.rs b/src/libstd/num/strconv.rs
index 8ab66f2328f..ce1da4742d1 100644
--- a/src/libstd/num/strconv.rs
+++ b/src/libstd/num/strconv.rs
@@ -7,8 +7,6 @@
 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
-//
-// ignore-lexer-test FIXME #15679
 
 #![allow(missing_docs)]
 #![allow(deprecated)]
diff --git a/src/libstd/rt/util.rs b/src/libstd/rt/util.rs
index 9919238c208..31e970a9550 100644
--- a/src/libstd/rt/util.rs
+++ b/src/libstd/rt/util.rs
@@ -7,8 +7,6 @@
 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
-//
-// ignore-lexer-test FIXME #15677
 
 use io::prelude::*;
 
diff --git a/src/libsyntax/codemap.rs b/src/libsyntax/codemap.rs
index a0bde8f6c52..dfdaa47d8b9 100644
--- a/src/libsyntax/codemap.rs
+++ b/src/libsyntax/codemap.rs
@@ -7,8 +7,6 @@
 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
-//
-// ignore-lexer-test FIXME #15679
 
 //! The CodeMap tracks all the source code used within a single crate, mapping
 //! from integer byte positions to the original source code location. Each bit
diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs
index 9c3a556b210..58df4038403 100644
--- a/src/libsyntax/ext/tt/macro_parser.rs
+++ b/src/libsyntax/ext/tt/macro_parser.rs
@@ -7,8 +7,6 @@
 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
-//
-// ignore-lexer-test FIXME #15679
 
 //! This is an Earley-like parser, without support for in-grammar nonterminals,
 //! only by calling out to the main rust parser for named nonterminals (which it
diff --git a/src/test/bench/core-std.rs b/src/test/bench/core-std.rs
index 46caed6f9f5..19f83c7817c 100644
--- a/src/test/bench/core-std.rs
+++ b/src/test/bench/core-std.rs
@@ -8,7 +8,6 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-// ignore-lexer-test FIXME #15679
 // Microbenchmarks for various functions in std and extra
 
 #![feature(rand, collections, std_misc)]
diff --git a/src/test/bench/msgsend-ring-mutex-arcs.rs b/src/test/bench/msgsend-ring-mutex-arcs.rs
index c87cdb617a4..8048f3dde96 100644
--- a/src/test/bench/msgsend-ring-mutex-arcs.rs
+++ b/src/test/bench/msgsend-ring-mutex-arcs.rs
@@ -16,7 +16,6 @@
 // This also serves as a pipes test, because Arcs are implemented with pipes.
 
 // no-pretty-expanded FIXME #15189
-// ignore-lexer-test FIXME #15679
 
 #![feature(std_misc)]
 
diff --git a/src/test/bench/noise.rs b/src/test/bench/noise.rs
index c21470d4bb3..530c499f5fd 100644
--- a/src/test/bench/noise.rs
+++ b/src/test/bench/noise.rs
@@ -10,7 +10,6 @@
 
 // Multi-language Perlin noise benchmark.
 // See https://github.com/nsf/pnoise for timings and alternative implementations.
-// ignore-lexer-test FIXME #15679
 
 #![feature(rand, core)]
 
diff --git a/src/test/compile-fail/utf8_idents.rs b/src/test/compile-fail/utf8_idents.rs
index a5471e87f22..8594c35f8dd 100644
--- a/src/test/compile-fail/utf8_idents.rs
+++ b/src/test/compile-fail/utf8_idents.rs
@@ -8,7 +8,6 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 //
-// ignore-lexer-test FIXME #15679
 
 fn foo<
     'β, //~ ERROR non-ascii idents are not fully supported
diff --git a/src/test/pretty/block-comment-wchar.pp b/src/test/pretty/block-comment-wchar.pp
index a5d82277d2f..2dc7e8f9525 100644
--- a/src/test/pretty/block-comment-wchar.pp
+++ b/src/test/pretty/block-comment-wchar.pp
@@ -14,7 +14,6 @@
 // ignore-tidy-cr
 // ignore-tidy-tab
 // pp-exact:block-comment-wchar.pp
-// ignore-lexer-test FIXME #15679
 fn f() {
     fn nested() {
         /*
diff --git a/src/test/pretty/block-comment-wchar.rs b/src/test/pretty/block-comment-wchar.rs
index eb6d2a4a0a1..6f4a95e7c9b 100644
--- a/src/test/pretty/block-comment-wchar.rs
+++ b/src/test/pretty/block-comment-wchar.rs
@@ -14,7 +14,6 @@
 // ignore-tidy-cr
 // ignore-tidy-tab
 // pp-exact:block-comment-wchar.pp
-// ignore-lexer-test FIXME #15679
 fn f() {
     fn nested() {
         /*
diff --git a/src/test/run-pass/byte-literals.rs b/src/test/run-pass/byte-literals.rs
index fbe2a65bc89..9f7b98a57fc 100644
--- a/src/test/run-pass/byte-literals.rs
+++ b/src/test/run-pass/byte-literals.rs
@@ -8,7 +8,6 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 //
-// ignore-lexer-test FIXME #15883
 
 
 static FOO: u8 = b'\xF0';
diff --git a/src/test/run-pass/default-method-supertrait-vtable.rs b/src/test/run-pass/default-method-supertrait-vtable.rs
index 3b1e04be78d..0d45a5d5212 100644
--- a/src/test/run-pass/default-method-supertrait-vtable.rs
+++ b/src/test/run-pass/default-method-supertrait-vtable.rs
@@ -8,7 +8,6 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 //
-// ignore-lexer-test FIXME #15877
 
 
 // Tests that we can call a function bounded over a supertrait from
diff --git a/src/test/run-pass/ifmt.rs b/src/test/run-pass/ifmt.rs
index ea9db9b1e1f..7ae1347f2c7 100644
--- a/src/test/run-pass/ifmt.rs
+++ b/src/test/run-pass/ifmt.rs
@@ -9,7 +9,6 @@
 // except according to those terms.
 
 // no-pretty-expanded unnecessary unsafe block generated
-// ignore-lexer-test FIXME #15679
 
 #![deny(warnings)]
 #![allow(unused_must_use)]
diff --git a/src/test/run-pass/issue-12582.rs b/src/test/run-pass/issue-12582.rs
index 4009d17139d..7bab2ddfed0 100644
--- a/src/test/run-pass/issue-12582.rs
+++ b/src/test/run-pass/issue-12582.rs
@@ -8,7 +8,6 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 //
-// ignore-lexer-test FIXME #15877
 
 pub fn main() {
     let x = 1;
diff --git a/src/test/run-pass/issue-13027.rs b/src/test/run-pass/issue-13027.rs
index dadd480dc6a..14987484711 100644
--- a/src/test/run-pass/issue-13027.rs
+++ b/src/test/run-pass/issue-13027.rs
@@ -8,7 +8,6 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 //
-// ignore-lexer-test FIXME #15877
 
 // Tests that match expression handles overlapped literal and range
 // properly in the presence of guard function.
diff --git a/src/test/run-pass/issue-2718.rs b/src/test/run-pass/issue-2718.rs
index 71d1439dd2b..0df89c72424 100644
--- a/src/test/run-pass/issue-2718.rs
+++ b/src/test/run-pass/issue-2718.rs
@@ -8,7 +8,6 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 //
-// ignore-lexer-test FIXME #15883
 
 #![feature(unsafe_destructor, std_misc)]
 
diff --git a/src/test/run-pass/issue-3683.rs b/src/test/run-pass/issue-3683.rs
index 096eec803ff..ed9b8066104 100644
--- a/src/test/run-pass/issue-3683.rs
+++ b/src/test/run-pass/issue-3683.rs
@@ -8,7 +8,6 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 //
-// ignore-lexer-test FIXME #15877
 
 
 trait Foo {
diff --git a/src/test/run-pass/issue-4759-1.rs b/src/test/run-pass/issue-4759-1.rs
index 3532a395b7a..a565460c42e 100644
--- a/src/test/run-pass/issue-4759-1.rs
+++ b/src/test/run-pass/issue-4759-1.rs
@@ -8,8 +8,6 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-// pretty-expanded FIXME #23616
-
 trait U { fn f(self); }
 impl U for isize { fn f(self) {} }
 pub fn main() { 4.f(); }
diff --git a/src/test/run-pass/issue-5280.rs b/src/test/run-pass/issue-5280.rs
index bd892465054..5e2e4df95b3 100644
--- a/src/test/run-pass/issue-5280.rs
+++ b/src/test/run-pass/issue-5280.rs
@@ -8,7 +8,6 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 //
-// ignore-lexer-test FIXME #15877
 
 type FontTableTag = u32;
 
diff --git a/src/test/run-pass/issue-5321-immediates-with-bare-self.rs b/src/test/run-pass/issue-5321-immediates-with-bare-self.rs
index d0bc396c368..dd00fab5020 100644
--- a/src/test/run-pass/issue-5321-immediates-with-bare-self.rs
+++ b/src/test/run-pass/issue-5321-immediates-with-bare-self.rs
@@ -8,7 +8,6 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 //
-// ignore-lexer-test FIXME #15877
 
 trait Fooable {
     fn yes(self);
diff --git a/src/test/run-pass/lexer-crlf-line-endings-string-literal-doc-comment.rs b/src/test/run-pass/lexer-crlf-line-endings-string-literal-doc-comment.rs
index 421ae8e9497..5c8db524cc2 100644
--- a/src/test/run-pass/lexer-crlf-line-endings-string-literal-doc-comment.rs
+++ b/src/test/run-pass/lexer-crlf-line-endings-string-literal-doc-comment.rs
@@ -16,7 +16,6 @@
 // this directory should enforce it.

 

 // ignore-pretty

-// ignore-lexer-test FIXME #15882

 

 /// Doc comment that ends in CRLF

 pub fn foo() {}

diff --git a/src/test/run-pass/lint-non-camel-case-types-non-uppercase-statics-unicode.rs b/src/test/run-pass/lint-non-camel-case-types-non-uppercase-statics-unicode.rs
index 6ddaee9c8bd..6e65cb2afd4 100644
--- a/src/test/run-pass/lint-non-camel-case-types-non-uppercase-statics-unicode.rs
+++ b/src/test/run-pass/lint-non-camel-case-types-non-uppercase-statics-unicode.rs
@@ -8,7 +8,6 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 //
-// ignore-lexer-test FIXME #15679
 
 
 #![forbid(non_camel_case_types)]
diff --git a/src/test/run-pass/match-range.rs b/src/test/run-pass/match-range.rs
index 68719090cff..0b2e19d6c79 100644
--- a/src/test/run-pass/match-range.rs
+++ b/src/test/run-pass/match-range.rs
@@ -8,7 +8,6 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 //
-// ignore-lexer-test FIXME #15877
 
 pub fn main() {
     match 5_usize {
diff --git a/src/test/run-pass/multibyte.rs b/src/test/run-pass/multibyte.rs
index 77084836408..0475dd10fde 100644
--- a/src/test/run-pass/multibyte.rs
+++ b/src/test/run-pass/multibyte.rs
@@ -8,7 +8,6 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 //
-// ignore-lexer-test FIXME #15679
 
 // Test that multibyte characters don't crash the compiler
 pub fn main() {
diff --git a/src/test/run-pass/raw-str.rs b/src/test/run-pass/raw-str.rs
index 298ac8f77eb..9ee824d4185 100644
--- a/src/test/run-pass/raw-str.rs
+++ b/src/test/run-pass/raw-str.rs
Binary files differdiff --git a/src/test/run-pass/shebang.rs b/src/test/run-pass/shebang.rs
index 87da814771b..15ab21bbc8d 100644
--- a/src/test/run-pass/shebang.rs
+++ b/src/test/run-pass/shebang.rs
@@ -11,6 +11,5 @@
 
 // ignore-pretty: `expand` adds some preludes before shebang
 //
-// ignore-lexer-test FIXME #15878
 
 pub fn main() { println!("Hello World"); }
diff --git a/src/test/run-pass/struct-return.rs b/src/test/run-pass/struct-return.rs
index 1ff13d4eaea..109287a83b1 100644
--- a/src/test/run-pass/struct-return.rs
+++ b/src/test/run-pass/struct-return.rs
@@ -8,7 +8,6 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 //
-// ignore-lexer-test FIXME #15883
 
 #[derive(Copy, Clone)]
 pub struct Quad { a: u64, b: u64, c: u64, d: u64 }
diff --git a/src/test/run-pass/trait-to-str.rs b/src/test/run-pass/trait-to-str.rs
index 3d84092c062..a29e0e932c0 100644
--- a/src/test/run-pass/trait-to-str.rs
+++ b/src/test/run-pass/trait-to-str.rs
@@ -8,7 +8,6 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 //
-// ignore-lexer-test FIXME #15883
 
 
 trait to_str {
diff --git a/src/test/run-pass/trait-with-bounds-default.rs b/src/test/run-pass/trait-with-bounds-default.rs
index 34a79c4cf31..cfd81240094 100644
--- a/src/test/run-pass/trait-with-bounds-default.rs
+++ b/src/test/run-pass/trait-with-bounds-default.rs
@@ -8,7 +8,6 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 //
-// ignore-lexer-test FIXME #15877
 
 pub trait Clone2 {
     /// Returns a copy of the value. The contents of owned pointers
diff --git a/src/test/run-pass/traits-default-method-self.rs b/src/test/run-pass/traits-default-method-self.rs
index d9536108f4d..36b0eb527b6 100644
--- a/src/test/run-pass/traits-default-method-self.rs
+++ b/src/test/run-pass/traits-default-method-self.rs
@@ -8,7 +8,6 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 //
-// ignore-lexer-test FIXME #15877
 
 
 trait Cat {
diff --git a/src/test/run-pass/traits-default-method-trivial.rs b/src/test/run-pass/traits-default-method-trivial.rs
index 0e71fcab9d1..a2e7f54bba6 100644
--- a/src/test/run-pass/traits-default-method-trivial.rs
+++ b/src/test/run-pass/traits-default-method-trivial.rs
@@ -8,7 +8,6 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 //
-// ignore-lexer-test FIXME #15877
 
 
 trait Cat {
diff --git a/src/test/run-pass/unsized.rs b/src/test/run-pass/unsized.rs
index 449d6b37e9f..26f7b767988 100644
--- a/src/test/run-pass/unsized.rs
+++ b/src/test/run-pass/unsized.rs
@@ -10,8 +10,6 @@
 
 // Test syntax checks for `?Sized` syntax.
 
-// pretty-expanded FIXME #23616
-
 use std::marker::PhantomData;
 
 trait T1  { }
diff --git a/src/test/run-pass/unsized2.rs b/src/test/run-pass/unsized2.rs
index 965ce6bad16..1cce98ae6b7 100644
--- a/src/test/run-pass/unsized2.rs
+++ b/src/test/run-pass/unsized2.rs
@@ -8,8 +8,6 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-// pretty-expanded FIXME #23616
-
 #![allow(unknown_features)]
 #![feature(box_syntax)]
 
diff --git a/src/test/run-pass/utf8-bom.rs b/src/test/run-pass/utf8-bom.rs
index baa4e941ff0..c3052a928d6 100644
--- a/src/test/run-pass/utf8-bom.rs
+++ b/src/test/run-pass/utf8-bom.rs
@@ -8,7 +8,6 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 //
-// ignore-lexer-test FIXME #15679
 
 // This file has utf-8 BOM, it should be compiled normally without error.
 
diff --git a/src/test/run-pass/utf8.rs b/src/test/run-pass/utf8.rs
index 07fd7b297b4..4782edf4e12 100644
--- a/src/test/run-pass/utf8.rs
+++ b/src/test/run-pass/utf8.rs
@@ -8,7 +8,6 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 //
-// ignore-lexer-test FIXME #15679
 // no-pretty-expanded FIXME #15189
 
 pub fn main() {
diff --git a/src/test/run-pass/utf8_chars.rs b/src/test/run-pass/utf8_chars.rs
index 45a3f2327aa..36b64551ef2 100644
--- a/src/test/run-pass/utf8_chars.rs
+++ b/src/test/run-pass/utf8_chars.rs
@@ -8,7 +8,6 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 //
-// ignore-lexer-test FIXME #15679
 
 #![feature(collections, core, str_char)]
 
diff --git a/src/test/run-pass/utf8_idents.rs b/src/test/run-pass/utf8_idents.rs
index b11b7e83eb6..559afcd1641 100644
--- a/src/test/run-pass/utf8_idents.rs
+++ b/src/test/run-pass/utf8_idents.rs
@@ -8,7 +8,6 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 //
-// ignore-lexer-test FIXME #15679
 
 #![feature(non_ascii_idents)]