Merge remote-tracking branch 'rust-lang/master' into hermit

6 files changed, 0 insertions, 1900 deletions

diff --git a/src/libsyntax/parse/lexer/comments.rs b/src/libsyntax/parse/lexer/comments.rs
deleted file mode 100644
index ac79ce323bf..00000000000
--- a/src/libsyntax/parse/lexer/comments.rs
+++ /dev/null
@@ -1,254 +0,0 @@
-pub use CommentStyle::*;
-
-use crate::ast;
-use crate::source_map::SourceMap;
-use crate::parse::lexer::is_block_doc_comment;
-use crate::parse::lexer::ParseSess;
-
-use syntax_pos::{BytePos, CharPos, Pos, FileName};
-
-use std::usize;
-
-#[cfg(test)]
-mod tests;
-
-#[derive(Clone, Copy, PartialEq, Debug)]
-pub enum CommentStyle {
-    /// No code on either side of each line of the comment
-    Isolated,
-    /// Code exists to the left of the comment
-    Trailing,
-    /// Code before /* foo */ and after the comment
-    Mixed,
-    /// Just a manual blank line "\n\n", for layout
-    BlankLine,
-}
-
-#[derive(Clone)]
-pub struct Comment {
-    pub style: CommentStyle,
-    pub lines: Vec<String>,
-    pub pos: BytePos,
-}
-
-fn is_doc_comment(s: &str) -> bool {
-    (s.starts_with("///") && super::is_doc_comment(s)) || s.starts_with("//!") ||
-    (s.starts_with("/**") && is_block_doc_comment(s)) || s.starts_with("/*!")
-}
-
-pub fn doc_comment_style(comment: &str) -> ast::AttrStyle {
-    assert!(is_doc_comment(comment));
-    if comment.starts_with("//!") || comment.starts_with("/*!") {
-        ast::AttrStyle::Inner
-    } else {
-        ast::AttrStyle::Outer
-    }
-}
-
-pub fn strip_doc_comment_decoration(comment: &str) -> String {
-    /// remove whitespace-only lines from the start/end of lines
-    fn vertical_trim(lines: Vec<String>) -> Vec<String> {
-        let mut i = 0;
-        let mut j = lines.len();
-        // first line of all-stars should be omitted
-        if !lines.is_empty() && lines[0].chars().all(|c| c == '*') {
-            i += 1;
-        }
-
-        while i < j && lines[i].trim().is_empty() {
-            i += 1;
-        }
-        // like the first, a last line of all stars should be omitted
-        if j > i &&
-           lines[j - 1]
-               .chars()
-               .skip(1)
-               .all(|c| c == '*') {
-            j -= 1;
-        }
-
-        while j > i && lines[j - 1].trim().is_empty() {
-            j -= 1;
-        }
-
-        lines[i..j].to_vec()
-    }
-
-    /// remove a "[ \t]*\*" block from each line, if possible
-    fn horizontal_trim(lines: Vec<String>) -> Vec<String> {
-        let mut i = usize::MAX;
-        let mut can_trim = true;
-        let mut first = true;
-
-        for line in &lines {
-            for (j, c) in line.chars().enumerate() {
-                if j > i || !"* \t".contains(c) {
-                    can_trim = false;
-                    break;
-                }
-                if c == '*' {
-                    if first {
-                        i = j;
-                        first = false;
-                    } else if i != j {
-                        can_trim = false;
-                    }
-                    break;
-                }
-            }
-            if i >= line.len() {
-                can_trim = false;
-            }
-            if !can_trim {
-                break;
-            }
-        }
-
-        if can_trim {
-            lines.iter()
-                 .map(|line| (&line[i + 1..line.len()]).to_string())
-                 .collect()
-        } else {
-            lines
-        }
-    }
-
-    // one-line comments lose their prefix
-    const ONELINERS: &[&str] = &["///!", "///", "//!", "//"];
-
-    for prefix in ONELINERS {
-        if comment.starts_with(*prefix) {
-            return (&comment[prefix.len()..]).to_string();
-        }
-    }
-
-    if comment.starts_with("/*") {
-        let lines = comment[3..comment.len() - 2]
-                        .lines()
-                        .map(|s| s.to_string())
-                        .collect::<Vec<String>>();
-
-        let lines = vertical_trim(lines);
-        let lines = horizontal_trim(lines);
-
-        return lines.join("\n");
-    }
-
-    panic!("not a doc-comment: {}", comment);
-}
-
-/// Returns `None` if the first `col` chars of `s` contain a non-whitespace char.
-/// Otherwise returns `Some(k)` where `k` is first char offset after that leading
-/// whitespace. Note that `k` may be outside bounds of `s`.
-fn all_whitespace(s: &str, col: CharPos) -> Option<usize> {
-    let mut idx = 0;
-    for (i, ch) in s.char_indices().take(col.to_usize()) {
-        if !ch.is_whitespace() {
-            return None;
-        }
-        idx = i + ch.len_utf8();
-    }
-    Some(idx)
-}
-
-fn trim_whitespace_prefix(s: &str, col: CharPos) -> &str {
-    let len = s.len();
-    match all_whitespace(&s, col) {
-        Some(col) => if col < len { &s[col..] } else { "" },
-        None => s,
-    }
-}
-
-fn split_block_comment_into_lines(
-    text: &str,
-    col: CharPos,
-) -> Vec<String> {
-    let mut res: Vec<String> = vec![];
-    let mut lines = text.lines();
-    // just push the first line
-    res.extend(lines.next().map(|it| it.to_string()));
-    // for other lines, strip common whitespace prefix
-    for line in lines {
-        res.push(trim_whitespace_prefix(line, col).to_string())
-    }
-    res
-}
-
-// it appears this function is called only from pprust... that's
-// probably not a good thing.
-crate fn gather_comments(sess: &ParseSess, path: FileName, src: String) -> Vec<Comment> {
-    let cm = SourceMap::new(sess.source_map().path_mapping().clone());
-    let source_file = cm.new_source_file(path, src);
-    let text = (*source_file.src.as_ref().unwrap()).clone();
-
-    let text: &str = text.as_str();
-    let start_bpos = source_file.start_pos;
-    let mut pos = 0;
-    let mut comments: Vec<Comment> = Vec::new();
-    let mut code_to_the_left = false;
-
-    if let Some(shebang_len) = rustc_lexer::strip_shebang(text) {
-        comments.push(Comment {
-            style: Isolated,
-            lines: vec![text[..shebang_len].to_string()],
-            pos: start_bpos,
-        });
-        pos += shebang_len;
-    }
-
-    for token in rustc_lexer::tokenize(&text[pos..]) {
-        let token_text = &text[pos..pos + token.len];
-        match token.kind {
-            rustc_lexer::TokenKind::Whitespace => {
-                if let Some(mut idx) = token_text.find('\n') {
-                    code_to_the_left = false;
-                    while let Some(next_newline) = &token_text[idx + 1..].find('\n') {
-                        idx = idx + 1 + next_newline;
-                        comments.push(Comment {
-                            style: BlankLine,
-                            lines: vec![],
-                            pos: start_bpos + BytePos((pos + idx) as u32),
-                        });
-                    }
-                }
-            }
-            rustc_lexer::TokenKind::BlockComment { terminated: _ } => {
-                if !is_block_doc_comment(token_text) {
-                    let code_to_the_right = match text[pos + token.len..].chars().next() {
-                        Some('\r') | Some('\n') => false,
-                        _ => true,
-                    };
-                    let style = match (code_to_the_left, code_to_the_right) {
-                        (true, true) | (false, true) => Mixed,
-                        (false, false) => Isolated,
-                        (true, false) => Trailing,
-                    };
-
-                    // Count the number of chars since the start of the line by rescanning.
-                    let pos_in_file = start_bpos + BytePos(pos as u32);
-                    let line_begin_in_file = source_file.line_begin_pos(pos_in_file);
-                    let line_begin_pos = (line_begin_in_file - start_bpos).to_usize();
-                    let col = CharPos(text[line_begin_pos..pos].chars().count());
-
-                    let lines = split_block_comment_into_lines(token_text, col);
-                    comments.push(Comment { style, lines, pos: pos_in_file })
-                }
-            }
-            rustc_lexer::TokenKind::LineComment => {
-                if !is_doc_comment(token_text) {
-                    comments.push(Comment {
-                        style: if code_to_the_left { Trailing } else { Isolated },
-                        lines: vec![token_text.to_string()],
-                        pos: start_bpos + BytePos(pos as u32),
-                    })
-                }
-            }
-            _ => {
-                code_to_the_left = true;
-            }
-        }
-        pos += token.len;
-    }
-
-    comments
-}
diff --git a/src/libsyntax/parse/lexer/comments/tests.rs b/src/libsyntax/parse/lexer/comments/tests.rs
deleted file mode 100644
index f9cd69fb50d..00000000000
--- a/src/libsyntax/parse/lexer/comments/tests.rs
+++ /dev/null
@@ -1,47 +0,0 @@
-use super::*;
-
-#[test]
-fn test_block_doc_comment_1() {
-    let comment = "/**\n * Test \n **  Test\n *   Test\n*/";
-    let stripped = strip_doc_comment_decoration(comment);
-    assert_eq!(stripped, " Test \n*  Test\n   Test");
-}
-
-#[test]
-fn test_block_doc_comment_2() {
-    let comment = "/**\n * Test\n *  Test\n*/";
-    let stripped = strip_doc_comment_decoration(comment);
-    assert_eq!(stripped, " Test\n  Test");
-}
-
-#[test]
-fn test_block_doc_comment_3() {
-    let comment = "/**\n let a: *i32;\n *a = 5;\n*/";
-    let stripped = strip_doc_comment_decoration(comment);
-    assert_eq!(stripped, " let a: *i32;\n *a = 5;");
-}
-
-#[test]
-fn test_block_doc_comment_4() {
-    let comment = "/*******************\n test\n *********************/";
-    let stripped = strip_doc_comment_decoration(comment);
-    assert_eq!(stripped, " test");
-}
-
-#[test]
-fn test_line_doc_comment() {
-    let stripped = strip_doc_comment_decoration("/// test");
-    assert_eq!(stripped, " test");
-    let stripped = strip_doc_comment_decoration("///! test");
-    assert_eq!(stripped, " test");
-    let stripped = strip_doc_comment_decoration("// test");
-    assert_eq!(stripped, " test");
-    let stripped = strip_doc_comment_decoration("// test");
-    assert_eq!(stripped, " test");
-    let stripped = strip_doc_comment_decoration("///test");
-    assert_eq!(stripped, "test");
-    let stripped = strip_doc_comment_decoration("///!test");
-    assert_eq!(stripped, "test");
-    let stripped = strip_doc_comment_decoration("//test");
-    assert_eq!(stripped, "test");
-}
diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs
deleted file mode 100644
index 48cf117f65a..00000000000
--- a/src/libsyntax/parse/lexer/mod.rs
+++ /dev/null
@@ -1,659 +0,0 @@
-use crate::parse::token::{self, Token, TokenKind};
-use crate::sess::ParseSess;
-use crate::symbol::{sym, Symbol};
-use crate::parse::unescape_error_reporting::{emit_unescape_error, push_escaped_char};
-
-use errors::{FatalError, DiagnosticBuilder};
-use syntax_pos::{BytePos, Pos, Span};
-use rustc_lexer::Base;
-use rustc_lexer::unescape;
-
-use std::char;
-use std::convert::TryInto;
-use rustc_data_structures::sync::Lrc;
-use log::debug;
-
-#[cfg(test)]
-mod tests;
-
-pub mod comments;
-mod tokentrees;
-mod unicode_chars;
-
-#[derive(Clone, Debug)]
-pub struct UnmatchedBrace {
-    pub expected_delim: token::DelimToken,
-    pub found_delim: Option<token::DelimToken>,
-    pub found_span: Span,
-    pub unclosed_span: Option<Span>,
-    pub candidate_span: Option<Span>,
-}
-
-pub struct StringReader<'a> {
-    sess: &'a ParseSess,
-    /// Initial position, read-only.
-    start_pos: BytePos,
-    /// The absolute offset within the source_map of the current character.
-    pos: BytePos,
-    /// Stop reading src at this index.
-    end_src_index: usize,
-    /// Source text to tokenize.
-    src: Lrc<String>,
-    override_span: Option<Span>,
-}
-
-impl<'a> StringReader<'a> {
-    pub fn new(sess: &'a ParseSess,
-               source_file: Lrc<syntax_pos::SourceFile>,
-               override_span: Option<Span>) -> Self {
-        if source_file.src.is_none() {
-            sess.span_diagnostic.bug(&format!("cannot lex `source_file` without source: {}",
-                                              source_file.name));
-        }
-
-        let src = (*source_file.src.as_ref().unwrap()).clone();
-
-        StringReader {
-            sess,
-            start_pos: source_file.start_pos,
-            pos: source_file.start_pos,
-            end_src_index: src.len(),
-            src,
-            override_span,
-        }
-    }
-
-    pub fn retokenize(sess: &'a ParseSess, mut span: Span) -> Self {
-        let begin = sess.source_map().lookup_byte_offset(span.lo());
-        let end = sess.source_map().lookup_byte_offset(span.hi());
-
-        // Make the range zero-length if the span is invalid.
-        if span.lo() > span.hi() || begin.sf.start_pos != end.sf.start_pos {
-            span = span.shrink_to_lo();
-        }
-
-        let mut sr = StringReader::new(sess, begin.sf, None);
-
-        // Seek the lexer to the right byte range.
-        sr.end_src_index = sr.src_index(span.hi());
-
-        sr
-    }
-
-
-    fn mk_sp(&self, lo: BytePos, hi: BytePos) -> Span {
-        self.override_span.unwrap_or_else(|| Span::with_root_ctxt(lo, hi))
-    }
-
-    /// Returns the next token, including trivia like whitespace or comments.
-    ///
-    /// `Err(())` means that some errors were encountered, which can be
-    /// retrieved using `buffer_fatal_errors`.
-    pub fn next_token(&mut self) -> Token {
-        let start_src_index = self.src_index(self.pos);
-        let text: &str = &self.src[start_src_index..self.end_src_index];
-
-        if text.is_empty() {
-            let span = self.mk_sp(self.pos, self.pos);
-            return Token::new(token::Eof, span);
-        }
-
-        {
-            let is_beginning_of_file = self.pos == self.start_pos;
-            if is_beginning_of_file {
-                if let Some(shebang_len) = rustc_lexer::strip_shebang(text) {
-                    let start = self.pos;
-                    self.pos = self.pos + BytePos::from_usize(shebang_len);
-
-                    let sym = self.symbol_from(start + BytePos::from_usize("#!".len()));
-                    let kind = token::Shebang(sym);
-
-                    let span = self.mk_sp(start, self.pos);
-                    return Token::new(kind, span);
-                }
-            }
-        }
-
-        let token = rustc_lexer::first_token(text);
-
-        let start = self.pos;
-        self.pos = self.pos + BytePos::from_usize(token.len);
-
-        debug!("try_next_token: {:?}({:?})", token.kind, self.str_from(start));
-
-        // This could use `?`, but that makes code significantly (10-20%) slower.
-        // https://github.com/rust-lang/rust/issues/37939
-        let kind = self.cook_lexer_token(token.kind, start);
-
-        let span = self.mk_sp(start, self.pos);
-        Token::new(kind, span)
-    }
-
-    /// Report a fatal lexical error with a given span.
-    fn fatal_span(&self, sp: Span, m: &str) -> FatalError {
-        self.sess.span_diagnostic.span_fatal(sp, m)
-    }
-
-    /// Report a lexical error with a given span.
-    fn err_span(&self, sp: Span, m: &str) {
-        self.sess.span_diagnostic.struct_span_err(sp, m).emit();
-    }
-
-
-    /// Report a fatal error spanning [`from_pos`, `to_pos`).
-    fn fatal_span_(&self, from_pos: BytePos, to_pos: BytePos, m: &str) -> FatalError {
-        self.fatal_span(self.mk_sp(from_pos, to_pos), m)
-    }
-
-    /// Report a lexical error spanning [`from_pos`, `to_pos`).
-    fn err_span_(&self, from_pos: BytePos, to_pos: BytePos, m: &str) {
-        self.err_span(self.mk_sp(from_pos, to_pos), m)
-    }
-
-    fn struct_span_fatal(&self, from_pos: BytePos, to_pos: BytePos, m: &str)
-        -> DiagnosticBuilder<'a>
-    {
-        self.sess.span_diagnostic.struct_span_fatal(self.mk_sp(from_pos, to_pos), m)
-    }
-
-    fn struct_fatal_span_char(&self, from_pos: BytePos, to_pos: BytePos, m: &str, c: char)
-        -> DiagnosticBuilder<'a>
-    {
-        let mut m = m.to_string();
-        m.push_str(": ");
-        push_escaped_char(&mut m, c);
-
-        self.sess.span_diagnostic.struct_span_fatal(self.mk_sp(from_pos, to_pos), &m[..])
-    }
-
-    /// Turns simple `rustc_lexer::TokenKind` enum into a rich
-    /// `libsyntax::TokenKind`. This turns strings into interned
-    /// symbols and runs additional validation.
-    fn cook_lexer_token(
-        &self,
-        token: rustc_lexer::TokenKind,
-        start: BytePos,
-    ) -> TokenKind {
-        match token {
-            rustc_lexer::TokenKind::LineComment => {
-                let string = self.str_from(start);
-                // comments with only more "/"s are not doc comments
-                let tok = if is_doc_comment(string) {
-                    self.forbid_bare_cr(start, string, "bare CR not allowed in doc-comment");
-                    token::DocComment(Symbol::intern(string))
-                } else {
-                    token::Comment
-                };
-
-                tok
-            }
-            rustc_lexer::TokenKind::BlockComment { terminated } => {
-                let string = self.str_from(start);
-                // block comments starting with "/**" or "/*!" are doc-comments
-                // but comments with only "*"s between two "/"s are not
-                let is_doc_comment = is_block_doc_comment(string);
-
-                if !terminated {
-                    let msg = if is_doc_comment {
-                        "unterminated block doc-comment"
-                    } else {
-                        "unterminated block comment"
-                    };
-                    let last_bpos = self.pos;
-                    self.fatal_span_(start, last_bpos, msg).raise();
-                }
-
-                let tok = if is_doc_comment {
-                    self.forbid_bare_cr(start,
-                                        string,
-                                        "bare CR not allowed in block doc-comment");
-                    token::DocComment(Symbol::intern(string))
-                } else {
-                    token::Comment
-                };
-
-                tok
-            }
-            rustc_lexer::TokenKind::Whitespace => token::Whitespace,
-            rustc_lexer::TokenKind::Ident | rustc_lexer::TokenKind::RawIdent => {
-                let is_raw_ident = token == rustc_lexer::TokenKind::RawIdent;
-                let mut ident_start = start;
-                if is_raw_ident {
-                    ident_start = ident_start + BytePos(2);
-                }
-                // FIXME: perform NFKC normalization here. (Issue #2253)
-                let sym = self.symbol_from(ident_start);
-                if is_raw_ident {
-                    let span = self.mk_sp(start, self.pos);
-                    if !sym.can_be_raw() {
-                        self.err_span(span, &format!("`{}` cannot be a raw identifier", sym));
-                    }
-                    self.sess.raw_identifier_spans.borrow_mut().push(span);
-                }
-                token::Ident(sym, is_raw_ident)
-            }
-            rustc_lexer::TokenKind::Literal { kind, suffix_start } => {
-                let suffix_start = start + BytePos(suffix_start as u32);
-                let (kind, symbol) = self.cook_lexer_literal(start, suffix_start, kind);
-                let suffix = if suffix_start < self.pos {
-                    let string = self.str_from(suffix_start);
-                    if string == "_" {
-                        self.sess.span_diagnostic
-                            .struct_span_warn(self.mk_sp(suffix_start, self.pos),
-                                              "underscore literal suffix is not allowed")
-                            .warn("this was previously accepted by the compiler but is \
-                                   being phased out; it will become a hard error in \
-                                   a future release!")
-                            .note("for more information, see issue #42326 \
-                                   <https://github.com/rust-lang/rust/issues/42326>")
-                            .emit();
-                        None
-                    } else {
-                        Some(Symbol::intern(string))
-                    }
-                } else {
-                    None
-                };
-                token::Literal(token::Lit { kind, symbol, suffix })
-            }
-            rustc_lexer::TokenKind::Lifetime { starts_with_number } => {
-                // Include the leading `'` in the real identifier, for macro
-                // expansion purposes. See #12512 for the gory details of why
-                // this is necessary.
-                let lifetime_name = self.str_from(start);
-                if starts_with_number {
-                    self.err_span_(
-                        start,
-                        self.pos,
-                        "lifetimes cannot start with a number",
-                    );
-                }
-                let ident = Symbol::intern(lifetime_name);
-                token::Lifetime(ident)
-            }
-            rustc_lexer::TokenKind::Semi => token::Semi,
-            rustc_lexer::TokenKind::Comma => token::Comma,
-            rustc_lexer::TokenKind::Dot => token::Dot,
-            rustc_lexer::TokenKind::OpenParen => token::OpenDelim(token::Paren),
-            rustc_lexer::TokenKind::CloseParen => token::CloseDelim(token::Paren),
-            rustc_lexer::TokenKind::OpenBrace => token::OpenDelim(token::Brace),
-            rustc_lexer::TokenKind::CloseBrace => token::CloseDelim(token::Brace),
-            rustc_lexer::TokenKind::OpenBracket => token::OpenDelim(token::Bracket),
-            rustc_lexer::TokenKind::CloseBracket => token::CloseDelim(token::Bracket),
-            rustc_lexer::TokenKind::At => token::At,
-            rustc_lexer::TokenKind::Pound => token::Pound,
-            rustc_lexer::TokenKind::Tilde => token::Tilde,
-            rustc_lexer::TokenKind::Question => token::Question,
-            rustc_lexer::TokenKind::Colon => token::Colon,
-            rustc_lexer::TokenKind::Dollar => token::Dollar,
-            rustc_lexer::TokenKind::Eq => token::Eq,
-            rustc_lexer::TokenKind::Not => token::Not,
-            rustc_lexer::TokenKind::Lt => token::Lt,
-            rustc_lexer::TokenKind::Gt => token::Gt,
-            rustc_lexer::TokenKind::Minus => token::BinOp(token::Minus),
-            rustc_lexer::TokenKind::And => token::BinOp(token::And),
-            rustc_lexer::TokenKind::Or => token::BinOp(token::Or),
-            rustc_lexer::TokenKind::Plus => token::BinOp(token::Plus),
-            rustc_lexer::TokenKind::Star => token::BinOp(token::Star),
-            rustc_lexer::TokenKind::Slash => token::BinOp(token::Slash),
-            rustc_lexer::TokenKind::Caret => token::BinOp(token::Caret),
-            rustc_lexer::TokenKind::Percent => token::BinOp(token::Percent),
-
-            rustc_lexer::TokenKind::Unknown => {
-                let c = self.str_from(start).chars().next().unwrap();
-                let mut err = self.struct_fatal_span_char(start,
-                                                          self.pos,
-                                                          "unknown start of token",
-                                                          c);
-                // FIXME: the lexer could be used to turn the ASCII version of unicode homoglyphs,
-                // instead of keeping a table in `check_for_substitution`into the token. Ideally,
-                // this should be inside `rustc_lexer`. However, we should first remove compound
-                // tokens like `<<` from `rustc_lexer`, and then add fancier error recovery to it,
-                // as there will be less overall work to do this way.
-                let token = unicode_chars::check_for_substitution(self, start, c, &mut err)
-                    .unwrap_or_else(|| token::Unknown(self.symbol_from(start)));
-                err.emit();
-                token
-            }
-        }
-    }
-
-    fn cook_lexer_literal(
-        &self,
-        start: BytePos,
-        suffix_start: BytePos,
-        kind: rustc_lexer::LiteralKind
-    ) -> (token::LitKind, Symbol) {
-        match kind {
-            rustc_lexer::LiteralKind::Char { terminated } => {
-                if !terminated {
-                    self.fatal_span_(start, suffix_start,
-                                     "unterminated character literal".into())
-                        .raise()
-                }
-                let content_start = start + BytePos(1);
-                let content_end = suffix_start - BytePos(1);
-                self.validate_char_escape(content_start, content_end);
-                let id = self.symbol_from_to(content_start, content_end);
-                (token::Char, id)
-            },
-            rustc_lexer::LiteralKind::Byte { terminated } => {
-                if !terminated {
-                    self.fatal_span_(start + BytePos(1), suffix_start,
-                                     "unterminated byte constant".into())
-                        .raise()
-                }
-                let content_start = start + BytePos(2);
-                let content_end = suffix_start - BytePos(1);
-                self.validate_byte_escape(content_start, content_end);
-                let id = self.symbol_from_to(content_start, content_end);
-                (token::Byte, id)
-            },
-            rustc_lexer::LiteralKind::Str { terminated } => {
-                if !terminated {
-                    self.fatal_span_(start, suffix_start,
-                                     "unterminated double quote string".into())
-                        .raise()
-                }
-                let content_start = start + BytePos(1);
-                let content_end = suffix_start - BytePos(1);
-                self.validate_str_escape(content_start, content_end);
-                let id = self.symbol_from_to(content_start, content_end);
-                (token::Str, id)
-            }
-            rustc_lexer::LiteralKind::ByteStr { terminated } => {
-                if !terminated {
-                    self.fatal_span_(start + BytePos(1), suffix_start,
-                                     "unterminated double quote byte string".into())
-                        .raise()
-                }
-                let content_start = start + BytePos(2);
-                let content_end = suffix_start - BytePos(1);
-                self.validate_byte_str_escape(content_start, content_end);
-                let id = self.symbol_from_to(content_start, content_end);
-                (token::ByteStr, id)
-            }
-            rustc_lexer::LiteralKind::RawStr { n_hashes, started, terminated } => {
-                if !started {
-                    self.report_non_started_raw_string(start);
-                }
-                if !terminated {
-                    self.report_unterminated_raw_string(start, n_hashes)
-                }
-                let n_hashes: u16 = self.restrict_n_hashes(start, n_hashes);
-                let n = u32::from(n_hashes);
-                let content_start = start + BytePos(2 + n);
-                let content_end = suffix_start - BytePos(1 + n);
-                self.validate_raw_str_escape(content_start, content_end);
-                let id = self.symbol_from_to(content_start, content_end);
-                (token::StrRaw(n_hashes), id)
-            }
-            rustc_lexer::LiteralKind::RawByteStr { n_hashes, started, terminated } => {
-                if !started {
-                    self.report_non_started_raw_string(start);
-                }
-                if !terminated {
-                    self.report_unterminated_raw_string(start, n_hashes)
-                }
-                let n_hashes: u16 = self.restrict_n_hashes(start, n_hashes);
-                let n = u32::from(n_hashes);
-                let content_start = start + BytePos(3 + n);
-                let content_end = suffix_start - BytePos(1 + n);
-                self.validate_raw_byte_str_escape(content_start, content_end);
-                let id = self.symbol_from_to(content_start, content_end);
-                (token::ByteStrRaw(n_hashes), id)
-            }
-            rustc_lexer::LiteralKind::Int { base, empty_int } => {
-                if empty_int {
-                    self.err_span_(start, suffix_start, "no valid digits found for number");
-                    (token::Integer, sym::integer(0))
-                } else {
-                    self.validate_int_literal(base, start, suffix_start);
-                    (token::Integer, self.symbol_from_to(start, suffix_start))
-                }
-            },
-            rustc_lexer::LiteralKind::Float { base, empty_exponent } => {
-                if empty_exponent {
-                    let mut err = self.struct_span_fatal(
-                        start, self.pos,
-                        "expected at least one digit in exponent"
-                    );
-                    err.emit();
-                }
-
-                match base {
-                    Base::Hexadecimal => {
-                        self.err_span_(start, suffix_start,
-                                       "hexadecimal float literal is not supported")
-                    }
-                    Base::Octal => {
-                        self.err_span_(start, suffix_start,
-                                       "octal float literal is not supported")
-                    }
-                    Base::Binary => {
-                        self.err_span_(start, suffix_start,
-                                       "binary float literal is not supported")
-                    }
-                    _ => ()
-                }
-
-                let id = self.symbol_from_to(start, suffix_start);
-                (token::Float, id)
-            },
-        }
-    }
-
-    #[inline]
-    fn src_index(&self, pos: BytePos) -> usize {
-        (pos - self.start_pos).to_usize()
-    }
-
-    /// Slice of the source text from `start` up to but excluding `self.pos`,
-    /// meaning the slice does not include the character `self.ch`.
-    fn str_from(&self, start: BytePos) -> &str
-    {
-        self.str_from_to(start, self.pos)
-    }
-
-    /// Creates a Symbol from a given offset to the current offset.
-    fn symbol_from(&self, start: BytePos) -> Symbol {
-        debug!("taking an ident from {:?} to {:?}", start, self.pos);
-        Symbol::intern(self.str_from(start))
-    }
-
-    /// As symbol_from, with an explicit endpoint.
-    fn symbol_from_to(&self, start: BytePos, end: BytePos) -> Symbol {
-        debug!("taking an ident from {:?} to {:?}", start, end);
-        Symbol::intern(self.str_from_to(start, end))
-    }
-
-    /// Slice of the source text spanning from `start` up to but excluding `end`.
-    fn str_from_to(&self, start: BytePos, end: BytePos) -> &str
-    {
-        &self.src[self.src_index(start)..self.src_index(end)]
-    }
-
-    fn forbid_bare_cr(&self, start: BytePos, s: &str, errmsg: &str) {
-        let mut idx = 0;
-        loop {
-            idx = match s[idx..].find('\r') {
-                None => break,
-                Some(it) => idx + it + 1
-            };
-            self.err_span_(start + BytePos(idx as u32 - 1),
-                           start + BytePos(idx as u32),
-                           errmsg);
-        }
-    }
-
-    fn report_non_started_raw_string(&self, start: BytePos) -> ! {
-        let bad_char = self.str_from(start).chars().last().unwrap();
-        self
-            .struct_fatal_span_char(
-                start,
-                self.pos,
-                "found invalid character; only `#` is allowed \
-                 in raw string delimitation",
-                bad_char,
-            )
-            .emit();
-        FatalError.raise()
-    }
-
-    fn report_unterminated_raw_string(&self, start: BytePos, n_hashes: usize) -> ! {
-        let mut err = self.struct_span_fatal(
-            start, start,
-            "unterminated raw string",
-        );
-        err.span_label(
-            self.mk_sp(start, start),
-            "unterminated raw string",
-        );
-
-        if n_hashes > 0 {
-            err.note(&format!("this raw string should be terminated with `\"{}`",
-                                "#".repeat(n_hashes as usize)));
-        }
-
-        err.emit();
-        FatalError.raise()
-    }
-
-    fn restrict_n_hashes(&self, start: BytePos, n_hashes: usize) -> u16 {
-        match n_hashes.try_into() {
-            Ok(n_hashes) => n_hashes,
-            Err(_) => {
-                self.fatal_span_(start,
-                                 self.pos,
-                                 "too many `#` symbols: raw strings may be \
-                                  delimited by up to 65535 `#` symbols").raise();
-            }
-        }
-    }
-
-    fn validate_char_escape(&self, content_start: BytePos, content_end: BytePos) {
-        let lit = self.str_from_to(content_start, content_end);
-        if let Err((off, err)) = unescape::unescape_char(lit) {
-            emit_unescape_error(
-                &self.sess.span_diagnostic,
-                lit,
-                self.mk_sp(content_start - BytePos(1), content_end + BytePos(1)),
-                unescape::Mode::Char,
-                0..off,
-                err,
-            )
-        }
-    }
-
-    fn validate_byte_escape(&self, content_start: BytePos, content_end: BytePos) {
-        let lit = self.str_from_to(content_start, content_end);
-        if let Err((off, err)) = unescape::unescape_byte(lit) {
-            emit_unescape_error(
-                &self.sess.span_diagnostic,
-                lit,
-                self.mk_sp(content_start - BytePos(1), content_end + BytePos(1)),
-                unescape::Mode::Byte,
-                0..off,
-                err,
-            )
-        }
-    }
-
-    fn validate_str_escape(&self, content_start: BytePos, content_end: BytePos) {
-        let lit = self.str_from_to(content_start, content_end);
-        unescape::unescape_str(lit, &mut |range, c| {
-            if let Err(err) = c {
-                emit_unescape_error(
-                    &self.sess.span_diagnostic,
-                    lit,
-                    self.mk_sp(content_start - BytePos(1), content_end + BytePos(1)),
-                    unescape::Mode::Str,
-                    range,
-                    err,
-                )
-            }
-        })
-    }
-
-    fn validate_raw_str_escape(&self, content_start: BytePos, content_end: BytePos) {
-        let lit = self.str_from_to(content_start, content_end);
-        unescape::unescape_raw_str(lit, &mut |range, c| {
-            if let Err(err) = c {
-                emit_unescape_error(
-                    &self.sess.span_diagnostic,
-                    lit,
-                    self.mk_sp(content_start - BytePos(1), content_end + BytePos(1)),
-                    unescape::Mode::Str,
-                    range,
-                    err,
-                )
-            }
-        })
-    }
-
-    fn validate_raw_byte_str_escape(&self, content_start: BytePos, content_end: BytePos) {
-        let lit = self.str_from_to(content_start, content_end);
-        unescape::unescape_raw_byte_str(lit, &mut |range, c| {
-            if let Err(err) = c {
-                emit_unescape_error(
-                    &self.sess.span_diagnostic,
-                    lit,
-                    self.mk_sp(content_start - BytePos(1), content_end + BytePos(1)),
-                    unescape::Mode::ByteStr,
-                    range,
-                    err,
-                )
-            }
-        })
-    }
-
-    fn validate_byte_str_escape(&self, content_start: BytePos, content_end: BytePos) {
-        let lit = self.str_from_to(content_start, content_end);
-        unescape::unescape_byte_str(lit, &mut |range, c| {
-            if let Err(err) = c {
-                emit_unescape_error(
-                    &self.sess.span_diagnostic,
-                    lit,
-                    self.mk_sp(content_start - BytePos(1), content_end + BytePos(1)),
-                    unescape::Mode::ByteStr,
-                    range,
-                    err,
-                )
-            }
-        })
-    }
-
-    fn validate_int_literal(&self, base: Base, content_start: BytePos, content_end: BytePos) {
-        let base = match base {
-            Base::Binary => 2,
-            Base::Octal => 8,
-            _ => return,
-        };
-        let s = self.str_from_to(content_start + BytePos(2), content_end);
-        for (idx, c) in s.char_indices() {
-            let idx = idx as u32;
-            if c != '_' && c.to_digit(base).is_none() {
-                let lo = content_start + BytePos(2 + idx);
-                let hi = content_start + BytePos(2 + idx + c.len_utf8() as u32);
-                self.err_span_(lo, hi,
-                               &format!("invalid digit for a base {} literal", base));
-
-            }
-        }
-    }
-}
-
-fn is_doc_comment(s: &str) -> bool {
-    let res = (s.starts_with("///") && *s.as_bytes().get(3).unwrap_or(&b' ') != b'/') ||
-              s.starts_with("//!");
-    debug!("is {:?} a doc comment? {}", s, res);
-    res
-}
-
-fn is_block_doc_comment(s: &str) -> bool {
-    // Prevent `/**/` from being parsed as a doc comment
-    let res = ((s.starts_with("/**") && *s.as_bytes().get(3).unwrap_or(&b' ') != b'*') ||
-               s.starts_with("/*!")) && s.len() >= 5;
-    debug!("is {:?} a doc comment? {}", s, res);
-    res
-}
diff --git a/src/libsyntax/parse/lexer/tests.rs b/src/libsyntax/parse/lexer/tests.rs
deleted file mode 100644
index de301b1fc49..00000000000
--- a/src/libsyntax/parse/lexer/tests.rs
+++ /dev/null
@@ -1,269 +0,0 @@
-use super::*;
-
-use crate::symbol::Symbol;
-use crate::source_map::{SourceMap, FilePathMapping};
-use crate::parse::token;
-use crate::with_default_globals;
-
-use errors::{Handler, emitter::EmitterWriter};
-use std::io;
-use std::path::PathBuf;
-use syntax_pos::{BytePos, Span};
-
-fn mk_sess(sm: Lrc<SourceMap>) -> ParseSess {
-    let emitter = EmitterWriter::new(
-        Box::new(io::sink()),
-        Some(sm.clone()),
-        false,
-        false,
-        false,
-        None,
-        false,
-    );
-    ParseSess::with_span_handler(Handler::with_emitter(true, None, Box::new(emitter)), sm)
-}
-
-// Creates a string reader for the given string.
-fn setup<'a>(sm: &SourceMap,
-                sess: &'a ParseSess,
-                teststr: String)
-                -> StringReader<'a> {
-    let sf = sm.new_source_file(PathBuf::from(teststr.clone()).into(), teststr);
-    StringReader::new(sess, sf, None)
-}
-
-#[test]
-fn t1() {
-    with_default_globals(|| {
-        let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
-        let sh = mk_sess(sm.clone());
-        let mut string_reader = setup(
-            &sm,
-            &sh,
-            "/* my source file */ fn main() { println!(\"zebra\"); }\n".to_string(),
-        );
-        assert_eq!(string_reader.next_token(), token::Comment);
-        assert_eq!(string_reader.next_token(), token::Whitespace);
-        let tok1 = string_reader.next_token();
-        let tok2 = Token::new(
-            mk_ident("fn"),
-            Span::with_root_ctxt(BytePos(21), BytePos(23)),
-        );
-        assert_eq!(tok1.kind, tok2.kind);
-        assert_eq!(tok1.span, tok2.span);
-        assert_eq!(string_reader.next_token(), token::Whitespace);
-        // Read another token.
-        let tok3 = string_reader.next_token();
-        assert_eq!(string_reader.pos.clone(), BytePos(28));
-        let tok4 = Token::new(
-            mk_ident("main"),
-            Span::with_root_ctxt(BytePos(24), BytePos(28)),
-        );
-        assert_eq!(tok3.kind, tok4.kind);
-        assert_eq!(tok3.span, tok4.span);
-
-        assert_eq!(string_reader.next_token(), token::OpenDelim(token::Paren));
-        assert_eq!(string_reader.pos.clone(), BytePos(29))
-    })
-}
-
-// Checks that the given reader produces the desired stream
-// of tokens (stop checking after exhausting `expected`).
-fn check_tokenization(mut string_reader: StringReader<'_>, expected: Vec<TokenKind>) {
-    for expected_tok in &expected {
-        assert_eq!(&string_reader.next_token(), expected_tok);
-    }
-}
-
-// Makes the identifier by looking up the string in the interner.
-fn mk_ident(id: &str) -> TokenKind {
-    token::Ident(Symbol::intern(id), false)
-}
-
-fn mk_lit(kind: token::LitKind, symbol: &str, suffix: Option<&str>) -> TokenKind {
-    TokenKind::lit(kind, Symbol::intern(symbol), suffix.map(Symbol::intern))
-}
-
-#[test]
-fn doublecolon_parsing() {
-    with_default_globals(|| {
-        let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
-        let sh = mk_sess(sm.clone());
-        check_tokenization(
-            setup(&sm, &sh, "a b".to_string()),
-            vec![mk_ident("a"), token::Whitespace, mk_ident("b")],
-        );
-    })
-}
-
-#[test]
-fn doublecolon_parsing_2() {
-    with_default_globals(|| {
-        let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
-        let sh = mk_sess(sm.clone());
-        check_tokenization(
-            setup(&sm, &sh, "a::b".to_string()),
-            vec![mk_ident("a"), token::Colon, token::Colon, mk_ident("b")],
-        );
-    })
-}
-
-#[test]
-fn doublecolon_parsing_3() {
-    with_default_globals(|| {
-        let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
-        let sh = mk_sess(sm.clone());
-        check_tokenization(
-            setup(&sm, &sh, "a ::b".to_string()),
-            vec![mk_ident("a"), token::Whitespace, token::Colon, token::Colon, mk_ident("b")],
-        );
-    })
-}
-
-#[test]
-fn doublecolon_parsing_4() {
-    with_default_globals(|| {
-        let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
-        let sh = mk_sess(sm.clone());
-        check_tokenization(
-            setup(&sm, &sh, "a:: b".to_string()),
-            vec![mk_ident("a"), token::Colon, token::Colon, token::Whitespace, mk_ident("b")],
-        );
-    })
-}
-
-#[test]
-fn character_a() {
-    with_default_globals(|| {
-        let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
-        let sh = mk_sess(sm.clone());
-        assert_eq!(
-            setup(&sm, &sh, "'a'".to_string()).next_token(),
-            mk_lit(token::Char, "a", None),
-        );
-    })
-}
-
-#[test]
-fn character_space() {
-    with_default_globals(|| {
-        let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
-        let sh = mk_sess(sm.clone());
-        assert_eq!(
-            setup(&sm, &sh, "' '".to_string()).next_token(),
-            mk_lit(token::Char, " ", None),
-        );
-    })
-}
-
-#[test]
-fn character_escaped() {
-    with_default_globals(|| {
-        let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
-        let sh = mk_sess(sm.clone());
-        assert_eq!(
-            setup(&sm, &sh, "'\\n'".to_string()).next_token(),
-            mk_lit(token::Char, "\\n", None),
-        );
-    })
-}
-
-#[test]
-fn lifetime_name() {
-    with_default_globals(|| {
-        let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
-        let sh = mk_sess(sm.clone());
-        assert_eq!(
-            setup(&sm, &sh, "'abc".to_string()).next_token(),
-            token::Lifetime(Symbol::intern("'abc")),
-        );
-    })
-}
-
-#[test]
-fn raw_string() {
-    with_default_globals(|| {
-        let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
-        let sh = mk_sess(sm.clone());
-        assert_eq!(
-            setup(&sm, &sh, "r###\"\"#a\\b\x00c\"\"###".to_string()).next_token(),
-            mk_lit(token::StrRaw(3), "\"#a\\b\x00c\"", None),
-        );
-    })
-}
-
-#[test]
-fn literal_suffixes() {
-    with_default_globals(|| {
-        let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
-        let sh = mk_sess(sm.clone());
-        macro_rules! test {
-            ($input: expr, $tok_type: ident, $tok_contents: expr) => {{
-                assert_eq!(
-                    setup(&sm, &sh, format!("{}suffix", $input)).next_token(),
-                    mk_lit(token::$tok_type, $tok_contents, Some("suffix")),
-                );
-                // with a whitespace separator
-                assert_eq!(
-                    setup(&sm, &sh, format!("{} suffix", $input)).next_token(),
-                    mk_lit(token::$tok_type, $tok_contents, None),
-                );
-            }}
-        }
-
-        test!("'a'", Char, "a");
-        test!("b'a'", Byte, "a");
-        test!("\"a\"", Str, "a");
-        test!("b\"a\"", ByteStr, "a");
-        test!("1234", Integer, "1234");
-        test!("0b101", Integer, "0b101");
-        test!("0xABC", Integer, "0xABC");
-        test!("1.0", Float, "1.0");
-        test!("1.0e10", Float, "1.0e10");
-
-        assert_eq!(
-            setup(&sm, &sh, "2us".to_string()).next_token(),
-            mk_lit(token::Integer, "2", Some("us")),
-        );
-        assert_eq!(
-            setup(&sm, &sh, "r###\"raw\"###suffix".to_string()).next_token(),
-            mk_lit(token::StrRaw(3), "raw", Some("suffix")),
-        );
-        assert_eq!(
-            setup(&sm, &sh, "br###\"raw\"###suffix".to_string()).next_token(),
-            mk_lit(token::ByteStrRaw(3), "raw", Some("suffix")),
-        );
-    })
-}
-
-#[test]
-fn line_doc_comments() {
-    assert!(is_doc_comment("///"));
-    assert!(is_doc_comment("/// blah"));
-    assert!(!is_doc_comment("////"));
-}
-
-#[test]
-fn nested_block_comments() {
-    with_default_globals(|| {
-        let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
-        let sh = mk_sess(sm.clone());
-        let mut lexer = setup(&sm, &sh, "/* /* */ */'a'".to_string());
-        assert_eq!(lexer.next_token(), token::Comment);
-        assert_eq!(lexer.next_token(), mk_lit(token::Char, "a", None));
-    })
-}
-
-#[test]
-fn crlf_comments() {
-    with_default_globals(|| {
-        let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
-        let sh = mk_sess(sm.clone());
-        let mut lexer = setup(&sm, &sh, "// test\r\n/// test\r\n".to_string());
-        let comment = lexer.next_token();
-        assert_eq!(comment.kind, token::Comment);
-        assert_eq!((comment.span.lo(), comment.span.hi()), (BytePos(0), BytePos(7)));
-        assert_eq!(lexer.next_token(), token::Whitespace);
-        assert_eq!(lexer.next_token(), token::DocComment(Symbol::intern("/// test")));
-    })
-}
diff --git a/src/libsyntax/parse/lexer/tokentrees.rs b/src/libsyntax/parse/lexer/tokentrees.rs
deleted file mode 100644
index de8ac2c71e8..00000000000
--- a/src/libsyntax/parse/lexer/tokentrees.rs
+++ /dev/null
@@ -1,279 +0,0 @@
-use rustc_data_structures::fx::FxHashMap;
-use syntax_pos::Span;
-
-use super::{StringReader, UnmatchedBrace};
-
-use crate::print::pprust::token_to_string;
-use crate::parse::token::{self, Token};
-use crate::parse::PResult;
-use crate::tokenstream::{DelimSpan, IsJoint::{self, *}, TokenStream, TokenTree, TreeAndJoint};
-
-impl<'a> StringReader<'a> {
-    crate fn into_token_trees(self) -> (PResult<'a, TokenStream>, Vec<UnmatchedBrace>) {
-        let mut tt_reader = TokenTreesReader {
-            string_reader: self,
-            token: Token::dummy(),
-            joint_to_prev: Joint,
-            open_braces: Vec::new(),
-            unmatched_braces: Vec::new(),
-            matching_delim_spans: Vec::new(),
-            last_unclosed_found_span: None,
-            last_delim_empty_block_spans: FxHashMap::default()
-        };
-        let res = tt_reader.parse_all_token_trees();
-        (res, tt_reader.unmatched_braces)
-    }
-}
-
-struct TokenTreesReader<'a> {
-    string_reader: StringReader<'a>,
-    token: Token,
-    joint_to_prev: IsJoint,
-    /// Stack of open delimiters and their spans. Used for error message.
-    open_braces: Vec<(token::DelimToken, Span)>,
-    unmatched_braces: Vec<UnmatchedBrace>,
-    /// The type and spans for all braces
-    ///
-    /// Used only for error recovery when arriving to EOF with mismatched braces.
-    matching_delim_spans: Vec<(token::DelimToken, Span, Span)>,
-    last_unclosed_found_span: Option<Span>,
-    last_delim_empty_block_spans: FxHashMap<token::DelimToken, Span>
-}
-
-impl<'a> TokenTreesReader<'a> {
-    // Parse a stream of tokens into a list of `TokenTree`s, up to an `Eof`.
-    fn parse_all_token_trees(&mut self) -> PResult<'a, TokenStream> {
-        let mut buf = TokenStreamBuilder::default();
-
-        self.real_token();
-        while self.token != token::Eof {
-            buf.push(self.parse_token_tree()?);
-        }
-
-        Ok(buf.into_token_stream())
-    }
-
-    // Parse a stream of tokens into a list of `TokenTree`s, up to a `CloseDelim`.
-    fn parse_token_trees_until_close_delim(&mut self) -> TokenStream {
-        let mut buf = TokenStreamBuilder::default();
-        loop {
-            if let token::CloseDelim(..) = self.token.kind {
-                return buf.into_token_stream();
-            }
-
-            match self.parse_token_tree() {
-                Ok(tree) => buf.push(tree),
-                Err(mut e) => {
-                    e.emit();
-                    return buf.into_token_stream();
-                }
-            }
-        }
-    }
-
-    fn parse_token_tree(&mut self) -> PResult<'a, TreeAndJoint> {
-        let sm = self.string_reader.sess.source_map();
-        match self.token.kind {
-            token::Eof => {
-                let msg = "this file contains an un-closed delimiter";
-                let mut err = self.string_reader.sess.span_diagnostic
-                    .struct_span_err(self.token.span, msg);
-                for &(_, sp) in &self.open_braces {
-                    err.span_label(sp, "un-closed delimiter");
-                    self.unmatched_braces.push(UnmatchedBrace {
-                        expected_delim: token::DelimToken::Brace,
-                        found_delim: None,
-                        found_span: self.token.span,
-                        unclosed_span: Some(sp),
-                        candidate_span: None,
-                    });
-                }
-
-                if let Some((delim, _)) = self.open_braces.last() {
-                    if let Some((_, open_sp, close_sp)) = self.matching_delim_spans.iter()
-                        .filter(|(d, open_sp, close_sp)| {
-                            if let Some(close_padding) = sm.span_to_margin(*close_sp) {
-                                if let Some(open_padding) = sm.span_to_margin(*open_sp) {
-                                    return delim == d && close_padding != open_padding;
-                                }
-                            }
-                            false
-                        }).next()  // these are in reverse order as they get inserted on close, but
-                    {              // we want the last open/first close
-                        err.span_label(
-                            *open_sp,
-                            "this delimiter might not be properly closed...",
-                        );
-                        err.span_label(
-                            *close_sp,
-                            "...as it matches this but it has different indentation",
-                        );
-                    }
-                }
-                Err(err)
-            },
-            token::OpenDelim(delim) => {
-                // The span for beginning of the delimited section
-                let pre_span = self.token.span;
-
-                // Parse the open delimiter.
-                self.open_braces.push((delim, self.token.span));
-                self.real_token();
-
-                // Parse the token trees within the delimiters.
-                // We stop at any delimiter so we can try to recover if the user
-                // uses an incorrect delimiter.
-                let tts = self.parse_token_trees_until_close_delim();
-
-                // Expand to cover the entire delimited token tree
-                let delim_span = DelimSpan::from_pair(pre_span, self.token.span);
-
-                match self.token.kind {
-                    // Correct delimiter.
-                    token::CloseDelim(d) if d == delim => {
-                        let (open_brace, open_brace_span) = self.open_braces.pop().unwrap();
-                        let close_brace_span = self.token.span;
-
-                        if tts.is_empty() {
-                            let empty_block_span = open_brace_span.to(close_brace_span);
-                            self.last_delim_empty_block_spans.insert(delim, empty_block_span);
-                        }
-
-                        if self.open_braces.len() == 0 {
-                            // Clear up these spans to avoid suggesting them as we've found
-                            // properly matched delimiters so far for an entire block.
-                            self.matching_delim_spans.clear();
-                        } else {
-                            self.matching_delim_spans.push(
-                                (open_brace, open_brace_span, close_brace_span),
-                            );
-                        }
-                        // Parse the close delimiter.
-                        self.real_token();
-                    }
-                    // Incorrect delimiter.
-                    token::CloseDelim(other) => {
-                        let mut unclosed_delimiter = None;
-                        let mut candidate = None;
-                        if self.last_unclosed_found_span != Some(self.token.span) {
-                            // do not complain about the same unclosed delimiter multiple times
-                            self.last_unclosed_found_span = Some(self.token.span);
-                            // This is a conservative error: only report the last unclosed
-                            // delimiter. The previous unclosed delimiters could actually be
-                            // closed! The parser just hasn't gotten to them yet.
-                            if let Some(&(_, sp)) = self.open_braces.last() {
-                                unclosed_delimiter = Some(sp);
-                            };
-                            if let Some(current_padding) = sm.span_to_margin(self.token.span) {
-                                for (brace, brace_span) in &self.open_braces {
-                                    if let Some(padding) = sm.span_to_margin(*brace_span) {
-                                        // high likelihood of these two corresponding
-                                        if current_padding == padding && brace == &other {
-                                            candidate = Some(*brace_span);
-                                        }
-                                    }
-                                }
-                            }
-                            let (tok, _) = self.open_braces.pop().unwrap();
-                            self.unmatched_braces.push(UnmatchedBrace {
-                                expected_delim: tok,
-                                found_delim: Some(other),
-                                found_span: self.token.span,
-                                unclosed_span: unclosed_delimiter,
-                                candidate_span: candidate,
-                            });
-                        } else {
-                            self.open_braces.pop();
-                        }
-
-                        // If the incorrect delimiter matches an earlier opening
-                        // delimiter, then don't consume it (it can be used to
-                        // close the earlier one). Otherwise, consume it.
-                        // E.g., we try to recover from:
-                        // fn foo() {
-                        //     bar(baz(
-                        // }  // Incorrect delimiter but matches the earlier `{`
-                        if !self.open_braces.iter().any(|&(b, _)| b == other) {
-                            self.real_token();
-                        }
-                    }
-                    token::Eof => {
-                        // Silently recover, the EOF token will be seen again
-                        // and an error emitted then. Thus we don't pop from
-                        // self.open_braces here.
-                    },
-                    _ => {}
-                }
-
-                Ok(TokenTree::Delimited(
-                    delim_span,
-                    delim,
-                    tts.into()
-                ).into())
-            },
-            token::CloseDelim(delim) => {
-                // An unexpected closing delimiter (i.e., there is no
-                // matching opening delimiter).
-                let token_str = token_to_string(&self.token);
-                let msg = format!("unexpected close delimiter: `{}`", token_str);
-                let mut err = self.string_reader.sess.span_diagnostic
-                    .struct_span_err(self.token.span, &msg);
-
-                if let Some(span) = self.last_delim_empty_block_spans.remove(&delim) {
-                    err.span_label(
-                        span,
-                        "this block is empty, you might have not meant to close it"
-                    );
-                }
-                err.span_label(self.token.span, "unexpected close delimiter");
-                Err(err)
-            },
-            _ => {
-                let tt = TokenTree::Token(self.token.take());
-                self.real_token();
-                let is_joint = self.joint_to_prev == Joint && self.token.is_op();
-                Ok((tt, if is_joint { Joint } else { NonJoint }))
-            }
-        }
-    }
-
-    fn real_token(&mut self) {
-        self.joint_to_prev = Joint;
-        loop {
-            let token = self.string_reader.next_token();
-            match token.kind {
-                token::Whitespace | token::Comment | token::Shebang(_) | token::Unknown(_) => {
-                    self.joint_to_prev = NonJoint;
-                }
-                _ => {
-                    self.token = token;
-                    return;
-                }
-            }
-        }
-    }
-}
-
-#[derive(Default)]
-struct TokenStreamBuilder {
-    buf: Vec<TreeAndJoint>,
-}
-
-impl TokenStreamBuilder {
-    fn push(&mut self, (tree, joint): TreeAndJoint) {
-        if let Some((TokenTree::Token(prev_token), Joint)) = self.buf.last() {
-            if let TokenTree::Token(token) = &tree {
-                if let Some(glued) = prev_token.glue(token) {
-                    self.buf.pop();
-                    self.buf.push((TokenTree::Token(glued), joint));
-                    return;
-                }
-            }
-        }
-        self.buf.push((tree, joint))
-    }
-
-    fn into_token_stream(self) -> TokenStream {
-        TokenStream::new(self.buf)
-    }
-}
diff --git a/src/libsyntax/parse/lexer/unicode_chars.rs b/src/libsyntax/parse/lexer/unicode_chars.rs
deleted file mode 100644
index 525b4215aff..00000000000
--- a/src/libsyntax/parse/lexer/unicode_chars.rs
+++ /dev/null
@@ -1,392 +0,0 @@
-// Characters and their corresponding confusables were collected from
-// http://www.unicode.org/Public/security/10.0.0/confusables.txt
-
-use super::StringReader;
-use errors::{Applicability, DiagnosticBuilder};
-use syntax_pos::{BytePos, Pos, Span, symbol::kw};
-use crate::parse::token;
-
-#[rustfmt::skip] // for line breaks
-const UNICODE_ARRAY: &[(char, &str, char)] = &[
-    ('
', "Line Separator", ' '),
-    ('
', "Paragraph Separator", ' '),
-    (' ', "Ogham Space mark", ' '),
-    (' ', "En Quad", ' '),
-    (' ', "Em Quad", ' '),
-    (' ', "En Space", ' '),
-    (' ', "Em Space", ' '),
-    (' ', "Three-Per-Em Space", ' '),
-    (' ', "Four-Per-Em Space", ' '),
-    (' ', "Six-Per-Em Space", ' '),
-    (' ', "Punctuation Space", ' '),
-    (' ', "Thin Space", ' '),
-    (' ', "Hair Space", ' '),
-    (' ', "Medium Mathematical Space", ' '),
-    (' ', "No-Break Space", ' '),
-    (' ', "Figure Space", ' '),
-    (' ', "Narrow No-Break Space", ' '),
-    ('　', "Ideographic Space", ' '),
-
-    ('ߺ', "Nko Lajanyalan", '_'),
-    ('﹍', "Dashed Low Line", '_'),
-    ('﹎', "Centreline Low Line", '_'),
-    ('﹏', "Wavy Low Line", '_'),
-    ('＿', "Fullwidth Low Line", '_'),
-
-    ('‐', "Hyphen", '-'),
-    ('‑', "Non-Breaking Hyphen", '-'),
-    ('‒', "Figure Dash", '-'),
-    ('–', "En Dash", '-'),
-    ('—', "Em Dash", '-'),
-    ('﹘', "Small Em Dash", '-'),
-    ('۔', "Arabic Full Stop", '-'),
-    ('⁃', "Hyphen Bullet", '-'),
-    ('˗', "Modifier Letter Minus Sign", '-'),
-    ('−', "Minus Sign", '-'),
-    ('➖', "Heavy Minus Sign", '-'),
-    ('Ⲻ', "Coptic Letter Dialect-P Ni", '-'),
-    ('ー', "Katakana-Hiragana Prolonged Sound Mark", '-'),
-    ('－', "Fullwidth Hyphen-Minus", '-'),
-    ('―', "Horizontal Bar", '-'),
-    ('─', "Box Drawings Light Horizontal", '-'),
-    ('━', "Box Drawings Heavy Horizontal", '-'),
-    ('㇐', "CJK Stroke H", '-'),
-    ('ꟷ', "Latin Epigraphic Letter Sideways I", '-'),
-    ('ᅳ', "Hangul Jungseong Eu", '-'),
-    ('ㅡ', "Hangul Letter Eu", '-'),
-    ('一', "CJK Unified Ideograph-4E00", '-'),
-    ('⼀', "Kangxi Radical One", '-'),
-
-    ('؍', "Arabic Date Separator", ','),
-    ('٫', "Arabic Decimal Separator", ','),
-    ('‚', "Single Low-9 Quotation Mark", ','),
-    ('¸', "Cedilla", ','),
-    ('ꓹ', "Lisu Letter Tone Na Po", ','),
-    ('，', "Fullwidth Comma", ','),
-
-    (';', "Greek Question Mark", ';'),
-    ('；', "Fullwidth Semicolon", ';'),
-    ('︔', "Presentation Form For Vertical Semicolon", ';'),
-
-    ('ः', "Devanagari Sign Visarga", ':'),
-    ('ઃ', "Gujarati Sign Visarga", ':'),
-    ('：', "Fullwidth Colon", ':'),
-    ('։', "Armenian Full Stop", ':'),
-    ('܃', "Syriac Supralinear Colon", ':'),
-    ('܄', "Syriac Sublinear Colon", ':'),
-    ('᛬', "Runic Multiple Punctuation", ':'),
-    ('︰', "Presentation Form For Vertical Two Dot Leader", ':'),
-    ('᠃', "Mongolian Full Stop", ':'),
-    ('᠉', "Mongolian Manchu Full Stop", ':'),
-    ('⁚', "Two Dot Punctuation", ':'),
-    ('׃', "Hebrew Punctuation Sof Pasuq", ':'),
-    ('˸', "Modifier Letter Raised Colon", ':'),
-    ('꞉', "Modifier Letter Colon", ':'),
-    ('∶', "Ratio", ':'),
-    ('ː', "Modifier Letter Triangular Colon", ':'),
-    ('ꓽ', "Lisu Letter Tone Mya Jeu", ':'),
-    ('︓', "Presentation Form For Vertical Colon", ':'),
-
-    ('！', "Fullwidth Exclamation Mark", '!'),
-    ('ǃ', "Latin Letter Retroflex Click", '!'),
-    ('ⵑ', "Tifinagh Letter Tuareg Yang", '!'),
-    ('︕', "Presentation Form For Vertical Exclamation Mark", '!'),
-
-    ('ʔ', "Latin Letter Glottal Stop", '?'),
-    ('Ɂ', "Latin Capital Letter Glottal Stop", '?'),
-    ('ॽ', "Devanagari Letter Glottal Stop", '?'),
-    ('Ꭾ', "Cherokee Letter He", '?'),
-    ('ꛫ', "Bamum Letter Ntuu", '?'),
-    ('？', "Fullwidth Question Mark", '?'),
-    ('︖', "Presentation Form For Vertical Question Mark", '?'),
-
-    ('𝅭', "Musical Symbol Combining Augmentation Dot", '.'),
-    ('․', "One Dot Leader", '.'),
-    ('܁', "Syriac Supralinear Full Stop", '.'),
-    ('܂', "Syriac Sublinear Full Stop", '.'),
-    ('꘎', "Vai Full Stop", '.'),
-    ('𐩐', "Kharoshthi Punctuation Dot", '.'),
-    ('٠', "Arabic-Indic Digit Zero", '.'),
-    ('۰', "Extended Arabic-Indic Digit Zero", '.'),
-    ('ꓸ', "Lisu Letter Tone Mya Ti", '.'),
-    ('·', "Middle Dot", '.'),
-    ('・', "Katakana Middle Dot", '.'),
-    ('･', "Halfwidth Katakana Middle Dot", '.'),
-    ('᛫', "Runic Single Punctuation", '.'),
-    ('·', "Greek Ano Teleia", '.'),
-    ('⸱', "Word Separator Middle Dot", '.'),
-    ('𐄁', "Aegean Word Separator Dot", '.'),
-    ('•', "Bullet", '.'),
-    ('‧', "Hyphenation Point", '.'),
-    ('∙', "Bullet Operator", '.'),
-    ('⋅', "Dot Operator", '.'),
-    ('ꞏ', "Latin Letter Sinological Dot", '.'),
-    ('ᐧ', "Canadian Syllabics Final Middle Dot", '.'),
-    ('ᐧ', "Canadian Syllabics Final Middle Dot", '.'),
-    ('．', "Fullwidth Full Stop", '.'),
-    ('。', "Ideographic Full Stop", '.'),
-    ('︒', "Presentation Form For Vertical Ideographic Full Stop", '.'),
-
-    ('՝', "Armenian Comma", '\''),
-    ('＇', "Fullwidth Apostrophe", '\''),
-    ('‘', "Left Single Quotation Mark", '\''),
-    ('’', "Right Single Quotation Mark", '\''),
-    ('‛', "Single High-Reversed-9 Quotation Mark", '\''),
-    ('′', "Prime", '\''),
-    ('‵', "Reversed Prime", '\''),
-    ('՚', "Armenian Apostrophe", '\''),
-    ('׳', "Hebrew Punctuation Geresh", '\''),
-    ('`', "Grave Accent", '\''),
-    ('`', "Greek Varia", '\''),
-    ('｀', "Fullwidth Grave Accent", '\''),
-    ('´', "Acute Accent", '\''),
-    ('΄', "Greek Tonos", '\''),
-    ('´', "Greek Oxia", '\''),
-    ('᾽', "Greek Koronis", '\''),
-    ('᾿', "Greek Psili", '\''),
-    ('῾', "Greek Dasia", '\''),
-    ('ʹ', "Modifier Letter Prime", '\''),
-    ('ʹ', "Greek Numeral Sign", '\''),
-    ('ˈ', "Modifier Letter Vertical Line", '\''),
-    ('ˊ', "Modifier Letter Acute Accent", '\''),
-    ('ˋ', "Modifier Letter Grave Accent", '\''),
-    ('˴', "Modifier Letter Middle Grave Accent", '\''),
-    ('ʻ', "Modifier Letter Turned Comma", '\''),
-    ('ʽ', "Modifier Letter Reversed Comma", '\''),
-    ('ʼ', "Modifier Letter Apostrophe", '\''),
-    ('ʾ', "Modifier Letter Right Half Ring", '\''),
-    ('ꞌ', "Latin Small Letter Saltillo", '\''),
-    ('י', "Hebrew Letter Yod", '\''),
-    ('ߴ', "Nko High Tone Apostrophe", '\''),
-    ('ߵ', "Nko Low Tone Apostrophe", '\''),
-    ('ᑊ', "Canadian Syllabics West-Cree P", '\''),
-    ('ᛌ', "Runic Letter Short-Twig-Sol S", '\''),
-    ('𖽑', "Miao Sign Aspiration", '\''),
-    ('𖽒', "Miao Sign Reformed Voicing", '\''),
-
-    ('᳓', "Vedic Sign Nihshvasa", '"'),
-    ('＂', "Fullwidth Quotation Mark", '"'),
-    ('“', "Left Double Quotation Mark", '"'),
-    ('”', "Right Double Quotation Mark", '"'),
-    ('‟', "Double High-Reversed-9 Quotation Mark", '"'),
-    ('″', "Double Prime", '"'),
-    ('‶', "Reversed Double Prime", '"'),
-    ('〃', "Ditto Mark", '"'),
-    ('״', "Hebrew Punctuation Gershayim", '"'),
-    ('˝', "Double Acute Accent", '"'),
-    ('ʺ', "Modifier Letter Double Prime", '"'),
-    ('˶', "Modifier Letter Middle Double Acute Accent", '"'),
-    ('˵', "Modifier Letter Middle Double Grave Accent", '"'),
-    ('ˮ', "Modifier Letter Double Apostrophe", '"'),
-    ('ײ', "Hebrew Ligature Yiddish Double Yod", '"'),
-    ('❞', "Heavy Double Comma Quotation Mark Ornament", '"'),
-    ('❝', "Heavy Double Turned Comma Quotation Mark Ornament", '"'),
-
-    ('（', "Fullwidth Left Parenthesis", '('),
-    ('❨', "Medium Left Parenthesis Ornament", '('),
-    ('﴾', "Ornate Left Parenthesis", '('),
-
-    ('）', "Fullwidth Right Parenthesis", ')'),
-    ('❩', "Medium Right Parenthesis Ornament", ')'),
-    ('﴿', "Ornate Right Parenthesis", ')'),
-
-    ('［', "Fullwidth Left Square Bracket", '['),
-    ('❲', "Light Left Tortoise Shell Bracket Ornament", '['),
-    ('「', "Left Corner Bracket", '['),
-    ('『', "Left White Corner Bracket", '['),
-    ('【', "Left Black Lenticular Bracket", '['),
-    ('〔', "Left Tortoise Shell Bracket", '['),
-    ('〖', "Left White Lenticular Bracket", '['),
-    ('〘', "Left White Tortoise Shell Bracket", '['),
-    ('〚', "Left White Square Bracket", '['),
-
-    ('］', "Fullwidth Right Square Bracket", ']'),
-    ('❳', "Light Right Tortoise Shell Bracket Ornament", ']'),
-    ('」', "Right Corner Bracket", ']'),
-    ('』', "Right White Corner Bracket", ']'),
-    ('】', "Right Black Lenticular Bracket", ']'),
-    ('〕', "Right Tortoise Shell Bracket", ']'),
-    ('〗', "Right White Lenticular Bracket", ']'),
-    ('〙', "Right White Tortoise Shell Bracket", ']'),
-    ('〛', "Right White Square Bracket", ']'),
-
-    ('❴', "Medium Left Curly Bracket Ornament", '{'),
-    ('𝄔', "Musical Symbol Brace", '{'),
-    ('｛', "Fullwidth Left Curly Bracket", '{'),
-
-    ('❵', "Medium Right Curly Bracket Ornament", '}'),
-    ('｝', "Fullwidth Right Curly Bracket", '}'),
-
-    ('⁎', "Low Asterisk", '*'),
-    ('٭', "Arabic Five Pointed Star", '*'),
-    ('∗', "Asterisk Operator", '*'),
-    ('𐌟', "Old Italic Letter Ess", '*'),
-    ('＊', "Fullwidth Asterisk", '*'),
-
-    ('᜵', "Philippine Single Punctuation", '/'),
-    ('⁁', "Caret Insertion Point", '/'),
-    ('∕', "Division Slash", '/'),
-    ('⁄', "Fraction Slash", '/'),
-    ('╱', "Box Drawings Light Diagonal Upper Right To Lower Left", '/'),
-    ('⟋', "Mathematical Rising Diagonal", '/'),
-    ('⧸', "Big Solidus", '/'),
-    ('𝈺', "Greek Instrumental Notation Symbol-47", '/'),
-    ('㇓', "CJK Stroke Sp", '/'),
-    ('〳', "Vertical Kana Repeat Mark Upper Half", '/'),
-    ('Ⳇ', "Coptic Capital Letter Old Coptic Esh", '/'),
-    ('ノ', "Katakana Letter No", '/'),
-    ('丿', "CJK Unified Ideograph-4E3F", '/'),
-    ('⼃', "Kangxi Radical Slash", '/'),
-    ('／', "Fullwidth Solidus", '/'),
-
-    ('＼', "Fullwidth Reverse Solidus", '\\'),
-    ('﹨', "Small Reverse Solidus", '\\'),
-    ('∖', "Set Minus", '\\'),
-    ('⟍', "Mathematical Falling Diagonal", '\\'),
-    ('⧵', "Reverse Solidus Operator", '\\'),
-    ('⧹', "Big Reverse Solidus", '\\'),
-    ('⧹', "Greek Vocal Notation Symbol-16", '\\'),
-    ('⧹', "Greek Instrumental Symbol-48", '\\'),
-    ('㇔', "CJK Stroke D", '\\'),
-    ('丶', "CJK Unified Ideograph-4E36", '\\'),
-    ('⼂', "Kangxi Radical Dot", '\\'),
-    ('、', "Ideographic Comma", '\\'),
-    ('ヽ', "Katakana Iteration Mark", '\\'),
-
-    ('ꝸ', "Latin Small Letter Um", '&'),
-    ('＆', "Fullwidth Ampersand", '&'),
-
-    ('᛭', "Runic Cross Punctuation", '+'),
-    ('➕', "Heavy Plus Sign", '+'),
-    ('𐊛', "Lycian Letter H", '+'),
-    ('﬩', "Hebrew Letter Alternative Plus Sign", '+'),
-    ('＋', "Fullwidth Plus Sign", '+'),
-
-    ('‹', "Single Left-Pointing Angle Quotation Mark", '<'),
-    ('❮', "Heavy Left-Pointing Angle Quotation Mark Ornament", '<'),
-    ('˂', "Modifier Letter Left Arrowhead", '<'),
-    ('𝈶', "Greek Instrumental Symbol-40", '<'),
-    ('ᐸ', "Canadian Syllabics Pa", '<'),
-    ('ᚲ', "Runic Letter Kauna", '<'),
-    ('❬', "Medium Left-Pointing Angle Bracket Ornament", '<'),
-    ('⟨', "Mathematical Left Angle Bracket", '<'),
-    ('〈', "Left-Pointing Angle Bracket", '<'),
-    ('〈', "Left Angle Bracket", '<'),
-    ('㇛', "CJK Stroke Pd", '<'),
-    ('く', "Hiragana Letter Ku", '<'),
-    ('𡿨', "CJK Unified Ideograph-21FE8", '<'),
-    ('《', "Left Double Angle Bracket", '<'),
-    ('＜', "Fullwidth Less-Than Sign", '<'),
-
-    ('᐀', "Canadian Syllabics Hyphen", '='),
-    ('⹀', "Double Hyphen", '='),
-    ('゠', "Katakana-Hiragana Double Hyphen", '='),
-    ('꓿', "Lisu Punctuation Full Stop", '='),
-    ('＝', "Fullwidth Equals Sign", '='),
-
-    ('›', "Single Right-Pointing Angle Quotation Mark", '>'),
-    ('❯', "Heavy Right-Pointing Angle Quotation Mark Ornament", '>'),
-    ('˃', "Modifier Letter Right Arrowhead", '>'),
-    ('𝈷', "Greek Instrumental Symbol-42", '>'),
-    ('ᐳ', "Canadian Syllabics Po", '>'),
-    ('𖼿', "Miao Letter Archaic Zza", '>'),
-    ('❭', "Medium Right-Pointing Angle Bracket Ornament", '>'),
-    ('⟩', "Mathematical Right Angle Bracket", '>'),
-    ('〉', "Right-Pointing Angle Bracket", '>'),
-    ('〉', "Right Angle Bracket", '>'),
-    ('》', "Right Double Angle Bracket", '>'),
-    ('＞', "Fullwidth Greater-Than Sign", '>'),
-];
-
-// FIXME: the lexer could be used to turn the ASCII version of unicode homoglyphs, instead of
-// keeping the substitution token in this table. Ideally, this should be inside `rustc_lexer`.
-// However, we should first remove compound tokens like `<<` from `rustc_lexer`, and then add
-// fancier error recovery to it, as there will be less overall work to do this way.
-const ASCII_ARRAY: &[(char, &str, Option<token::TokenKind>)] = &[
-    (' ', "Space", Some(token::Whitespace)),
-    ('_', "Underscore", Some(token::Ident(kw::Underscore, false))),
-    ('-', "Minus/Hyphen", Some(token::BinOp(token::Minus))),
-    (',', "Comma", Some(token::Comma)),
-    (';', "Semicolon", Some(token::Semi)),
-    (':', "Colon", Some(token::Colon)),
-    ('!', "Exclamation Mark", Some(token::Not)),
-    ('?', "Question Mark", Some(token::Question)),
-    ('.', "Period", Some(token::Dot)),
-    ('(', "Left Parenthesis", Some(token::OpenDelim(token::Paren))),
-    (')', "Right Parenthesis", Some(token::CloseDelim(token::Paren))),
-    ('[', "Left Square Bracket", Some(token::OpenDelim(token::Bracket))),
-    (']', "Right Square Bracket", Some(token::CloseDelim(token::Bracket))),
-    ('{', "Left Curly Brace", Some(token::OpenDelim(token::Brace))),
-    ('}', "Right Curly Brace", Some(token::CloseDelim(token::Brace))),
-    ('*', "Asterisk", Some(token::BinOp(token::Star))),
-    ('/', "Slash", Some(token::BinOp(token::Slash))),
-    ('\\', "Backslash", None),
-    ('&', "Ampersand", Some(token::BinOp(token::And))),
-    ('+', "Plus Sign", Some(token::BinOp(token::Plus))),
-    ('<', "Less-Than Sign", Some(token::Lt)),
-    ('=', "Equals Sign", Some(token::Eq)),
-    ('>', "Greater-Than Sign", Some(token::Gt)),
-    // FIXME: Literals are already lexed by this point, so we can't recover gracefully just by
-    // spitting the correct token out.
-    ('\'', "Single Quote", None),
-    ('"', "Quotation Mark", None),
-];
-
-crate fn check_for_substitution<'a>(
-    reader: &StringReader<'a>,
-    pos: BytePos,
-    ch: char,
-    err: &mut DiagnosticBuilder<'a>,
-) -> Option<token::TokenKind> {
-    let (u_name, ascii_char) = match UNICODE_ARRAY.iter().find(|&&(c, _, _)| c == ch) {
-        Some(&(_u_char, u_name, ascii_char)) => (u_name, ascii_char),
-        None => return None,
-    };
-
-    let span = Span::with_root_ctxt(pos, pos + Pos::from_usize(ch.len_utf8()));
-
-    let (ascii_name, token) = match ASCII_ARRAY.iter().find(|&&(c, _, _)| c == ascii_char) {
-        Some((_ascii_char, ascii_name, token)) => (ascii_name, token),
-        None => {
-            let msg = format!("substitution character not found for '{}'", ch);
-            reader.sess.span_diagnostic.span_bug_no_panic(span, &msg);
-            return None;
-        }
-    };
-
-    // special help suggestion for "directed" double quotes
-    if let Some(s) = peek_delimited(&reader.src[reader.src_index(pos)..], '“', '”') {
-        let msg = format!(
-            "Unicode characters '“' (Left Double Quotation Mark) and \
-             '”' (Right Double Quotation Mark) look like '{}' ({}), but are not",
-            ascii_char, ascii_name
-        );
-        err.span_suggestion(
-            Span::with_root_ctxt(
-                pos,
-                pos + Pos::from_usize('“'.len_utf8() + s.len() + '”'.len_utf8()),
-            ),
-            &msg,
-            format!("\"{}\"", s),
-            Applicability::MaybeIncorrect,
-        );
-    } else {
-        let msg = format!(
-            "Unicode character '{}' ({}) looks like '{}' ({}), but it is not",
-            ch, u_name, ascii_char, ascii_name
-        );
-        err.span_suggestion(span, &msg, ascii_char.to_string(), Applicability::MaybeIncorrect);
-    }
-    token.clone()
-}
-
-/// Extract string if found at current position with given delimiters
-fn peek_delimited(text: &str, from_ch: char, to_ch: char) -> Option<&str> {
-    let mut chars = text.chars();
-    let first_char = chars.next()?;
-    if first_char != from_ch {
-        return None;
-    }
-    let last_char_idx = chars.as_str().find(to_ch)?;
-    Some(&chars.as_str()[..last_char_idx])
-}