diff options
Diffstat (limited to 'src/libsyntax/parse/lexer')
| -rw-r--r-- | src/libsyntax/parse/lexer/mod.rs | 110 | ||||
| -rw-r--r-- | src/libsyntax/parse/lexer/tests.rs | 68 | ||||
| -rw-r--r-- | src/libsyntax/parse/lexer/tokentrees.rs | 40 | ||||
| -rw-r--r-- | src/libsyntax/parse/lexer/unicode_chars.rs | 7 |
4 files changed, 78 insertions, 147 deletions
diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs index e86d4c7fde6..66add869359 100644 --- a/src/libsyntax/parse/lexer/mod.rs +++ b/src/libsyntax/parse/lexer/mod.rs @@ -4,13 +4,11 @@ use crate::symbol::{sym, Symbol}; use crate::parse::unescape_error_reporting::{emit_unescape_error, push_escaped_char}; use errors::{FatalError, DiagnosticBuilder}; -use syntax_pos::{BytePos, Pos, Span, NO_EXPANSION}; +use syntax_pos::{BytePos, Pos, Span}; use rustc_lexer::Base; use rustc_lexer::unescape; -use std::borrow::Cow; use std::char; -use std::iter; use std::convert::TryInto; use rustc_data_structures::sync::Lrc; use log::debug; @@ -84,7 +82,7 @@ impl<'a> StringReader<'a> { fn mk_sp(&self, lo: BytePos, hi: BytePos) -> Span { - self.override_span.unwrap_or_else(|| Span::new(lo, hi, NO_EXPANSION)) + self.override_span.unwrap_or_else(|| Span::with_root_ctxt(lo, hi)) } /// Returns the next token, including trivia like whitespace or comments. @@ -181,18 +179,7 @@ impl<'a> StringReader<'a> { let string = self.str_from(start); // comments with only more "/"s are not doc comments let tok = if is_doc_comment(string) { - let mut idx = 0; - loop { - idx = match string[idx..].find('\r') { - None => break, - Some(it) => idx + it + 1 - }; - if string[idx..].chars().next() != Some('\n') { - self.err_span_(start + BytePos(idx as u32 - 1), - start + BytePos(idx as u32), - "bare CR not allowed in doc-comment"); - } - } + self.forbid_bare_cr(start, string, "bare CR not allowed in doc-comment"); token::DocComment(Symbol::intern(string)) } else { token::Comment @@ -217,15 +204,10 @@ impl<'a> StringReader<'a> { } let tok = if is_doc_comment { - let has_cr = string.contains('\r'); - let string = if has_cr { - self.translate_crlf(start, - string, - "bare CR not allowed in block doc-comment") - } else { - string.into() - }; - token::DocComment(Symbol::intern(&string[..])) + self.forbid_bare_cr(start, + string, + "bare CR not allowed in block doc-comment"); + token::DocComment(Symbol::intern(string)) } else { token::Comment }; @@ -291,9 +273,6 @@ impl<'a> StringReader<'a> { } rustc_lexer::TokenKind::Semi => token::Semi, rustc_lexer::TokenKind::Comma => token::Comma, - rustc_lexer::TokenKind::DotDotDot => token::DotDotDot, - rustc_lexer::TokenKind::DotDotEq => token::DotDotEq, - rustc_lexer::TokenKind::DotDot => token::DotDot, rustc_lexer::TokenKind::Dot => token::Dot, rustc_lexer::TokenKind::OpenParen => token::OpenDelim(token::Paren), rustc_lexer::TokenKind::CloseParen => token::CloseDelim(token::Paren), @@ -305,42 +284,20 @@ impl<'a> StringReader<'a> { rustc_lexer::TokenKind::Pound => token::Pound, rustc_lexer::TokenKind::Tilde => token::Tilde, rustc_lexer::TokenKind::Question => token::Question, - rustc_lexer::TokenKind::ColonColon => token::ModSep, rustc_lexer::TokenKind::Colon => token::Colon, rustc_lexer::TokenKind::Dollar => token::Dollar, - rustc_lexer::TokenKind::EqEq => token::EqEq, rustc_lexer::TokenKind::Eq => token::Eq, - rustc_lexer::TokenKind::FatArrow => token::FatArrow, - rustc_lexer::TokenKind::Ne => token::Ne, rustc_lexer::TokenKind::Not => token::Not, - rustc_lexer::TokenKind::Le => token::Le, - rustc_lexer::TokenKind::LArrow => token::LArrow, rustc_lexer::TokenKind::Lt => token::Lt, - rustc_lexer::TokenKind::ShlEq => token::BinOpEq(token::Shl), - rustc_lexer::TokenKind::Shl => token::BinOp(token::Shl), - rustc_lexer::TokenKind::Ge => token::Ge, rustc_lexer::TokenKind::Gt => token::Gt, - rustc_lexer::TokenKind::ShrEq => token::BinOpEq(token::Shr), - rustc_lexer::TokenKind::Shr => token::BinOp(token::Shr), - rustc_lexer::TokenKind::RArrow => token::RArrow, rustc_lexer::TokenKind::Minus => token::BinOp(token::Minus), - rustc_lexer::TokenKind::MinusEq => token::BinOpEq(token::Minus), rustc_lexer::TokenKind::And => token::BinOp(token::And), - rustc_lexer::TokenKind::AndEq => token::BinOpEq(token::And), - rustc_lexer::TokenKind::AndAnd => token::AndAnd, rustc_lexer::TokenKind::Or => token::BinOp(token::Or), - rustc_lexer::TokenKind::OrEq => token::BinOpEq(token::Or), - rustc_lexer::TokenKind::OrOr => token::OrOr, rustc_lexer::TokenKind::Plus => token::BinOp(token::Plus), - rustc_lexer::TokenKind::PlusEq => token::BinOpEq(token::Plus), rustc_lexer::TokenKind::Star => token::BinOp(token::Star), - rustc_lexer::TokenKind::StarEq => token::BinOpEq(token::Star), rustc_lexer::TokenKind::Slash => token::BinOp(token::Slash), - rustc_lexer::TokenKind::SlashEq => token::BinOpEq(token::Slash), rustc_lexer::TokenKind::Caret => token::BinOp(token::Caret), - rustc_lexer::TokenKind::CaretEq => token::BinOpEq(token::Caret), rustc_lexer::TokenKind::Percent => token::BinOp(token::Percent), - rustc_lexer::TokenKind::PercentEq => token::BinOpEq(token::Percent), rustc_lexer::TokenKind::Unknown => { let c = self.str_from(start).chars().next().unwrap(); @@ -516,49 +473,16 @@ impl<'a> StringReader<'a> { &self.src[self.src_index(start)..self.src_index(end)] } - /// Converts CRLF to LF in the given string, raising an error on bare CR. - fn translate_crlf<'b>(&self, start: BytePos, s: &'b str, errmsg: &'b str) -> Cow<'b, str> { - let mut chars = s.char_indices().peekable(); - while let Some((i, ch)) = chars.next() { - if ch == '\r' { - if let Some((lf_idx, '\n')) = chars.peek() { - return translate_crlf_(self, start, s, *lf_idx, chars, errmsg).into(); - } - let pos = start + BytePos(i as u32); - let end_pos = start + BytePos((i + ch.len_utf8()) as u32); - self.err_span_(pos, end_pos, errmsg); - } - } - return s.into(); - - fn translate_crlf_(rdr: &StringReader<'_>, - start: BytePos, - s: &str, - mut j: usize, - mut chars: iter::Peekable<impl Iterator<Item = (usize, char)>>, - errmsg: &str) - -> String { - let mut buf = String::with_capacity(s.len()); - // Skip first CR - buf.push_str(&s[.. j - 1]); - while let Some((i, ch)) = chars.next() { - if ch == '\r' { - if j < i { - buf.push_str(&s[j..i]); - } - let next = i + ch.len_utf8(); - j = next; - if chars.peek().map(|(_, ch)| *ch) != Some('\n') { - let pos = start + BytePos(i as u32); - let end_pos = start + BytePos(next as u32); - rdr.err_span_(pos, end_pos, errmsg); - } - } - } - if j < s.len() { - buf.push_str(&s[j..]); - } - buf + fn forbid_bare_cr(&self, start: BytePos, s: &str, errmsg: &str) { + let mut idx = 0; + loop { + idx = match s[idx..].find('\r') { + None => break, + Some(it) => idx + it + 1 + }; + self.err_span_(start + BytePos(idx as u32 - 1), + start + BytePos(idx as u32), + errmsg); } } diff --git a/src/libsyntax/parse/lexer/tests.rs b/src/libsyntax/parse/lexer/tests.rs index fc47e4f0b18..a915aa42fd1 100644 --- a/src/libsyntax/parse/lexer/tests.rs +++ b/src/libsyntax/parse/lexer/tests.rs @@ -1,41 +1,17 @@ use super::*; -use crate::ast::CrateConfig; use crate::symbol::Symbol; use crate::source_map::{SourceMap, FilePathMapping}; -use crate::feature_gate::UnstableFeatures; use crate::parse::token; -use crate::diagnostics::plugin::ErrorMap; use crate::with_default_globals; use std::io; use std::path::PathBuf; -use syntax_pos::{BytePos, Span, NO_EXPANSION, edition::Edition}; -use rustc_data_structures::fx::{FxHashSet, FxHashMap}; -use rustc_data_structures::sync::{Lock, Once}; +use errors::{Handler, emitter::EmitterWriter}; +use syntax_pos::{BytePos, Span}; fn mk_sess(sm: Lrc<SourceMap>) -> ParseSess { - let emitter = errors::emitter::EmitterWriter::new(Box::new(io::sink()), - Some(sm.clone()), - false, - false, - false); - ParseSess { - span_diagnostic: errors::Handler::with_emitter(true, None, Box::new(emitter)), - unstable_features: UnstableFeatures::from_environment(), - config: CrateConfig::default(), - included_mod_stack: Lock::new(Vec::new()), - source_map: sm, - missing_fragment_specifiers: Lock::new(FxHashSet::default()), - raw_identifier_spans: Lock::new(Vec::new()), - registered_diagnostics: Lock::new(ErrorMap::new()), - buffered_lints: Lock::new(vec![]), - edition: Edition::from_session(), - ambiguous_block_expr_parse: Lock::new(FxHashMap::default()), - param_attr_spans: Lock::new(Vec::new()), - let_chains_spans: Lock::new(Vec::new()), - async_closure_spans: Lock::new(Vec::new()), - injected_crate_name: Once::new(), - } + let emitter = EmitterWriter::new(Box::new(io::sink()), Some(sm.clone()), false, false, false); + ParseSess::with_span_handler(Handler::with_emitter(true, None, Box::new(emitter)), sm) } // open a string reader for the given string @@ -61,7 +37,7 @@ fn t1() { let tok1 = string_reader.next_token(); let tok2 = Token::new( mk_ident("fn"), - Span::new(BytePos(21), BytePos(23), NO_EXPANSION), + Span::with_root_ctxt(BytePos(21), BytePos(23)), ); assert_eq!(tok1.kind, tok2.kind); assert_eq!(tok1.span, tok2.span); @@ -71,7 +47,7 @@ fn t1() { assert_eq!(string_reader.pos.clone(), BytePos(28)); let tok4 = Token::new( mk_ident("main"), - Span::new(BytePos(24), BytePos(28), NO_EXPANSION), + Span::with_root_ctxt(BytePos(24), BytePos(28)), ); assert_eq!(tok3.kind, tok4.kind); assert_eq!(tok3.span, tok4.span); @@ -99,42 +75,50 @@ fn mk_lit(kind: token::LitKind, symbol: &str, suffix: Option<&str>) -> TokenKind } #[test] -fn doublecolonparsing() { +fn doublecolon_parsing() { with_default_globals(|| { let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); let sh = mk_sess(sm.clone()); - check_tokenization(setup(&sm, &sh, "a b".to_string()), - vec![mk_ident("a"), token::Whitespace, mk_ident("b")]); + check_tokenization( + setup(&sm, &sh, "a b".to_string()), + vec![mk_ident("a"), token::Whitespace, mk_ident("b")], + ); }) } #[test] -fn dcparsing_2() { +fn doublecolon_parsing_2() { with_default_globals(|| { let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); let sh = mk_sess(sm.clone()); - check_tokenization(setup(&sm, &sh, "a::b".to_string()), - vec![mk_ident("a"), token::ModSep, mk_ident("b")]); + check_tokenization( + setup(&sm, &sh, "a::b".to_string()), + vec![mk_ident("a"), token::Colon, token::Colon, mk_ident("b")], + ); }) } #[test] -fn dcparsing_3() { +fn doublecolon_parsing_3() { with_default_globals(|| { let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); let sh = mk_sess(sm.clone()); - check_tokenization(setup(&sm, &sh, "a ::b".to_string()), - vec![mk_ident("a"), token::Whitespace, token::ModSep, mk_ident("b")]); + check_tokenization( + setup(&sm, &sh, "a ::b".to_string()), + vec![mk_ident("a"), token::Whitespace, token::Colon, token::Colon, mk_ident("b")], + ); }) } #[test] -fn dcparsing_4() { +fn doublecolon_parsing_4() { with_default_globals(|| { let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); let sh = mk_sess(sm.clone()); - check_tokenization(setup(&sm, &sh, "a:: b".to_string()), - vec![mk_ident("a"), token::ModSep, token::Whitespace, mk_ident("b")]); + check_tokenization( + setup(&sm, &sh, "a:: b".to_string()), + vec![mk_ident("a"), token::Colon, token::Colon, token::Whitespace, mk_ident("b")], + ); }) } diff --git a/src/libsyntax/parse/lexer/tokentrees.rs b/src/libsyntax/parse/lexer/tokentrees.rs index 37e67a2729e..e5ba7e45309 100644 --- a/src/libsyntax/parse/lexer/tokentrees.rs +++ b/src/libsyntax/parse/lexer/tokentrees.rs @@ -39,29 +39,29 @@ struct TokenTreesReader<'a> { impl<'a> TokenTreesReader<'a> { // Parse a stream of tokens into a list of `TokenTree`s, up to an `Eof`. fn parse_all_token_trees(&mut self) -> PResult<'a, TokenStream> { - let mut tts = Vec::new(); + let mut buf = TokenStreamBuilder::default(); self.real_token(); while self.token != token::Eof { - tts.push(self.parse_token_tree()?); + buf.push(self.parse_token_tree()?); } - Ok(TokenStream::new(tts)) + Ok(buf.into_token_stream()) } // Parse a stream of tokens into a list of `TokenTree`s, up to a `CloseDelim`. fn parse_token_trees_until_close_delim(&mut self) -> TokenStream { - let mut tts = vec![]; + let mut buf = TokenStreamBuilder::default(); loop { if let token::CloseDelim(..) = self.token.kind { - return TokenStream::new(tts); + return buf.into_token_stream(); } match self.parse_token_tree() { - Ok(tree) => tts.push(tree), + Ok(tree) => buf.push(tree), Err(mut e) => { e.emit(); - return TokenStream::new(tts); + return buf.into_token_stream(); } } } @@ -223,8 +223,32 @@ impl<'a> TokenTreesReader<'a> { _ => { self.token = token; return; - }, + } + } + } + } +} + +#[derive(Default)] +struct TokenStreamBuilder { + buf: Vec<TreeAndJoint>, +} + +impl TokenStreamBuilder { + fn push(&mut self, (tree, joint): TreeAndJoint) { + if let Some((TokenTree::Token(prev_token), Joint)) = self.buf.last() { + if let TokenTree::Token(token) = &tree { + if let Some(glued) = prev_token.glue(token) { + self.buf.pop(); + self.buf.push((TokenTree::Token(glued), joint)); + return; + } } } + self.buf.push((tree, joint)) + } + + fn into_token_stream(self) -> TokenStream { + TokenStream::new(self.buf) } } diff --git a/src/libsyntax/parse/lexer/unicode_chars.rs b/src/libsyntax/parse/lexer/unicode_chars.rs index eaa736c6a35..525b4215aff 100644 --- a/src/libsyntax/parse/lexer/unicode_chars.rs +++ b/src/libsyntax/parse/lexer/unicode_chars.rs @@ -3,7 +3,7 @@ use super::StringReader; use errors::{Applicability, DiagnosticBuilder}; -use syntax_pos::{BytePos, Pos, Span, NO_EXPANSION, symbol::kw}; +use syntax_pos::{BytePos, Pos, Span, symbol::kw}; use crate::parse::token; #[rustfmt::skip] // for line breaks @@ -343,7 +343,7 @@ crate fn check_for_substitution<'a>( None => return None, }; - let span = Span::new(pos, pos + Pos::from_usize(ch.len_utf8()), NO_EXPANSION); + let span = Span::with_root_ctxt(pos, pos + Pos::from_usize(ch.len_utf8())); let (ascii_name, token) = match ASCII_ARRAY.iter().find(|&&(c, _, _)| c == ascii_char) { Some((_ascii_char, ascii_name, token)) => (ascii_name, token), @@ -362,10 +362,9 @@ crate fn check_for_substitution<'a>( ascii_char, ascii_name ); err.span_suggestion( - Span::new( + Span::with_root_ctxt( pos, pos + Pos::from_usize('“'.len_utf8() + s.len() + '”'.len_utf8()), - NO_EXPANSION, ), &msg, format!("\"{}\"", s), |