diff options
Diffstat (limited to 'src/libsyntax/parse')
| -rw-r--r-- | src/libsyntax/parse/lexer/mod.rs | 23 | ||||
| -rw-r--r-- | src/libsyntax/parse/parser.rs | 9 | ||||
| -rw-r--r-- | src/libsyntax/parse/token.rs | 13 |
3 files changed, 27 insertions, 18 deletions
diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs index 3cd5464f357..263eb1ac7a4 100644 --- a/src/libsyntax/parse/lexer/mod.rs +++ b/src/libsyntax/parse/lexer/mod.rs @@ -29,16 +29,15 @@ pub struct UnmatchedBrace { } pub struct StringReader<'a> { - crate sess: &'a ParseSess, - /// The absolute offset within the source_map of the current character - crate pos: BytePos, - /// The current character (which has been read from self.pos) - crate source_file: Lrc<syntax_pos::SourceFile>, + sess: &'a ParseSess, + /// Initial position, read-only. + start_pos: BytePos, + /// The absolute offset within the source_map of the current character. + pos: BytePos, /// Stop reading src at this index. - crate end_src_index: usize, + end_src_index: usize, fatal_errs: Vec<DiagnosticBuilder<'a>>, - // cache a direct reference to the source text, so that we don't have to - // retrieve it via `self.source_file.src.as_ref().unwrap()` all the time. + /// Source text to tokenize. src: Lrc<String>, override_span: Option<Span>, } @@ -56,8 +55,8 @@ impl<'a> StringReader<'a> { StringReader { sess, + start_pos: source_file.start_pos, pos: source_file.start_pos, - source_file, end_src_index: src.len(), src, fatal_errs: Vec::new(), @@ -108,12 +107,12 @@ impl<'a> StringReader<'a> { let text: &str = &self.src[start_src_index..self.end_src_index]; if text.is_empty() { - let span = self.mk_sp(self.source_file.end_pos, self.source_file.end_pos); + let span = self.mk_sp(self.pos, self.pos); return Ok(Token::new(token::Eof, span)); } { - let is_beginning_of_file = self.pos == self.source_file.start_pos; + let is_beginning_of_file = self.pos == self.start_pos; if is_beginning_of_file { if let Some(shebang_len) = rustc_lexer::strip_shebang(text) { let start = self.pos; @@ -533,7 +532,7 @@ impl<'a> StringReader<'a> { #[inline] fn src_index(&self, pos: BytePos) -> usize { - (pos - self.source_file.start_pos).to_usize() + (pos - self.start_pos).to_usize() } /// Slice of the source text from `start` up to but excluding `self.pos`, diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index 7fda9158b4b..7096d6799e2 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -143,6 +143,7 @@ macro_rules! maybe_whole_expr { $p.token.span, ExprKind::Block(block, None), ThinVec::new() )); } + // N.B: `NtIdent(ident)` is normalized to `Ident` in `fn bump`. _ => {}, }; } @@ -2756,12 +2757,7 @@ impl<'a> Parser<'a> { // can't continue an expression after an ident token::Ident(name, is_raw) => token::ident_can_begin_expr(name, t.span, is_raw), token::Literal(..) | token::Pound => true, - token::Interpolated(ref nt) => match **nt { - token::NtIdent(..) | token::NtExpr(..) | - token::NtBlock(..) | token::NtPath(..) => true, - _ => false, - }, - _ => false + _ => t.is_whole_expr(), }; let cannot_continue_expr = self.look_ahead(1, token_cannot_continue_expr); if cannot_continue_expr { @@ -3728,6 +3724,7 @@ impl<'a> Parser<'a> { self.token.is_path_start() // e.g. `MY_CONST`; || self.token == token::Dot // e.g. `.5` for recovery; || self.token.can_begin_literal_or_bool() // e.g. `42`. + || self.token.is_whole_expr() } // Helper function to decide whether to parse as ident binding diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index 472e4b474d6..73adb5c947c 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -476,6 +476,19 @@ impl Token { false } + /// Would `maybe_whole_expr` in `parser.rs` return `Ok(..)`? + /// That is, is this a pre-parsed expression dropped into the token stream + /// (which happens while parsing the result of macro expansion)? + crate fn is_whole_expr(&self) -> bool { + if let Interpolated(ref nt) = self.kind { + if let NtExpr(_) | NtLiteral(_) | NtPath(_) | NtIdent(..) | NtBlock(_) = **nt { + return true; + } + } + + false + } + /// Returns `true` if the token is either the `mut` or `const` keyword. crate fn is_mutability(&self) -> bool { self.is_keyword(kw::Mut) || |
