diff options
Diffstat (limited to 'compiler/rustc_parse/src/lexer')
| -rw-r--r-- | compiler/rustc_parse/src/lexer/mod.rs | 35 |
1 files changed, 30 insertions, 5 deletions
diff --git a/compiler/rustc_parse/src/lexer/mod.rs b/compiler/rustc_parse/src/lexer/mod.rs index e3bd6a9a327..f5f081efc49 100644 --- a/compiler/rustc_parse/src/lexer/mod.rs +++ b/compiler/rustc_parse/src/lexer/mod.rs @@ -44,19 +44,44 @@ pub(crate) struct UnmatchedDelim { pub candidate_span: Option<Span>, } +/// Which tokens should be stripped before lexing the tokens. +pub(crate) enum StripTokens { + /// Strip both shebang and frontmatter. + ShebangAndFrontmatter, + /// Strip the shebang but not frontmatter. + /// + /// That means that char sequences looking like frontmatter are simply + /// interpreted as regular Rust lexemes. + Shebang, + /// Strip nothing. + /// + /// In other words, char sequences looking like a shebang or frontmatter + /// are simply interpreted as regular Rust lexemes. + Nothing, +} + pub(crate) fn lex_token_trees<'psess, 'src>( psess: &'psess ParseSess, mut src: &'src str, mut start_pos: BytePos, override_span: Option<Span>, - frontmatter_allowed: FrontmatterAllowed, + strip_tokens: StripTokens, ) -> Result<TokenStream, Vec<Diag<'psess>>> { - // Skip `#!`, if present. - if let Some(shebang_len) = rustc_lexer::strip_shebang(src) { - src = &src[shebang_len..]; - start_pos = start_pos + BytePos::from_usize(shebang_len); + match strip_tokens { + StripTokens::Shebang | StripTokens::ShebangAndFrontmatter => { + if let Some(shebang_len) = rustc_lexer::strip_shebang(src) { + src = &src[shebang_len..]; + start_pos = start_pos + BytePos::from_usize(shebang_len); + } + } + StripTokens::Nothing => {} } + let frontmatter_allowed = match strip_tokens { + StripTokens::ShebangAndFrontmatter => FrontmatterAllowed::Yes, + StripTokens::Shebang | StripTokens::Nothing => FrontmatterAllowed::No, + }; + let cursor = Cursor::new(src, frontmatter_allowed); let mut lexer = Lexer { psess, |
