From 7a66925a8138df105f77d646f9206024baf7ea4b Mon Sep 17 00:00:00 2001 From: León Orell Valerian Liehr Date: Mon, 8 Sep 2025 19:25:22 +0200 Subject: Strip frontmatter in fewer places --- compiler/rustc_parse/src/lexer/mod.rs | 2 +- compiler/rustc_parse/src/lib.rs | 36 ++++++++++++++------------------ compiler/rustc_parse/src/parser/tests.rs | 7 +++++-- 3 files changed, 22 insertions(+), 23 deletions(-) (limited to 'compiler/rustc_parse') diff --git a/compiler/rustc_parse/src/lexer/mod.rs b/compiler/rustc_parse/src/lexer/mod.rs index f5f081efc49..51019db7c00 100644 --- a/compiler/rustc_parse/src/lexer/mod.rs +++ b/compiler/rustc_parse/src/lexer/mod.rs @@ -45,7 +45,7 @@ pub(crate) struct UnmatchedDelim { } /// Which tokens should be stripped before lexing the tokens. -pub(crate) enum StripTokens { +pub enum StripTokens { /// Strip both shebang and frontmatter. ShebangAndFrontmatter, /// Strip the shebang but not frontmatter. diff --git a/compiler/rustc_parse/src/lib.rs b/compiler/rustc_parse/src/lib.rs index d8792d7af4c..88b67d792de 100644 --- a/compiler/rustc_parse/src/lib.rs +++ b/compiler/rustc_parse/src/lib.rs @@ -54,29 +54,18 @@ pub fn unwrap_or_emit_fatal(expr: Result>>) -> T { } } -/// Creates a new parser from a source string. On failure, the errors must be consumed via -/// `unwrap_or_emit_fatal`, `emit`, `cancel`, etc., otherwise a panic will occur when they are -/// dropped. -pub fn new_parser_from_source_str( - psess: &ParseSess, - name: FileName, - source: String, -) -> Result, Vec>> { - let source_file = psess.source_map().new_source_file(name, source); - new_parser_from_source_file(psess, source_file, StripTokens::ShebangAndFrontmatter) -} - -/// Creates a new parser from a simple (no shebang, no frontmatter) source string. +/// Creates a new parser from a source string. /// /// On failure, the errors must be consumed via `unwrap_or_emit_fatal`, `emit`, `cancel`, /// etc., otherwise a panic will occur when they are dropped. -pub fn new_parser_from_simple_source_str( +pub fn new_parser_from_source_str( psess: &ParseSess, name: FileName, source: String, + strip_tokens: StripTokens, ) -> Result, Vec>> { let source_file = psess.source_map().new_source_file(name, source); - new_parser_from_source_file(psess, source_file, StripTokens::Nothing) + new_parser_from_source_file(psess, source_file, strip_tokens) } /// Creates a new parser from a filename. On failure, the errors must be consumed via @@ -87,6 +76,7 @@ pub fn new_parser_from_simple_source_str( pub fn new_parser_from_file<'a>( psess: &'a ParseSess, path: &Path, + strip_tokens: StripTokens, sp: Option, ) -> Result, Vec>> { let sm = psess.source_map(); @@ -110,7 +100,7 @@ pub fn new_parser_from_file<'a>( } err.emit(); }); - new_parser_from_source_file(psess, source_file, StripTokens::ShebangAndFrontmatter) + new_parser_from_source_file(psess, source_file, strip_tokens) } pub fn utf8_error( @@ -172,6 +162,9 @@ fn new_parser_from_source_file( Ok(parser) } +/// Given a source string, produces a sequence of token trees. +/// +/// NOTE: This only strips shebangs, not frontmatter! pub fn source_str_to_stream( psess: &ParseSess, name: FileName, @@ -179,13 +172,16 @@ pub fn source_str_to_stream( override_span: Option, ) -> Result>> { let source_file = psess.source_map().new_source_file(name, source); - // used mainly for `proc_macro` and the likes, not for our parsing purposes, so don't parse - // frontmatters as frontmatters, but for compatibility reason still strip the shebang + // FIXME(frontmatter): Consider stripping frontmatter in a future edition. We can't strip them + // in the current edition since that would be breaking. + // See also . + // Alternatively, stop stripping shebangs here, too, if T-lang and crater approve. source_file_to_stream(psess, source_file, override_span, StripTokens::Shebang) } -/// Given a source file, produces a sequence of token trees. Returns any buffered errors from -/// parsing the token stream. +/// Given a source file, produces a sequence of token trees. +/// +/// Returns any buffered errors from parsing the token stream. fn source_file_to_stream<'psess>( psess: &'psess ParseSess, source_file: Arc, diff --git a/compiler/rustc_parse/src/parser/tests.rs b/compiler/rustc_parse/src/parser/tests.rs index a6e7266e71b..e645fb47b9e 100644 --- a/compiler/rustc_parse/src/parser/tests.rs +++ b/compiler/rustc_parse/src/parser/tests.rs @@ -22,6 +22,7 @@ use rustc_span::{ }; use termcolor::WriteColor; +use crate::lexer::StripTokens; use crate::parser::{ForceCollect, Parser}; use crate::{new_parser_from_source_str, source_str_to_stream, unwrap_or_emit_fatal}; @@ -35,6 +36,7 @@ fn string_to_parser(psess: &ParseSess, source_str: String) -> Parser<'_> { psess, PathBuf::from("bogofile").into(), source_str, + StripTokens::Nothing, )) } @@ -2240,7 +2242,7 @@ fn parse_item_from_source_str( source: String, psess: &ParseSess, ) -> PResult<'_, Option>> { - unwrap_or_emit_fatal(new_parser_from_source_str(psess, name, source)) + unwrap_or_emit_fatal(new_parser_from_source_str(psess, name, source, StripTokens::Nothing)) .parse_item(ForceCollect::No) } @@ -2520,7 +2522,8 @@ fn ttdelim_span() { source: String, psess: &ParseSess, ) -> PResult<'_, Box> { - unwrap_or_emit_fatal(new_parser_from_source_str(psess, name, source)).parse_expr() + unwrap_or_emit_fatal(new_parser_from_source_str(psess, name, source, StripTokens::Nothing)) + .parse_expr() } create_default_session_globals_then(|| { -- cgit 1.4.1-3-g733a5