Rollup merge of #146340 - fmease:frontmatter-containment, r=fee1-dead,Urgau

Strip frontmatter in fewer places

* Stop stripping frontmatter in `proc_macro::Literal::from_str` (RUST-146132)
* Stop stripping frontmatter in expr-ctxt (but not item-ctxt!) `include`s (RUST-145945)
* Stop stripping shebang (!) in `proc_macro::Literal::from_str`
  * Not a breaking change because it did compare spans already to ensure there wasn't extra whitespace or comments (`Literal::from_str("#!\n0")` already yields `Err(_)` thankfully!)
* Stop stripping frontmatter+shebang inside some rustdoc code where it doesn't make any observable difference (see self review comments)
* (Stop stripping frontmatter+shebang inside internal test code)

Fixes https://github.com/rust-lang/rust/issues/145945.
Fixes https://github.com/rust-lang/rust/issues/146132.

r? fee1-dead

2 files changed, 15 insertions, 5 deletions

diff --git a/compiler/rustc_expand/src/module.rs b/compiler/rustc_expand/src/module.rs
index 19f3cdbc549..79ab3cab22c 100644
--- a/compiler/rustc_expand/src/module.rs
+++ b/compiler/rustc_expand/src/module.rs
@@ -4,6 +4,7 @@ use std::path::{self, Path, PathBuf};
 use rustc_ast::{AttrVec, Attribute, Inline, Item, ModSpans};
 use rustc_attr_parsing::validate_attr;
 use rustc_errors::{Diag, ErrorGuaranteed};
+use rustc_parse::lexer::StripTokens;
 use rustc_parse::{exp, new_parser_from_file, unwrap_or_emit_fatal};
 use rustc_session::Session;
 use rustc_session::parse::ParseSess;
@@ -67,8 +68,12 @@ pub(crate) fn parse_external_mod(
         }
 
         // Actually parse the external file as a module.
-        let mut parser =
-            unwrap_or_emit_fatal(new_parser_from_file(&sess.psess, &mp.file_path, Some(span)));
+        let mut parser = unwrap_or_emit_fatal(new_parser_from_file(
+            &sess.psess,
+            &mp.file_path,
+            StripTokens::ShebangAndFrontmatter,
+            Some(span),
+        ));
         let (inner_attrs, items, inner_span) =
             parser.parse_mod(exp!(Eof)).map_err(|err| ModError::ParserError(err))?;
         attrs.extend(inner_attrs);
diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs
index 5b1d3d6d35b..295573f4492 100644
--- a/compiler/rustc_expand/src/proc_macro_server.rs
+++ b/compiler/rustc_expand/src/proc_macro_server.rs
@@ -8,7 +8,7 @@ use rustc_ast::util::literal::escape_byte_str_symbol;
 use rustc_ast_pretty::pprust;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_errors::{Diag, ErrorGuaranteed, MultiSpan, PResult};
-use rustc_parse::lexer::nfc_normalize;
+use rustc_parse::lexer::{StripTokens, nfc_normalize};
 use rustc_parse::parser::Parser;
 use rustc_parse::{exp, new_parser_from_source_str, source_str_to_stream, unwrap_or_emit_fatal};
 use rustc_proc_macro::bridge::{
@@ -485,8 +485,13 @@ impl server::FreeFunctions for Rustc<'_, '_> {
 
     fn literal_from_str(&mut self, s: &str) -> Result<Literal<Self::Span, Self::Symbol>, ()> {
         let name = FileName::proc_macro_source_code(s);
-        let mut parser =
-            unwrap_or_emit_fatal(new_parser_from_source_str(self.psess(), name, s.to_owned()));
+
+        let mut parser = unwrap_or_emit_fatal(new_parser_from_source_str(
+            self.psess(),
+            name,
+            s.to_owned(),
+            StripTokens::Nothing,
+        ));
 
         let first_span = parser.token.span.data();
         let minus_present = parser.eat(exp!(Minus));