Rollup merge of #146340 - fmease:frontmatter-containment, r=fee1-dead,Urgau

Strip frontmatter in fewer places

* Stop stripping frontmatter in `proc_macro::Literal::from_str` (RUST-146132)
* Stop stripping frontmatter in expr-ctxt (but not item-ctxt!) `include`s (RUST-145945)
* Stop stripping shebang (!) in `proc_macro::Literal::from_str`
  * Not a breaking change because it did compare spans already to ensure there wasn't extra whitespace or comments (`Literal::from_str("#!\n0")` already yields `Err(_)` thankfully!)
* Stop stripping frontmatter+shebang inside some rustdoc code where it doesn't make any observable difference (see self review comments)
* (Stop stripping frontmatter+shebang inside internal test code)

Fixes https://github.com/rust-lang/rust/issues/145945.
Fixes https://github.com/rust-lang/rust/issues/146132.

r? fee1-dead

4 files changed, 44 insertions, 26 deletions

diff --git a/src/librustdoc/clean/render_macro_matchers.rs b/src/librustdoc/clean/render_macro_matchers.rs
index d684e6f8650..b5a8d64ff4f 100644
--- a/src/librustdoc/clean/render_macro_matchers.rs
+++ b/src/librustdoc/clean/render_macro_matchers.rs
@@ -3,6 +3,7 @@ use rustc_ast::tokenstream::{TokenStream, TokenTree};
 use rustc_ast_pretty::pprust::PrintState;
 use rustc_ast_pretty::pprust::state::State as Printer;
 use rustc_middle::ty::TyCtxt;
+use rustc_parse::lexer::StripTokens;
 use rustc_session::parse::ParseSess;
 use rustc_span::symbol::{Ident, Symbol, kw};
 use rustc_span::{FileName, Span};
@@ -64,14 +65,18 @@ fn snippet_equal_to_token(tcx: TyCtxt<'_>, matcher: &TokenTree) -> Option<String
     // Create a Parser.
     let psess = ParseSess::new(rustc_driver::DEFAULT_LOCALE_RESOURCES.to_vec());
     let file_name = FileName::macro_expansion_source_code(&snippet);
-    let mut parser =
-        match rustc_parse::new_parser_from_source_str(&psess, file_name, snippet.clone()) {
-            Ok(parser) => parser,
-            Err(errs) => {
-                errs.into_iter().for_each(|err| err.cancel());
-                return None;
-            }
-        };
+    let mut parser = match rustc_parse::new_parser_from_source_str(
+        &psess,
+        file_name,
+        snippet.clone(),
+        StripTokens::Nothing,
+    ) {
+        Ok(parser) => parser,
+        Err(errs) => {
+            errs.into_iter().for_each(|err| err.cancel());
+            return None;
+        }
+    };
 
     // Reparse a single token tree.
     if parser.token == token::Eof {
diff --git a/src/librustdoc/doctest/make.rs b/src/librustdoc/doctest/make.rs
index f229f77c978..5eaadc9eb45 100644
--- a/src/librustdoc/doctest/make.rs
+++ b/src/librustdoc/doctest/make.rs
@@ -10,6 +10,7 @@ use rustc_ast::tokenstream::TokenTree;
 use rustc_ast::{self as ast, AttrStyle, HasAttrs, StmtKind};
 use rustc_errors::emitter::stderr_destination;
 use rustc_errors::{ColorConfig, DiagCtxtHandle};
+use rustc_parse::lexer::StripTokens;
 use rustc_parse::new_parser_from_source_str;
 use rustc_session::parse::ParseSess;
 use rustc_span::edition::{DEFAULT_EDITION, Edition};
@@ -468,14 +469,16 @@ fn parse_source(
     let dcx = DiagCtxt::new(Box::new(emitter)).disable_warnings();
     let psess = ParseSess::with_dcx(dcx, sm);
 
-    let mut parser = match new_parser_from_source_str(&psess, filename, wrapped_source) {
-        Ok(p) => p,
-        Err(errs) => {
-            errs.into_iter().for_each(|err| err.cancel());
-            reset_error_count(&psess);
-            return Err(());
-        }
-    };
+    // Don't strip any tokens; it wouldn't matter anyway because the source is wrapped in a function.
+    let mut parser =
+        match new_parser_from_source_str(&psess, filename, wrapped_source, StripTokens::Nothing) {
+            Ok(p) => p,
+            Err(errs) => {
+                errs.into_iter().for_each(|err| err.cancel());
+                reset_error_count(&psess);
+                return Err(());
+            }
+        };
 
     fn push_to_s(s: &mut String, source: &str, span: rustc_span::Span, prev_span_hi: &mut usize) {
         let extra_len = DOCTEST_CODE_WRAPPER.len();
diff --git a/src/tools/clippy/clippy_lints/src/doc/needless_doctest_main.rs b/src/tools/clippy/clippy_lints/src/doc/needless_doctest_main.rs
index 74283d7ba86..43bb9723555 100644
--- a/src/tools/clippy/clippy_lints/src/doc/needless_doctest_main.rs
+++ b/src/tools/clippy/clippy_lints/src/doc/needless_doctest_main.rs
@@ -8,6 +8,7 @@ use rustc_ast::{CoroutineKind, Fn, FnRetTy, Item, ItemKind};
 use rustc_errors::emitter::HumanEmitter;
 use rustc_errors::{Diag, DiagCtxt};
 use rustc_lint::LateContext;
+use rustc_parse::lexer::StripTokens;
 use rustc_parse::new_parser_from_source_str;
 use rustc_parse::parser::ForceCollect;
 use rustc_session::parse::ParseSess;
@@ -49,13 +50,14 @@ pub fn check(
                 let sm = Arc::new(SourceMap::new(FilePathMapping::empty()));
                 let psess = ParseSess::with_dcx(dcx, sm);
 
-                let mut parser = match new_parser_from_source_str(&psess, filename, code) {
-                    Ok(p) => p,
-                    Err(errs) => {
-                        errs.into_iter().for_each(Diag::cancel);
-                        return (false, test_attr_spans);
-                    },
-                };
+                let mut parser =
+                    match new_parser_from_source_str(&psess, filename, code, StripTokens::ShebangAndFrontmatter) {
+                        Ok(p) => p,
+                        Err(errs) => {
+                            errs.into_iter().for_each(Diag::cancel);
+                            return (false, test_attr_spans);
+                        },
+                    };
 
                 let mut relevant_main_found = false;
                 let mut eligible = true;
diff --git a/src/tools/rustfmt/src/parse/parser.rs b/src/tools/rustfmt/src/parse/parser.rs
index 2ec8769c45f..63c6c8c99d0 100644
--- a/src/tools/rustfmt/src/parse/parser.rs
+++ b/src/tools/rustfmt/src/parse/parser.rs
@@ -3,6 +3,7 @@ use std::path::{Path, PathBuf};
 
 use rustc_ast::{ast, attr};
 use rustc_errors::Diag;
+use rustc_parse::lexer::StripTokens;
 use rustc_parse::parser::Parser as RawParser;
 use rustc_parse::{exp, new_parser_from_file, new_parser_from_source_str, unwrap_or_emit_fatal};
 use rustc_span::{Span, sym};
@@ -64,11 +65,14 @@ impl<'a> ParserBuilder<'a> {
         input: Input,
     ) -> Result<RawParser<'a>, Vec<Diag<'a>>> {
         match input {
-            Input::File(ref file) => new_parser_from_file(psess, file, None),
+            Input::File(ref file) => {
+                new_parser_from_file(psess, file, StripTokens::ShebangAndFrontmatter, None)
+            }
             Input::Text(text) => new_parser_from_source_str(
                 psess,
                 rustc_span::FileName::Custom("stdin".to_owned()),
                 text,
+                StripTokens::ShebangAndFrontmatter,
             ),
         }
     }
@@ -104,8 +108,12 @@ impl<'a> Parser<'a> {
         span: Span,
     ) -> Result<(ast::AttrVec, ThinVec<Box<ast::Item>>, Span), ParserError> {
         let result = catch_unwind(AssertUnwindSafe(|| {
-            let mut parser =
-                unwrap_or_emit_fatal(new_parser_from_file(psess.inner(), path, Some(span)));
+            let mut parser = unwrap_or_emit_fatal(new_parser_from_file(
+                psess.inner(),
+                path,
+                StripTokens::ShebangAndFrontmatter,
+                Some(span),
+            ));
             match parser.parse_mod(exp!(Eof)) {
                 Ok((a, i, spans)) => Some((a, i, spans.inner_span)),
                 Err(e) => {