8 files changed, 64 insertions, 14 deletions
diff --git a/compiler/rustc_expand/Cargo.toml b/compiler/rustc_expand/Cargo.toml
index 9bb7143af51..f897833d85c 100644
--- a/compiler/rustc_expand/Cargo.toml
+++ b/compiler/rustc_expand/Cargo.toml
@@ -29,6 +29,6 @@ rustc_serialize = { path = "../rustc_serialize" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
-thin-vec.workspace = true
-tracing.workspace = true
+thin-vec = "0.2.12"
+tracing = "0.1"
 # tidy-alphabetical-end
diff --git a/compiler/rustc_expand/src/base.rs b/compiler/rustc_expand/src/base.rs
index 8ff21509f4a..88f88f30a8c 100644
--- a/compiler/rustc_expand/src/base.rs
+++ b/compiler/rustc_expand/src/base.rs
@@ -18,13 +18,14 @@ use rustc_feature::Features;
 use rustc_hir as hir;
 use rustc_hir::attrs::{AttributeKind, CfgEntry, Deprecation};
 use rustc_hir::def::MacroKinds;
+use rustc_hir::limit::Limit;
 use rustc_hir::{Stability, find_attr};
 use rustc_lint_defs::RegisteredTools;
 use rustc_parse::MACRO_ARGUMENTS;
 use rustc_parse::parser::{ForceCollect, Parser};
+use rustc_session::Session;
 use rustc_session::config::CollapseMacroDebuginfo;
 use rustc_session::parse::ParseSess;
-use rustc_session::{Limit, Session};
 use rustc_span::def_id::{CrateNum, DefId, LocalDefId};
 use rustc_span::edition::Edition;
 use rustc_span::hygiene::{AstPass, ExpnData, ExpnKind, LocalExpnId, MacroKind};
diff --git a/compiler/rustc_expand/src/errors.rs b/compiler/rustc_expand/src/errors.rs
index ba9d76970f0..6eb5cd65846 100644
--- a/compiler/rustc_expand/src/errors.rs
+++ b/compiler/rustc_expand/src/errors.rs
@@ -2,8 +2,8 @@ use std::borrow::Cow;
 
 use rustc_ast::ast;
 use rustc_errors::codes::*;
+use rustc_hir::limit::Limit;
 use rustc_macros::{Diagnostic, Subdiagnostic};
-use rustc_session::Limit;
 use rustc_span::{Ident, MacroRulesNormalizedIdent, Span, Symbol};
 
 #[derive(Diagnostic)]
diff --git a/compiler/rustc_expand/src/expand.rs b/compiler/rustc_expand/src/expand.rs
index dbb4a9de9e0..38e057d2776 100644
--- a/compiler/rustc_expand/src/expand.rs
+++ b/compiler/rustc_expand/src/expand.rs
@@ -19,14 +19,15 @@ use rustc_errors::PResult;
 use rustc_feature::Features;
 use rustc_hir::Target;
 use rustc_hir::def::MacroKinds;
+use rustc_hir::limit::Limit;
 use rustc_parse::parser::{
     AttemptLocalParseRecovery, CommaRecoveryMode, ForceCollect, Parser, RecoverColon, RecoverComma,
     token_descr,
 };
+use rustc_session::Session;
 use rustc_session::lint::BuiltinLintDiag;
 use rustc_session::lint::builtin::{UNUSED_ATTRIBUTES, UNUSED_DOC_COMMENTS};
 use rustc_session::parse::feature_err;
-use rustc_session::{Limit, Session};
 use rustc_span::hygiene::SyntaxContext;
 use rustc_span::{ErrorGuaranteed, FileName, Ident, LocalExpnId, Span, Symbol, sym};
 use smallvec::SmallVec;
@@ -2529,6 +2530,7 @@ impl ExpansionConfig<'_> {
         ExpansionConfig {
             crate_name,
             features,
+            // FIXME should this limit be configurable?
             recursion_limit: Limit::new(1024),
             trace_mac: false,
             should_test: false,
diff --git a/compiler/rustc_expand/src/mbe/macro_parser.rs b/compiler/rustc_expand/src/mbe/macro_parser.rs
index 0324057e331..ab8e059b7b7 100644
--- a/compiler/rustc_expand/src/mbe/macro_parser.rs
+++ b/compiler/rustc_expand/src/mbe/macro_parser.rs
@@ -77,7 +77,7 @@ use std::rc::Rc;
 
 pub(crate) use NamedMatch::*;
 pub(crate) use ParseResult::*;
-use rustc_ast::token::{self, DocComment, NonterminalKind, Token};
+use rustc_ast::token::{self, DocComment, NonterminalKind, Token, TokenKind};
 use rustc_data_structures::fx::FxHashMap;
 use rustc_errors::ErrorGuaranteed;
 use rustc_lint_defs::pluralize;
@@ -397,7 +397,23 @@ fn token_name_eq(t1: &Token, t2: &Token) -> bool {
     {
         ident1.name == ident2.name && is_raw1 == is_raw2
     } else {
-        t1.kind == t2.kind
+        // Note: we SHOULD NOT use `t1.kind == t2.kind` here, and we should instead compare the
+        // tokens using the special comparison logic below.
+        // It makes sure that variants containing `InvisibleOrigin` will
+        // never compare equal to one another.
+        //
+        // When we had AST-based nonterminals we couldn't compare them, and the
+        // old `Nonterminal` type had an `eq` that always returned false,
+        // resulting in this restriction:
+        // <https://doc.rust-lang.org/nightly/reference/macros-by-example.html#forwarding-a-matched-fragment>
+        // This comparison logic emulates that behaviour. We could consider lifting this
+        // restriction now but there are still cases involving invisible
+        // delimiters that make it harder than it first appears.
+        match (t1.kind, t2.kind) {
+            (TokenKind::OpenInvisible(_) | TokenKind::CloseInvisible(_), _)
+            | (_, TokenKind::OpenInvisible(_) | TokenKind::CloseInvisible(_)) => false,
+            (a, b) => a == b,
+        }
     }
 }
 
diff --git a/compiler/rustc_expand/src/mbe/transcribe.rs b/compiler/rustc_expand/src/mbe/transcribe.rs
index 174844d6ad6..ed8aa71d59d 100644
--- a/compiler/rustc_expand/src/mbe/transcribe.rs
+++ b/compiler/rustc_expand/src/mbe/transcribe.rs
@@ -556,7 +556,12 @@ fn metavar_expr_concat<'tx>(
                         };
                         match &named_matches[*curr_idx] {
                             // FIXME(c410-f3r) Nested repetitions are unimplemented
-                            MatchedSeq(_) => unimplemented!(),
+                            MatchedSeq(_) => {
+                                return Err(dcx.struct_span_err(
+                                    ident.span,
+                                    "nested repetitions with `${concat(...)}` metavariable expressions are not yet supported",
+                                ));
+                            }
                             MatchedSingle(pnr) => extract_symbol_from_pnr(dcx, pnr, ident.span)?,
                         }
                     }
@@ -935,11 +940,27 @@ fn extract_symbol_from_pnr<'a>(
         {
             Ok(*symbol)
         }
+        ParseNtResult::Literal(expr)
+            if let ExprKind::Lit(lit @ Lit { kind: LitKind::Integer, symbol, suffix }) =
+                &expr.kind =>
+        {
+            if lit.is_semantic_float() {
+                Err(dcx
+                    .struct_err("floats are not supported as metavariables of `${concat(..)}`")
+                    .with_span(span_err))
+            } else if suffix.is_none() {
+                Ok(*symbol)
+            } else {
+                Err(dcx
+                    .struct_err("integer metavariables of `${concat(..)}` must not be suffixed")
+                    .with_span(span_err))
+            }
+        }
         _ => Err(dcx
             .struct_err(
                 "metavariables of `${concat(..)}` must be of type `ident`, `literal` or `tt`",
             )
-            .with_note("currently only string literals are supported")
+            .with_note("currently only string and integer literals are supported")
             .with_span(span_err)),
     }
 }
diff --git a/compiler/rustc_expand/src/module.rs b/compiler/rustc_expand/src/module.rs
index 19f3cdbc549..79ab3cab22c 100644
--- a/compiler/rustc_expand/src/module.rs
+++ b/compiler/rustc_expand/src/module.rs
@@ -4,6 +4,7 @@ use std::path::{self, Path, PathBuf};
 use rustc_ast::{AttrVec, Attribute, Inline, Item, ModSpans};
 use rustc_attr_parsing::validate_attr;
 use rustc_errors::{Diag, ErrorGuaranteed};
+use rustc_parse::lexer::StripTokens;
 use rustc_parse::{exp, new_parser_from_file, unwrap_or_emit_fatal};
 use rustc_session::Session;
 use rustc_session::parse::ParseSess;
@@ -67,8 +68,12 @@ pub(crate) fn parse_external_mod(
         }
 
         // Actually parse the external file as a module.
-        let mut parser =
-            unwrap_or_emit_fatal(new_parser_from_file(&sess.psess, &mp.file_path, Some(span)));
+        let mut parser = unwrap_or_emit_fatal(new_parser_from_file(
+            &sess.psess,
+            &mp.file_path,
+            StripTokens::ShebangAndFrontmatter,
+            Some(span),
+        ));
         let (inner_attrs, items, inner_span) =
             parser.parse_mod(exp!(Eof)).map_err(|err| ModError::ParserError(err))?;
         attrs.extend(inner_attrs);
diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs
index 5b1d3d6d35b..295573f4492 100644
--- a/compiler/rustc_expand/src/proc_macro_server.rs
+++ b/compiler/rustc_expand/src/proc_macro_server.rs
@@ -8,7 +8,7 @@ use rustc_ast::util::literal::escape_byte_str_symbol;
 use rustc_ast_pretty::pprust;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_errors::{Diag, ErrorGuaranteed, MultiSpan, PResult};
-use rustc_parse::lexer::nfc_normalize;
+use rustc_parse::lexer::{StripTokens, nfc_normalize};
 use rustc_parse::parser::Parser;
 use rustc_parse::{exp, new_parser_from_source_str, source_str_to_stream, unwrap_or_emit_fatal};
 use rustc_proc_macro::bridge::{
@@ -485,8 +485,13 @@ impl server::FreeFunctions for Rustc<'_, '_> {
 
     fn literal_from_str(&mut self, s: &str) -> Result<Literal<Self::Span, Self::Symbol>, ()> {
         let name = FileName::proc_macro_source_code(s);
-        let mut parser =
-            unwrap_or_emit_fatal(new_parser_from_source_str(self.psess(), name, s.to_owned()));
+
+        let mut parser = unwrap_or_emit_fatal(new_parser_from_source_str(
+            self.psess(),
+            name,
+            s.to_owned(),
+            StripTokens::Nothing,
+        ));
 
         let first_span = parser.token.span.data();
         let minus_present = parser.eat(exp!(Minus));