diff options
Diffstat (limited to 'compiler/rustc_expand/src/mbe/transcribe.rs')
| -rw-r--r-- | compiler/rustc_expand/src/mbe/transcribe.rs | 187 |
1 files changed, 106 insertions, 81 deletions
diff --git a/compiler/rustc_expand/src/mbe/transcribe.rs b/compiler/rustc_expand/src/mbe/transcribe.rs index dad83984c8b..e1f50876b05 100644 --- a/compiler/rustc_expand/src/mbe/transcribe.rs +++ b/compiler/rustc_expand/src/mbe/transcribe.rs @@ -1,4 +1,3 @@ -use crate::base::ExtCtxt; use crate::errors::{ CountRepetitionMisplaced, MetaVarExprUnrecognizedVar, MetaVarsDifSeqMatchers, MustRepeatOnce, NoSyntaxVarsExprRepeat, VarStillRepeating, @@ -9,12 +8,13 @@ use rustc_ast::mut_visit::{self, MutVisitor}; use rustc_ast::token::{self, Delimiter, Token, TokenKind}; use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree}; use rustc_data_structures::fx::FxHashMap; -use rustc_errors::{pluralize, Diag, PResult}; +use rustc_errors::{pluralize, Diag, DiagCtxt, PResult}; use rustc_parse::parser::ParseNtResult; use rustc_span::hygiene::{LocalExpnId, Transparency}; use rustc_span::symbol::{sym, Ident, MacroRulesNormalizedIdent}; use rustc_span::{with_metavar_spans, Span, SyntaxContext}; +use rustc_session::parse::ParseSess; use smallvec::{smallvec, SmallVec}; use std::mem; @@ -39,26 +39,32 @@ impl MutVisitor for Marker { } /// An iterator over the token trees in a delimited token tree (`{ ... }`) or a sequence (`$(...)`). -enum Frame<'a> { - Delimited { - tts: &'a [mbe::TokenTree], - idx: usize, - delim: Delimiter, - span: DelimSpan, - spacing: DelimSpacing, - }, - Sequence { - tts: &'a [mbe::TokenTree], - idx: usize, - sep: Option<Token>, - kleene_op: KleeneOp, - }, +struct Frame<'a> { + tts: &'a [mbe::TokenTree], + idx: usize, + kind: FrameKind, +} + +enum FrameKind { + Delimited { delim: Delimiter, span: DelimSpan, spacing: DelimSpacing }, + Sequence { sep: Option<Token>, kleene_op: KleeneOp }, } impl<'a> Frame<'a> { - /// Construct a new frame around the delimited set of tokens. - fn new(src: &'a mbe::Delimited, span: DelimSpan, spacing: DelimSpacing) -> Frame<'a> { - Frame::Delimited { tts: &src.tts, idx: 0, delim: src.delim, span, spacing } + fn new_delimited(src: &'a mbe::Delimited, span: DelimSpan, spacing: DelimSpacing) -> Frame<'a> { + Frame { + tts: &src.tts, + idx: 0, + kind: FrameKind::Delimited { delim: src.delim, span, spacing }, + } + } + + fn new_sequence( + src: &'a mbe::SequenceRepetition, + sep: Option<Token>, + kleene_op: KleeneOp, + ) -> Frame<'a> { + Frame { tts: &src.tts, idx: 0, kind: FrameKind::Sequence { sep, kleene_op } } } } @@ -66,13 +72,9 @@ impl<'a> Iterator for Frame<'a> { type Item = &'a mbe::TokenTree; fn next(&mut self) -> Option<&'a mbe::TokenTree> { - match self { - Frame::Delimited { tts, idx, .. } | Frame::Sequence { tts, idx, .. } => { - let res = tts.get(*idx); - *idx += 1; - res - } - } + let res = self.tts.get(self.idx); + self.idx += 1; + res } } @@ -97,11 +99,12 @@ impl<'a> Iterator for Frame<'a> { /// /// Along the way, we do some additional error checking. pub(super) fn transcribe<'a>( - cx: &ExtCtxt<'a>, + psess: &'a ParseSess, interp: &FxHashMap<MacroRulesNormalizedIdent, NamedMatch>, src: &mbe::Delimited, src_span: DelimSpan, transparency: Transparency, + expand_id: LocalExpnId, ) -> PResult<'a, TokenStream> { // Nothing for us to transcribe... if src.tts.is_empty() { @@ -111,13 +114,16 @@ pub(super) fn transcribe<'a>( // We descend into the RHS (`src`), expanding things as we go. This stack contains the things // we have yet to expand/are still expanding. We start the stack off with the whole RHS. The // choice of spacing values doesn't matter. - let mut stack: SmallVec<[Frame<'_>; 1]> = - smallvec![Frame::new(src, src_span, DelimSpacing::new(Spacing::Alone, Spacing::Alone))]; + let mut stack: SmallVec<[Frame<'_>; 1]> = smallvec![Frame::new_delimited( + src, + src_span, + DelimSpacing::new(Spacing::Alone, Spacing::Alone) + )]; // As we descend in the RHS, we will need to be able to match nested sequences of matchers. // `repeats` keeps track of where we are in matching at each level, with the last element being // the most deeply nested sequence. This is used as a stack. - let mut repeats = Vec::new(); + let mut repeats: Vec<(usize, usize)> = Vec::new(); // `result` contains resulting token stream from the TokenTree we just finished processing. At // the end, this will contain the full result of transcription, but at arbitrary points during @@ -132,8 +138,9 @@ pub(super) fn transcribe<'a>( // again, and we are done transcribing. let mut result: Vec<TokenTree> = Vec::new(); let mut result_stack = Vec::new(); - let mut marker = Marker(cx.current_expansion.id, transparency, Default::default()); + let mut marker = Marker(expand_id, transparency, Default::default()); + let dcx = &psess.dcx; loop { // Look at the last frame on the stack. // If it still has a TokenTree we have not looked at yet, use that tree. @@ -142,11 +149,12 @@ pub(super) fn transcribe<'a>( // Otherwise, if we have just reached the end of a sequence and we can keep repeating, // go back to the beginning of the sequence. - if let Frame::Sequence { idx, sep, .. } = stack.last_mut().unwrap() { + let frame = stack.last_mut().unwrap(); + if let FrameKind::Sequence { sep, .. } = &frame.kind { let (repeat_idx, repeat_len) = repeats.last_mut().unwrap(); *repeat_idx += 1; if repeat_idx < repeat_len { - *idx = 0; + frame.idx = 0; if let Some(sep) = sep { result.push(TokenTree::Token(sep.clone(), Spacing::Alone)); } @@ -157,16 +165,16 @@ pub(super) fn transcribe<'a>( // We are done with the top of the stack. Pop it. Depending on what it was, we do // different things. Note that the outermost item must be the delimited, wrapped RHS // that was passed in originally to `transcribe`. - match stack.pop().unwrap() { + match stack.pop().unwrap().kind { // Done with a sequence. Pop from repeats. - Frame::Sequence { .. } => { + FrameKind::Sequence { .. } => { repeats.pop(); } // We are done processing a Delimited. If this is the top-level delimited, we are // done. Otherwise, we unwind the result_stack to append what we have produced to // any previous results. - Frame::Delimited { delim, span, mut spacing, .. } => { + FrameKind::Delimited { delim, span, mut spacing, .. } => { // Hack to force-insert a space after `]` in certain case. // See discussion of the `hex-literal` crate in #114571. if delim == Delimiter::Bracket { @@ -192,12 +200,10 @@ pub(super) fn transcribe<'a>( // We are descending into a sequence. We first make sure that the matchers in the RHS // and the matches in `interp` have the same shape. Otherwise, either the caller or the // macro writer has made a mistake. - seq @ mbe::TokenTree::Sequence(_, delimited) => { + seq @ mbe::TokenTree::Sequence(_, seq_rep) => { match lockstep_iter_size(seq, interp, &repeats) { LockstepIterSize::Unconstrained => { - return Err(cx - .dcx() - .create_err(NoSyntaxVarsExprRepeat { span: seq.span() })); + return Err(dcx.create_err(NoSyntaxVarsExprRepeat { span: seq.span() })); } LockstepIterSize::Contradiction(msg) => { @@ -205,9 +211,9 @@ pub(super) fn transcribe<'a>( // happens when two meta-variables are used in the same repetition in a // sequence, but they come from different sequence matchers and repeat // different amounts. - return Err(cx - .dcx() - .create_err(MetaVarsDifSeqMatchers { span: seq.span(), msg })); + return Err( + dcx.create_err(MetaVarsDifSeqMatchers { span: seq.span(), msg }) + ); } LockstepIterSize::Constraint(len, _) => { @@ -221,9 +227,7 @@ pub(super) fn transcribe<'a>( // FIXME: this really ought to be caught at macro definition // time... It happens when the Kleene operator in the matcher and // the body for the same meta-variable do not match. - return Err(cx - .dcx() - .create_err(MustRepeatOnce { span: sp.entire() })); + return Err(dcx.create_err(MustRepeatOnce { span: sp.entire() })); } } else { // 0 is the initial counter (we have done 0 repetitions so far). `len` @@ -233,12 +237,11 @@ pub(super) fn transcribe<'a>( // The first time we encounter the sequence we push it to the stack. It // then gets reused (see the beginning of the loop) until we are done // repeating. - stack.push(Frame::Sequence { - idx: 0, - sep: seq.separator.clone(), - tts: &delimited.tts, - kleene_op: seq.kleene.op, - }); + stack.push(Frame::new_sequence( + seq_rep, + seq.separator.clone(), + seq.kleene.op, + )); } } } @@ -248,13 +251,38 @@ pub(super) fn transcribe<'a>( mbe::TokenTree::MetaVar(mut sp, mut original_ident) => { // Find the matched nonterminal from the macro invocation, and use it to replace // the meta-var. + // + // We use `Spacing::Alone` everywhere here, because that's the conservative choice + // and spacing of declarative macros is tricky. E.g. in this macro: + // ``` + // macro_rules! idents { + // ($($a:ident,)*) => { stringify!($($a)*) } + // } + // ``` + // `$a` has no whitespace after it and will be marked `JointHidden`. If you then + // call `idents!(x,y,z,)`, each of `x`, `y`, and `z` will be marked as `Joint`. So + // if you choose to use `$x`'s spacing or the identifier's spacing, you'll end up + // producing "xyz", which is bad because it effectively merges tokens. + // `Spacing::Alone` is the safer option. Fortunately, `space_between` will avoid + // some of the unnecessary whitespace. let ident = MacroRulesNormalizedIdent::new(original_ident); if let Some(cur_matched) = lookup_cur_matched(ident, interp, &repeats) { + // njn: explain the use of alone here let tt = match cur_matched { MatchedSingle(ParseNtResult::Tt(tt)) => { // `tt`s are emitted into the output stream directly as "raw tokens", // without wrapping them into groups. - maybe_use_metavar_location(cx, &stack, sp, tt, &mut marker) + maybe_use_metavar_location(psess, &stack, sp, tt, &mut marker) + } + MatchedSingle(ParseNtResult::Ident(ident, is_raw)) => { + marker.visit_span(&mut sp); + let kind = token::NtIdent(*ident, *is_raw); + TokenTree::token_alone(kind, sp) + } + MatchedSingle(ParseNtResult::Lifetime(ident)) => { + marker.visit_span(&mut sp); + let kind = token::NtLifetime(*ident); + TokenTree::token_alone(kind, sp) } MatchedSingle(ParseNtResult::Nt(nt)) => { // Other variables are emitted into the output stream as groups with @@ -265,7 +293,7 @@ pub(super) fn transcribe<'a>( } MatchedSeq(..) => { // We were unable to descend far enough. This is an error. - return Err(cx.dcx().create_err(VarStillRepeating { span: sp, ident })); + return Err(dcx.create_err(VarStillRepeating { span: sp, ident })); } }; result.push(tt) @@ -284,7 +312,7 @@ pub(super) fn transcribe<'a>( // Replace meta-variable expressions with the result of their expansion. mbe::TokenTree::MetaVarExpr(sp, expr) => { - transcribe_metavar_expr(cx, expr, interp, &mut marker, &repeats, &mut result, sp)?; + transcribe_metavar_expr(dcx, expr, interp, &mut marker, &repeats, &mut result, sp)?; } // If we are entering a new delimiter, we push its contents to the `stack` to be @@ -294,13 +322,7 @@ pub(super) fn transcribe<'a>( // the previous results (from outside the Delimited). mbe::TokenTree::Delimited(mut span, spacing, delimited) => { mut_visit::visit_delim_span(&mut span, &mut marker); - stack.push(Frame::Delimited { - tts: &delimited.tts, - delim: delimited.delim, - idx: 0, - span, - spacing: *spacing, - }); + stack.push(Frame::new_delimited(delimited, span, *spacing)); result_stack.push(mem::take(&mut result)); } @@ -350,7 +372,7 @@ pub(super) fn transcribe<'a>( /// combine with each other and not with tokens outside of the sequence. /// - The metavariable span comes from a different crate, then we prefer the more local span. fn maybe_use_metavar_location( - cx: &ExtCtxt<'_>, + psess: &ParseSess, stack: &[Frame<'_>], mut metavar_span: Span, orig_tt: &TokenTree, @@ -358,10 +380,13 @@ fn maybe_use_metavar_location( ) -> TokenTree { let undelimited_seq = matches!( stack.last(), - Some(Frame::Sequence { + Some(Frame { tts: [_], - sep: None, - kleene_op: KleeneOp::ZeroOrMore | KleeneOp::OneOrMore, + kind: FrameKind::Sequence { + sep: None, + kleene_op: KleeneOp::ZeroOrMore | KleeneOp::OneOrMore, + .. + }, .. }) ); @@ -385,7 +410,7 @@ fn maybe_use_metavar_location( && insert(mspans, dspan.entire(), metavar_span) }), }; - if no_collision || cx.source_map().is_imported(metavar_span) { + if no_collision || psess.source_map().is_imported(metavar_span) { return orig_tt.clone(); } @@ -546,7 +571,7 @@ fn lockstep_iter_size( /// * `[ $( ${count(foo, 1)} ),* ]` will return an error because `${count(foo, 1)}` is /// declared inside a single repetition and the index `1` implies two nested repetitions. fn count_repetitions<'a>( - cx: &ExtCtxt<'a>, + dcx: &'a DiagCtxt, depth_user: usize, mut matched: &NamedMatch, repeats: &[(usize, usize)], @@ -583,7 +608,7 @@ fn count_repetitions<'a>( .and_then(|el| el.checked_sub(repeats.len())) .unwrap_or_default(); if depth_user > depth_max { - return Err(out_of_bounds_err(cx, depth_max + 1, sp.entire(), "count")); + return Err(out_of_bounds_err(dcx, depth_max + 1, sp.entire(), "count")); } // `repeats` records all of the nested levels at which we are currently @@ -599,7 +624,7 @@ fn count_repetitions<'a>( } if let MatchedSingle(_) = matched { - return Err(cx.dcx().create_err(CountRepetitionMisplaced { span: sp.entire() })); + return Err(dcx.create_err(CountRepetitionMisplaced { span: sp.entire() })); } count(depth_user, depth_max, matched) @@ -607,7 +632,7 @@ fn count_repetitions<'a>( /// Returns a `NamedMatch` item declared on the LHS given an arbitrary [Ident] fn matched_from_ident<'ctx, 'interp, 'rslt>( - cx: &ExtCtxt<'ctx>, + dcx: &'ctx DiagCtxt, ident: Ident, interp: &'interp FxHashMap<MacroRulesNormalizedIdent, NamedMatch>, ) -> PResult<'ctx, &'rslt NamedMatch> @@ -616,12 +641,12 @@ where { let span = ident.span; let key = MacroRulesNormalizedIdent::new(ident); - interp.get(&key).ok_or_else(|| cx.dcx().create_err(MetaVarExprUnrecognizedVar { span, key })) + interp.get(&key).ok_or_else(|| dcx.create_err(MetaVarExprUnrecognizedVar { span, key })) } /// Used by meta-variable expressions when an user input is out of the actual declared bounds. For /// example, index(999999) in an repetition of only three elements. -fn out_of_bounds_err<'a>(cx: &ExtCtxt<'a>, max: usize, span: Span, ty: &str) -> Diag<'a> { +fn out_of_bounds_err<'a>(dcx: &'a DiagCtxt, max: usize, span: Span, ty: &str) -> Diag<'a> { let msg = if max == 0 { format!( "meta-variable expression `{ty}` with depth parameter \ @@ -633,11 +658,11 @@ fn out_of_bounds_err<'a>(cx: &ExtCtxt<'a>, max: usize, span: Span, ty: &str) -> must be less than {max}" ) }; - cx.dcx().struct_span_err(span, msg) + dcx.struct_span_err(span, msg) } fn transcribe_metavar_expr<'a>( - cx: &ExtCtxt<'a>, + dcx: &'a DiagCtxt, expr: &MetaVarExpr, interp: &FxHashMap<MacroRulesNormalizedIdent, NamedMatch>, marker: &mut Marker, @@ -652,8 +677,8 @@ fn transcribe_metavar_expr<'a>( }; match *expr { MetaVarExpr::Count(original_ident, depth) => { - let matched = matched_from_ident(cx, original_ident, interp)?; - let count = count_repetitions(cx, depth, matched, repeats, sp)?; + let matched = matched_from_ident(dcx, original_ident, interp)?; + let count = count_repetitions(dcx, depth, matched, repeats, sp)?; let tt = TokenTree::token_alone( TokenKind::lit(token::Integer, sym::integer(count), None), visited_span(), @@ -662,7 +687,7 @@ fn transcribe_metavar_expr<'a>( } MetaVarExpr::Ignore(original_ident) => { // Used to ensure that `original_ident` is present in the LHS - let _ = matched_from_ident(cx, original_ident, interp)?; + let _ = matched_from_ident(dcx, original_ident, interp)?; } MetaVarExpr::Index(depth) => match repeats.iter().nth_back(depth) { Some((index, _)) => { @@ -671,16 +696,16 @@ fn transcribe_metavar_expr<'a>( visited_span(), )); } - None => return Err(out_of_bounds_err(cx, repeats.len(), sp.entire(), "index")), + None => return Err(out_of_bounds_err(dcx, repeats.len(), sp.entire(), "index")), }, - MetaVarExpr::Length(depth) => match repeats.iter().nth_back(depth) { + MetaVarExpr::Len(depth) => match repeats.iter().nth_back(depth) { Some((_, length)) => { result.push(TokenTree::token_alone( TokenKind::lit(token::Integer, sym::integer(*length), None), visited_span(), )); } - None => return Err(out_of_bounds_err(cx, repeats.len(), sp.entire(), "length")), + None => return Err(out_of_bounds_err(dcx, repeats.len(), sp.entire(), "len")), }, } Ok(()) |