diff options
| author | Nicholas Nethercote <n.nethercote@gmail.com> | 2023-10-12 15:36:14 +1100 |
|---|---|---|
| committer | Nicholas Nethercote <n.nethercote@gmail.com> | 2023-12-11 09:36:40 +1100 |
| commit | 4cfdbd328b7171b2328d11b950b1af0978d6b1ef (patch) | |
| tree | 544de5216adec4838810b031c084a025a787573d /compiler/rustc_expand/src | |
| parent | 925f7fad576b6a8659d93846faf8d9610e59bab0 (diff) | |
| download | rust-4cfdbd328b7171b2328d11b950b1af0978d6b1ef.tar.gz rust-4cfdbd328b7171b2328d11b950b1af0978d6b1ef.zip | |
Add spacing information to delimiters.
This is an extension of the previous commit. It means the output of something like this: ``` stringify!(let a: Vec<u32> = vec![];) ``` goes from this: ``` let a: Vec<u32> = vec![] ; ``` With this PR, it now produces this string: ``` let a: Vec<u32> = vec![]; ```
Diffstat (limited to 'compiler/rustc_expand/src')
| -rw-r--r-- | compiler/rustc_expand/src/config.rs | 10 | ||||
| -rw-r--r-- | compiler/rustc_expand/src/mbe.rs | 6 | ||||
| -rw-r--r-- | compiler/rustc_expand/src/mbe/macro_check.rs | 12 | ||||
| -rw-r--r-- | compiler/rustc_expand/src/mbe/macro_parser.rs | 4 | ||||
| -rw-r--r-- | compiler/rustc_expand/src/mbe/macro_rules.rs | 18 | ||||
| -rw-r--r-- | compiler/rustc_expand/src/mbe/metavar_expr.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_expand/src/mbe/quoted.rs | 7 | ||||
| -rw-r--r-- | compiler/rustc_expand/src/mbe/transcribe.rs | 40 | ||||
| -rw-r--r-- | compiler/rustc_expand/src/parse/tests.rs | 15 | ||||
| -rw-r--r-- | compiler/rustc_expand/src/proc_macro_server.rs | 14 |
10 files changed, 78 insertions, 50 deletions
diff --git a/compiler/rustc_expand/src/config.rs b/compiler/rustc_expand/src/config.rs index 3106fad0432..b4693f5a71a 100644 --- a/compiler/rustc_expand/src/config.rs +++ b/compiler/rustc_expand/src/config.rs @@ -6,8 +6,7 @@ use crate::errors::{ }; use rustc_ast::ptr::P; use rustc_ast::token::{Delimiter, Token, TokenKind}; -use rustc_ast::tokenstream::{AttrTokenStream, AttrTokenTree}; -use rustc_ast::tokenstream::{DelimSpan, Spacing}; +use rustc_ast::tokenstream::{AttrTokenStream, AttrTokenTree, DelimSpacing, DelimSpan, Spacing}; use rustc_ast::tokenstream::{LazyAttrTokenStream, TokenTree}; use rustc_ast::NodeId; use rustc_ast::{self as ast, AttrStyle, Attribute, HasAttrs, HasTokens, MetaItem}; @@ -242,7 +241,7 @@ impl<'a> StripUnconfigured<'a> { stream.0.iter().all(|tree| match tree { AttrTokenTree::Attributes(_) => false, AttrTokenTree::Token(..) => true, - AttrTokenTree::Delimited(_, _, inner) => can_skip(inner), + AttrTokenTree::Delimited(.., inner) => can_skip(inner), }) } @@ -266,9 +265,9 @@ impl<'a> StripUnconfigured<'a> { None.into_iter() } } - AttrTokenTree::Delimited(sp, delim, mut inner) => { + AttrTokenTree::Delimited(sp, spacing, delim, mut inner) => { inner = self.configure_tokens(&inner); - Some(AttrTokenTree::Delimited(sp, delim, inner)).into_iter() + Some(AttrTokenTree::Delimited(sp, spacing, delim, inner)).into_iter() } AttrTokenTree::Token(ref token, _) if let TokenKind::Interpolated(nt) = &token.kind => @@ -376,6 +375,7 @@ impl<'a> StripUnconfigured<'a> { // in `#[attr]`, so just use the span of the `#` token. let bracket_group = AttrTokenTree::Delimited( DelimSpan::from_single(pound_span), + DelimSpacing::new(Spacing::JointHidden, Spacing::Alone), Delimiter::Bracket, item.tokens .as_ref() diff --git a/compiler/rustc_expand/src/mbe.rs b/compiler/rustc_expand/src/mbe.rs index a43b2a00188..ca4a1f327ad 100644 --- a/compiler/rustc_expand/src/mbe.rs +++ b/compiler/rustc_expand/src/mbe.rs @@ -13,7 +13,7 @@ pub(crate) mod transcribe; use metavar_expr::MetaVarExpr; use rustc_ast::token::{Delimiter, NonterminalKind, Token, TokenKind}; -use rustc_ast::tokenstream::DelimSpan; +use rustc_ast::tokenstream::{DelimSpacing, DelimSpan}; use rustc_span::symbol::Ident; use rustc_span::Span; @@ -68,7 +68,7 @@ pub(crate) enum KleeneOp { enum TokenTree { Token(Token), /// A delimited sequence, e.g. `($e:expr)` (RHS) or `{ $e }` (LHS). - Delimited(DelimSpan, Delimited), + Delimited(DelimSpan, DelimSpacing, Delimited), /// A kleene-style repetition sequence, e.g. `$($e:expr)*` (RHS) or `$($e),*` (LHS). Sequence(DelimSpan, SequenceRepetition), /// e.g., `$var`. @@ -99,7 +99,7 @@ impl TokenTree { TokenTree::Token(Token { span, .. }) | TokenTree::MetaVar(span, _) | TokenTree::MetaVarDecl(span, _, _) => span, - TokenTree::Delimited(span, _) + TokenTree::Delimited(span, ..) | TokenTree::MetaVarExpr(span, _) | TokenTree::Sequence(span, _) => span.entire(), } diff --git a/compiler/rustc_expand/src/mbe/macro_check.rs b/compiler/rustc_expand/src/mbe/macro_check.rs index 42c91824baf..0b1f25b67c8 100644 --- a/compiler/rustc_expand/src/mbe/macro_check.rs +++ b/compiler/rustc_expand/src/mbe/macro_check.rs @@ -290,7 +290,7 @@ fn check_binders( } // `MetaVarExpr` can not appear in the LHS of a macro arm TokenTree::MetaVarExpr(..) => {} - TokenTree::Delimited(_, ref del) => { + TokenTree::Delimited(.., ref del) => { for tt in &del.tts { check_binders(sess, node_id, tt, macros, binders, ops, valid); } @@ -353,7 +353,7 @@ fn check_occurrences( }; check_ops_is_prefix(sess, node_id, macros, binders, ops, dl.entire(), name); } - TokenTree::Delimited(_, ref del) => { + TokenTree::Delimited(.., ref del) => { check_nested_occurrences(sess, node_id, &del.tts, macros, binders, ops, valid); } TokenTree::Sequence(_, ref seq) => { @@ -435,8 +435,8 @@ fn check_nested_occurrences( // We check that the meta-variable is correctly used. check_occurrences(sess, node_id, tt, macros, binders, ops, valid); } - (NestedMacroState::MacroRulesNotName, TokenTree::Delimited(_, del)) - | (NestedMacroState::MacroName, TokenTree::Delimited(_, del)) + (NestedMacroState::MacroRulesNotName, TokenTree::Delimited(.., del)) + | (NestedMacroState::MacroName, TokenTree::Delimited(.., del)) if del.delim == Delimiter::Brace => { let macro_rules = state == NestedMacroState::MacroRulesNotName; @@ -466,7 +466,7 @@ fn check_nested_occurrences( // We check that the meta-variable is correctly used. check_occurrences(sess, node_id, tt, macros, binders, ops, valid); } - (NestedMacroState::MacroName, TokenTree::Delimited(_, del)) + (NestedMacroState::MacroName, TokenTree::Delimited(.., del)) if del.delim == Delimiter::Parenthesis => { state = NestedMacroState::MacroNameParen; @@ -481,7 +481,7 @@ fn check_nested_occurrences( valid, ); } - (NestedMacroState::MacroNameParen, TokenTree::Delimited(_, del)) + (NestedMacroState::MacroNameParen, TokenTree::Delimited(.., del)) if del.delim == Delimiter::Brace => { state = NestedMacroState::Empty; diff --git a/compiler/rustc_expand/src/mbe/macro_parser.rs b/compiler/rustc_expand/src/mbe/macro_parser.rs index 965beb9bf84..b248a1fe349 100644 --- a/compiler/rustc_expand/src/mbe/macro_parser.rs +++ b/compiler/rustc_expand/src/mbe/macro_parser.rs @@ -184,7 +184,7 @@ pub(super) fn compute_locs(matcher: &[TokenTree]) -> Vec<MatcherLoc> { TokenTree::Token(token) => { locs.push(MatcherLoc::Token { token: token.clone() }); } - TokenTree::Delimited(span, delimited) => { + TokenTree::Delimited(span, _, delimited) => { let open_token = Token::new(token::OpenDelim(delimited.delim), span.open); let close_token = Token::new(token::CloseDelim(delimited.delim), span.close); @@ -335,7 +335,7 @@ pub(super) fn count_metavar_decls(matcher: &[TokenTree]) -> usize { .map(|tt| match tt { TokenTree::MetaVarDecl(..) => 1, TokenTree::Sequence(_, seq) => seq.num_captures, - TokenTree::Delimited(_, delim) => count_metavar_decls(&delim.tts), + TokenTree::Delimited(.., delim) => count_metavar_decls(&delim.tts), TokenTree::Token(..) => 0, TokenTree::MetaVar(..) | TokenTree::MetaVarExpr(..) => unreachable!(), }) diff --git a/compiler/rustc_expand/src/mbe/macro_rules.rs b/compiler/rustc_expand/src/mbe/macro_rules.rs index 19734394382..393eec3997b 100644 --- a/compiler/rustc_expand/src/mbe/macro_rules.rs +++ b/compiler/rustc_expand/src/mbe/macro_rules.rs @@ -207,7 +207,7 @@ fn expand_macro<'cx>( match try_success_result { Ok((i, named_matches)) => { let (rhs, rhs_span): (&mbe::Delimited, DelimSpan) = match &rhses[i] { - mbe::TokenTree::Delimited(span, delimited) => (&delimited, *span), + mbe::TokenTree::Delimited(span, _, delimited) => (&delimited, *span), _ => cx.span_bug(sp, "malformed macro rhs"), }; let arm_span = rhses[i].span(); @@ -589,7 +589,7 @@ pub fn compile_declarative_macro( .map(|lhs| { // Ignore the delimiters around the matcher. match lhs { - mbe::TokenTree::Delimited(_, delimited) => { + mbe::TokenTree::Delimited(.., delimited) => { mbe::macro_parser::compute_locs(&delimited.tts) } _ => sess.diagnostic().span_bug(def.span, "malformed macro lhs"), @@ -615,7 +615,7 @@ pub fn compile_declarative_macro( fn check_lhs_nt_follows(sess: &ParseSess, def: &ast::Item, lhs: &mbe::TokenTree) -> bool { // lhs is going to be like TokenTree::Delimited(...), where the // entire lhs is those tts. Or, it can be a "bare sequence", not wrapped in parens. - if let mbe::TokenTree::Delimited(_, delimited) = lhs { + if let mbe::TokenTree::Delimited(.., delimited) = lhs { check_matcher(sess, def, &delimited.tts) } else { let msg = "invalid macro matcher; matchers must be contained in balanced delimiters"; @@ -668,7 +668,7 @@ fn check_lhs_no_empty_seq(sess: &ParseSess, tts: &[mbe::TokenTree]) -> bool { | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) | TokenTree::MetaVarExpr(..) => (), - TokenTree::Delimited(_, del) => { + TokenTree::Delimited(.., del) => { if !check_lhs_no_empty_seq(sess, &del.tts) { return false; } @@ -709,14 +709,14 @@ fn check_matcher(sess: &ParseSess, def: &ast::Item, matcher: &[mbe::TokenTree]) fn has_compile_error_macro(rhs: &mbe::TokenTree) -> bool { match rhs { - mbe::TokenTree::Delimited(_sp, d) => { + mbe::TokenTree::Delimited(.., d) => { let has_compile_error = d.tts.array_windows::<3>().any(|[ident, bang, args]| { if let mbe::TokenTree::Token(ident) = ident && let TokenKind::Ident(ident, _) = ident.kind && ident == sym::compile_error && let mbe::TokenTree::Token(bang) = bang && let TokenKind::Not = bang.kind - && let mbe::TokenTree::Delimited(_, del) = args + && let mbe::TokenTree::Delimited(.., del) = args && del.delim != Delimiter::Invisible { true @@ -773,7 +773,7 @@ impl<'tt> FirstSets<'tt> { | TokenTree::MetaVarExpr(..) => { first.replace_with(TtHandle::TtRef(tt)); } - TokenTree::Delimited(span, delimited) => { + TokenTree::Delimited(span, _, delimited) => { build_recur(sets, &delimited.tts); first.replace_with(TtHandle::from_token_kind( token::OpenDelim(delimited.delim), @@ -842,7 +842,7 @@ impl<'tt> FirstSets<'tt> { first.add_one(TtHandle::TtRef(tt)); return first; } - TokenTree::Delimited(span, delimited) => { + TokenTree::Delimited(span, _, delimited) => { first.add_one(TtHandle::from_token_kind( token::OpenDelim(delimited.delim), span.open, @@ -1087,7 +1087,7 @@ fn check_matcher_core<'tt>( suffix_first = build_suffix_first(); } } - TokenTree::Delimited(span, d) => { + TokenTree::Delimited(span, _, d) => { let my_suffix = TokenSet::singleton(TtHandle::from_token_kind( token::CloseDelim(d.delim), span.close, diff --git a/compiler/rustc_expand/src/mbe/metavar_expr.rs b/compiler/rustc_expand/src/mbe/metavar_expr.rs index 7cb279a9812..c29edc3dc9f 100644 --- a/compiler/rustc_expand/src/mbe/metavar_expr.rs +++ b/compiler/rustc_expand/src/mbe/metavar_expr.rs @@ -35,7 +35,7 @@ impl MetaVarExpr { ) -> PResult<'sess, MetaVarExpr> { let mut tts = input.trees(); let ident = parse_ident(&mut tts, sess, outer_span)?; - let Some(TokenTree::Delimited(_, Delimiter::Parenthesis, args)) = tts.next() else { + let Some(TokenTree::Delimited(.., Delimiter::Parenthesis, args)) = tts.next() else { let msg = "meta-variable expression parameter must be wrapped in parentheses"; return Err(sess.span_diagnostic.struct_span_err(ident.span, msg)); }; diff --git a/compiler/rustc_expand/src/mbe/quoted.rs b/compiler/rustc_expand/src/mbe/quoted.rs index 6a99412fc5b..ab9fb20b364 100644 --- a/compiler/rustc_expand/src/mbe/quoted.rs +++ b/compiler/rustc_expand/src/mbe/quoted.rs @@ -151,7 +151,7 @@ fn parse_tree<'a>( // during parsing. let mut next = outer_trees.next(); let mut trees: Box<dyn Iterator<Item = &tokenstream::TokenTree>>; - if let Some(tokenstream::TokenTree::Delimited(_, Delimiter::Invisible, tts)) = next { + if let Some(tokenstream::TokenTree::Delimited(.., Delimiter::Invisible, tts)) = next { trees = Box::new(tts.trees()); next = trees.next(); } else { @@ -160,7 +160,7 @@ fn parse_tree<'a>( match next { // `tree` is followed by a delimited set of token trees. - Some(&tokenstream::TokenTree::Delimited(delim_span, delim, ref tts)) => { + Some(&tokenstream::TokenTree::Delimited(delim_span, _, delim, ref tts)) => { if parsing_patterns { if delim != Delimiter::Parenthesis { span_dollar_dollar_or_metavar_in_the_lhs_err( @@ -258,8 +258,9 @@ fn parse_tree<'a>( // `tree` is the beginning of a delimited set of tokens (e.g., `(` or `{`). We need to // descend into the delimited set and further parse it. - &tokenstream::TokenTree::Delimited(span, delim, ref tts) => TokenTree::Delimited( + &tokenstream::TokenTree::Delimited(span, spacing, delim, ref tts) => TokenTree::Delimited( span, + spacing, Delimited { delim, tts: parse(tts, parsing_patterns, sess, node_id, features, edition), diff --git a/compiler/rustc_expand/src/mbe/transcribe.rs b/compiler/rustc_expand/src/mbe/transcribe.rs index 854f551ea1b..18707cebcd5 100644 --- a/compiler/rustc_expand/src/mbe/transcribe.rs +++ b/compiler/rustc_expand/src/mbe/transcribe.rs @@ -7,7 +7,7 @@ use crate::mbe::macro_parser::{MatchedNonterminal, MatchedSeq, MatchedTokenTree, use crate::mbe::{self, MetaVarExpr}; use rustc_ast::mut_visit::{self, MutVisitor}; use rustc_ast::token::{self, Delimiter, Token, TokenKind}; -use rustc_ast::tokenstream::{DelimSpan, Spacing, TokenStream, TokenTree}; +use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree}; use rustc_data_structures::fx::FxHashMap; use rustc_errors::{pluralize, PResult}; use rustc_errors::{DiagnosticBuilder, ErrorGuaranteed}; @@ -31,14 +31,24 @@ impl MutVisitor for Marker { /// An iterator over the token trees in a delimited token tree (`{ ... }`) or a sequence (`$(...)`). enum Frame<'a> { - Delimited { tts: &'a [mbe::TokenTree], idx: usize, delim: Delimiter, span: DelimSpan }, - Sequence { tts: &'a [mbe::TokenTree], idx: usize, sep: Option<Token> }, + Delimited { + tts: &'a [mbe::TokenTree], + idx: usize, + delim: Delimiter, + span: DelimSpan, + spacing: DelimSpacing, + }, + Sequence { + tts: &'a [mbe::TokenTree], + idx: usize, + sep: Option<Token>, + }, } impl<'a> Frame<'a> { /// Construct a new frame around the delimited set of tokens. - fn new(src: &'a mbe::Delimited, span: DelimSpan) -> Frame<'a> { - Frame::Delimited { tts: &src.tts, idx: 0, delim: src.delim, span } + fn new(src: &'a mbe::Delimited, span: DelimSpan, spacing: DelimSpacing) -> Frame<'a> { + Frame::Delimited { tts: &src.tts, idx: 0, delim: src.delim, span, spacing } } } @@ -89,8 +99,10 @@ pub(super) fn transcribe<'a>( } // We descend into the RHS (`src`), expanding things as we go. This stack contains the things - // we have yet to expand/are still expanding. We start the stack off with the whole RHS. - let mut stack: SmallVec<[Frame<'_>; 1]> = smallvec![Frame::new(src, src_span)]; + // we have yet to expand/are still expanding. We start the stack off with the whole RHS. The + // choice of spacing values doesn't matter. + let mut stack: SmallVec<[Frame<'_>; 1]> = + smallvec![Frame::new(src, src_span, DelimSpacing::new(Spacing::Alone, Spacing::Alone))]; // As we descend in the RHS, we will need to be able to match nested sequences of matchers. // `repeats` keeps track of where we are in matching at each level, with the last element being @@ -144,14 +156,19 @@ pub(super) fn transcribe<'a>( // We are done processing a Delimited. If this is the top-level delimited, we are // done. Otherwise, we unwind the result_stack to append what we have produced to // any previous results. - Frame::Delimited { delim, span, .. } => { + Frame::Delimited { delim, span, mut spacing, .. } => { + // Hack to force-insert a space after `]` in certain case. + // See discussion of the `hex-literal` crate in #114571. + if delim == Delimiter::Bracket { + spacing.close = Spacing::Alone; + } if result_stack.is_empty() { // No results left to compute! We are back at the top-level. return Ok(TokenStream::new(result)); } // Step back into the parent Delimited. - let tree = TokenTree::Delimited(span, delim, TokenStream::new(result)); + let tree = TokenTree::Delimited(span, spacing, delim, TokenStream::new(result)); result = result_stack.pop().unwrap(); result.push(tree); } @@ -258,13 +275,14 @@ pub(super) fn transcribe<'a>( // We will produce all of the results of the inside of the `Delimited` and then we will // jump back out of the Delimited, pop the result_stack and add the new results back to // the previous results (from outside the Delimited). - mbe::TokenTree::Delimited(mut span, delimited) => { + mbe::TokenTree::Delimited(mut span, spacing, delimited) => { mut_visit::visit_delim_span(&mut span, &mut marker); stack.push(Frame::Delimited { tts: &delimited.tts, delim: delimited.delim, idx: 0, span, + spacing: *spacing, }); result_stack.push(mem::take(&mut result)); } @@ -374,7 +392,7 @@ fn lockstep_iter_size( ) -> LockstepIterSize { use mbe::TokenTree; match tree { - TokenTree::Delimited(_, delimited) => { + TokenTree::Delimited(.., delimited) => { delimited.tts.iter().fold(LockstepIterSize::Unconstrained, |size, tt| { size.with(lockstep_iter_size(tt, interpolations, repeats)) }) diff --git a/compiler/rustc_expand/src/parse/tests.rs b/compiler/rustc_expand/src/parse/tests.rs index a6c6d82b575..7a888250ca1 100644 --- a/compiler/rustc_expand/src/parse/tests.rs +++ b/compiler/rustc_expand/src/parse/tests.rs @@ -4,7 +4,7 @@ use crate::tests::{ use rustc_ast::ptr::P; use rustc_ast::token::{self, Delimiter, Token}; -use rustc_ast::tokenstream::{DelimSpan, TokenStream, TokenTree}; +use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree}; use rustc_ast::visit; use rustc_ast::{self as ast, PatKind}; use rustc_ast_pretty::pprust::item_to_string; @@ -77,14 +77,14 @@ fn string_to_tts_macro() { TokenTree::Token(Token { kind: token::Ident(name_macro_rules, false), .. }, _), TokenTree::Token(Token { kind: token::Not, .. }, _), TokenTree::Token(Token { kind: token::Ident(name_zip, false), .. }, _), - TokenTree::Delimited(_, macro_delim, macro_tts), + TokenTree::Delimited(.., macro_delim, macro_tts), ] if name_macro_rules == &kw::MacroRules && name_zip.as_str() == "zip" => { let tts = ¯o_tts.trees().collect::<Vec<_>>(); match &tts[..] { [ - TokenTree::Delimited(_, first_delim, first_tts), + TokenTree::Delimited(.., first_delim, first_tts), TokenTree::Token(Token { kind: token::FatArrow, .. }, _), - TokenTree::Delimited(_, second_delim, second_tts), + TokenTree::Delimited(.., second_delim, second_tts), ] if macro_delim == &Delimiter::Parenthesis => { let tts = &first_tts.trees().collect::<Vec<_>>(); match &tts[..] { @@ -123,6 +123,9 @@ fn string_to_tts_1() { TokenTree::token_joint_hidden(token::Ident(Symbol::intern("a"), false), sp(3, 4)), TokenTree::Delimited( DelimSpan::from_pair(sp(4, 5), sp(11, 12)), + // `JointHidden` because the `(` is followed immediately by + // `b`, `Alone` because the `)` is followed by whitespace. + DelimSpacing::new(Spacing::JointHidden, Spacing::Alone), Delimiter::Parenthesis, TokenStream::new(vec![ TokenTree::token_joint(token::Ident(Symbol::intern("b"), false), sp(5, 6)), @@ -134,6 +137,10 @@ fn string_to_tts_1() { ), TokenTree::Delimited( DelimSpan::from_pair(sp(13, 14), sp(18, 19)), + // First `Alone` because the `{` is followed by whitespace, + // second `Alone` because the `}` is followed immediately by + // EOF. + DelimSpacing::new(Spacing::Alone, Spacing::Alone), Delimiter::Brace, TokenStream::new(vec![ TokenTree::token_joint(token::Ident(Symbol::intern("b"), false), sp(15, 16)), diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index b3b78330801..5308e338d7f 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -5,7 +5,7 @@ use pm::bridge::{ use pm::{Delimiter, Level}; use rustc_ast as ast; use rustc_ast::token; -use rustc_ast::tokenstream::{self, Spacing, TokenStream}; +use rustc_ast::tokenstream::{self, DelimSpacing, Spacing, TokenStream}; use rustc_ast::util::literal::escape_byte_str_symbol; use rustc_ast_pretty::pprust; use rustc_data_structures::fx::FxHashMap; @@ -98,7 +98,7 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre while let Some(tree) = cursor.next() { let (Token { kind, span }, joint) = match tree.clone() { - tokenstream::TokenTree::Delimited(span, delim, tts) => { + tokenstream::TokenTree::Delimited(span, _, delim, tts) => { let delimiter = pm::Delimiter::from_internal(delim); trees.push(TokenTree::Group(Group { delimiter, @@ -284,10 +284,11 @@ impl ToInternal<SmallVec<[tokenstream::TokenTree; 2]>> fn to_internal(self) -> SmallVec<[tokenstream::TokenTree; 2]> { use rustc_ast::token::*; - // The code below is conservative and uses `token_alone` in most - // places. When the resulting code is pretty-printed by `print_tts` it - // ends up with spaces between most tokens, which is safe but ugly. - // It's hard in general to do better when working at the token level. + // The code below is conservative, using `token_alone`/`Spacing::Alone` + // in most places. When the resulting code is pretty-printed by + // `print_tts` it ends up with spaces between most tokens, which is + // safe but ugly. It's hard in general to do better when working at the + // token level. let (tree, rustc) = self; match tree { TokenTree::Punct(Punct { ch, joint, span }) => { @@ -330,6 +331,7 @@ impl ToInternal<SmallVec<[tokenstream::TokenTree; 2]>> TokenTree::Group(Group { delimiter, stream, span: DelimSpan { open, close, .. } }) => { smallvec![tokenstream::TokenTree::Delimited( tokenstream::DelimSpan { open, close }, + DelimSpacing::new(Spacing::Alone, Spacing::Alone), delimiter.to_internal(), stream.unwrap_or_default(), )] |
