diff options
23 files changed, 317 insertions, 307 deletions
diff --git a/compiler/rustc_ast/src/attr/mod.rs b/compiler/rustc_ast/src/attr/mod.rs index 988918b0505..86af7769d1b 100644 --- a/compiler/rustc_ast/src/attr/mod.rs +++ b/compiler/rustc_ast/src/attr/mod.rs @@ -8,7 +8,7 @@ use crate::ast::{Path, PathSegment}; use crate::ptr::P; use crate::token::{self, CommentKind, Delimiter, Token}; use crate::tokenstream::{AttrAnnotatedTokenStream, AttrAnnotatedTokenTree}; -use crate::tokenstream::{DelimSpan, Spacing, TokenTree, TreeAndSpacing}; +use crate::tokenstream::{DelimSpan, Spacing, TokenTree}; use crate::tokenstream::{LazyTokenStream, TokenStream}; use crate::util::comments; @@ -388,7 +388,7 @@ pub fn list_contains_name(items: &[NestedMetaItem], name: Symbol) -> bool { } impl MetaItem { - fn token_trees_and_spacings(&self) -> Vec<TreeAndSpacing> { + fn token_trees(&self) -> Vec<TokenTree> { let mut idents = vec![]; let mut last_pos = BytePos(0_u32); for (i, segment) in self.path.segments.iter().enumerate() { @@ -396,12 +396,12 @@ impl MetaItem { if !is_first { let mod_sep_span = Span::new(last_pos, segment.ident.span.lo(), segment.ident.span.ctxt(), None); - idents.push(TokenTree::token(token::ModSep, mod_sep_span).into()); + idents.push(TokenTree::token_alone(token::ModSep, mod_sep_span)); } - idents.push(TokenTree::Token(Token::from_ast_ident(segment.ident)).into()); + idents.push(TokenTree::Token(Token::from_ast_ident(segment.ident), Spacing::Alone)); last_pos = segment.ident.span.hi(); } - idents.extend(self.kind.token_trees_and_spacings(self.span)); + idents.extend(self.kind.token_trees(self.span)); idents } @@ -411,12 +411,13 @@ impl MetaItem { { // FIXME: Share code with `parse_path`. let path = match tokens.next().map(TokenTree::uninterpolate) { - Some(TokenTree::Token(Token { - kind: kind @ (token::Ident(..) | token::ModSep), - span, - })) => 'arm: { + Some(TokenTree::Token( + Token { kind: kind @ (token::Ident(..) | token::ModSep), span }, + _, + )) => 'arm: { let mut segments = if let token::Ident(name, _) = kind { - if let Some(TokenTree::Token(Token { kind: token::ModSep, .. })) = tokens.peek() + if let Some(TokenTree::Token(Token { kind: token::ModSep, .. }, _)) = + tokens.peek() { tokens.next(); vec![PathSegment::from_ident(Ident::new(name, span))] @@ -427,14 +428,15 @@ impl MetaItem { vec![PathSegment::path_root(span)] }; loop { - if let Some(TokenTree::Token(Token { kind: token::Ident(name, _), span })) = + if let Some(TokenTree::Token(Token { kind: token::Ident(name, _), span }, _)) = tokens.next().map(TokenTree::uninterpolate) { segments.push(PathSegment::from_ident(Ident::new(name, span))); } else { return None; } - if let Some(TokenTree::Token(Token { kind: token::ModSep, .. })) = tokens.peek() + if let Some(TokenTree::Token(Token { kind: token::ModSep, .. }, _)) = + tokens.peek() { tokens.next(); } else { @@ -444,7 +446,7 @@ impl MetaItem { let span = span.with_hi(segments.last().unwrap().ident.span.hi()); Path { span, segments, tokens: None } } - Some(TokenTree::Token(Token { kind: token::Interpolated(nt), .. })) => match *nt { + Some(TokenTree::Token(Token { kind: token::Interpolated(nt), .. }, _)) => match *nt { token::Nonterminal::NtMeta(ref item) => return item.meta(item.path.span), token::Nonterminal::NtPath(ref path) => (**path).clone(), _ => return None, @@ -491,9 +493,9 @@ impl MetaItemKind { let mut tts = Vec::new(); for (i, item) in list.iter().enumerate() { if i > 0 { - tts.push(TokenTree::token(token::Comma, span).into()); + tts.push(TokenTree::token_alone(token::Comma, span)); } - tts.extend(item.token_trees_and_spacings()) + tts.extend(item.token_trees()) } MacArgs::Delimited( DelimSpan::from_single(span), @@ -504,31 +506,28 @@ impl MetaItemKind { } } - fn token_trees_and_spacings(&self, span: Span) -> Vec<TreeAndSpacing> { + fn token_trees(&self, span: Span) -> Vec<TokenTree> { match *self { MetaItemKind::Word => vec![], MetaItemKind::NameValue(ref lit) => { vec![ - TokenTree::token(token::Eq, span).into(), - TokenTree::Token(lit.to_token()).into(), + TokenTree::token_alone(token::Eq, span), + TokenTree::Token(lit.to_token(), Spacing::Alone), ] } MetaItemKind::List(ref list) => { let mut tokens = Vec::new(); for (i, item) in list.iter().enumerate() { if i > 0 { - tokens.push(TokenTree::token(token::Comma, span).into()); + tokens.push(TokenTree::token_alone(token::Comma, span)); } - tokens.extend(item.token_trees_and_spacings()) + tokens.extend(item.token_trees()) } - vec![ - TokenTree::Delimited( - DelimSpan::from_single(span), - Delimiter::Parenthesis, - TokenStream::new(tokens), - ) - .into(), - ] + vec![TokenTree::Delimited( + DelimSpan::from_single(span), + Delimiter::Parenthesis, + TokenStream::new(tokens), + )] } } } @@ -540,7 +539,7 @@ impl MetaItemKind { let item = NestedMetaItem::from_tokens(&mut tokens)?; result.push(item); match tokens.next() { - None | Some(TokenTree::Token(Token { kind: token::Comma, .. })) => {} + None | Some(TokenTree::Token(Token { kind: token::Comma, .. }, _)) => {} _ => return None, } } @@ -554,7 +553,7 @@ impl MetaItemKind { Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens)) => { MetaItemKind::name_value_from_tokens(&mut inner_tokens.into_trees()) } - Some(TokenTree::Token(token)) => { + Some(TokenTree::Token(token, _)) => { Lit::from_token(&token).ok().map(MetaItemKind::NameValue) } _ => None, @@ -586,7 +585,7 @@ impl MetaItemKind { MetaItemKind::list_from_tokens(inner_tokens) } Some(TokenTree::Delimited(..)) => None, - Some(TokenTree::Token(Token { kind: token::Eq, .. })) => { + Some(TokenTree::Token(Token { kind: token::Eq, .. }, _)) => { tokens.next(); MetaItemKind::name_value_from_tokens(tokens) } @@ -603,10 +602,12 @@ impl NestedMetaItem { } } - fn token_trees_and_spacings(&self) -> Vec<TreeAndSpacing> { + fn token_trees(&self) -> Vec<TokenTree> { match *self { - NestedMetaItem::MetaItem(ref item) => item.token_trees_and_spacings(), - NestedMetaItem::Literal(ref lit) => vec![TokenTree::Token(lit.to_token()).into()], + NestedMetaItem::MetaItem(ref item) => item.token_trees(), + NestedMetaItem::Literal(ref lit) => { + vec![TokenTree::Token(lit.to_token(), Spacing::Alone)] + } } } @@ -615,7 +616,7 @@ impl NestedMetaItem { I: Iterator<Item = TokenTree>, { match tokens.peek() { - Some(TokenTree::Token(token)) + Some(TokenTree::Token(token, _)) if let Ok(lit) = Lit::from_token(token) => { tokens.next(); diff --git a/compiler/rustc_ast/src/mut_visit.rs b/compiler/rustc_ast/src/mut_visit.rs index d933ea2da9e..01bd498b377 100644 --- a/compiler/rustc_ast/src/mut_visit.rs +++ b/compiler/rustc_ast/src/mut_visit.rs @@ -675,7 +675,7 @@ pub fn visit_attr_annotated_tt<T: MutVisitor>(tt: &mut AttrAnnotatedTokenTree, v // No `noop_` prefix because there isn't a corresponding method in `MutVisitor`. pub fn visit_tt<T: MutVisitor>(tt: &mut TokenTree, vis: &mut T) { match tt { - TokenTree::Token(token) => { + TokenTree::Token(token, _) => { visit_token(token, vis); } TokenTree::Delimited(DelimSpan { open, close }, _delim, tts) => { @@ -690,7 +690,7 @@ pub fn visit_tt<T: MutVisitor>(tt: &mut TokenTree, vis: &mut T) { pub fn visit_tts<T: MutVisitor>(TokenStream(tts): &mut TokenStream, vis: &mut T) { if T::VISIT_TOKENS && !tts.is_empty() { let tts = Lrc::make_mut(tts); - visit_vec(tts, |(tree, _is_joint)| visit_tt(tree, vis)); + visit_vec(tts, |tree| visit_tt(tree, vis)); } } diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index 37de90d64c7..9e4a22e1fa3 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -42,11 +42,15 @@ use std::{fmt, iter}; #[derive(Debug, Clone, PartialEq, Encodable, Decodable, HashStable_Generic)] pub enum TokenTree { /// A single token. - Token(Token), + Token(Token, Spacing), /// A delimited sequence of token trees. Delimited(DelimSpan, Delimiter, TokenStream), } +// This type is used a lot. Make sure it doesn't unintentionally get bigger. +#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))] +rustc_data_structures::static_assert_size!(TokenTree, 32); + // Ensure all fields of `TokenTree` is `Send` and `Sync`. #[cfg(parallel_compiler)] fn _dummy() @@ -62,7 +66,7 @@ impl TokenTree { /// Checks if this `TokenTree` is equal to the other, regardless of span information. pub fn eq_unspanned(&self, other: &TokenTree) -> bool { match (self, other) { - (TokenTree::Token(token), TokenTree::Token(token2)) => token.kind == token2.kind, + (TokenTree::Token(token, _), TokenTree::Token(token2, _)) => token.kind == token2.kind, (TokenTree::Delimited(_, delim, tts), TokenTree::Delimited(_, delim2, tts2)) => { delim == delim2 && tts.eq_unspanned(&tts2) } @@ -73,7 +77,7 @@ impl TokenTree { /// Retrieves the `TokenTree`'s span. pub fn span(&self) -> Span { match self { - TokenTree::Token(token) => token.span, + TokenTree::Token(token, _) => token.span, TokenTree::Delimited(sp, ..) => sp.entire(), } } @@ -81,18 +85,26 @@ impl TokenTree { /// Modify the `TokenTree`'s span in-place. pub fn set_span(&mut self, span: Span) { match self { - TokenTree::Token(token) => token.span = span, + TokenTree::Token(token, _) => token.span = span, TokenTree::Delimited(dspan, ..) => *dspan = DelimSpan::from_single(span), } } - pub fn token(kind: TokenKind, span: Span) -> TokenTree { - TokenTree::Token(Token::new(kind, span)) + // Create a `TokenTree::Token` with alone spacing. + pub fn token_alone(kind: TokenKind, span: Span) -> TokenTree { + TokenTree::Token(Token::new(kind, span), Spacing::Alone) + } + + // Create a `TokenTree::Token` with joint spacing. + pub fn token_joint(kind: TokenKind, span: Span) -> TokenTree { + TokenTree::Token(Token::new(kind, span), Spacing::Joint) } pub fn uninterpolate(self) -> TokenTree { match self { - TokenTree::Token(token) => TokenTree::Token(token.uninterpolate().into_owned()), + TokenTree::Token(token, spacing) => { + TokenTree::Token(token.uninterpolate().into_owned(), spacing) + } tt => tt, } } @@ -194,13 +206,12 @@ impl AttrAnnotatedTokenStream { .iter() .flat_map(|tree| match &tree.0 { AttrAnnotatedTokenTree::Token(inner) => { - smallvec![(TokenTree::Token(inner.clone()), tree.1)].into_iter() + smallvec![TokenTree::Token(inner.clone(), tree.1)].into_iter() + } + AttrAnnotatedTokenTree::Delimited(span, delim, stream) => { + smallvec![TokenTree::Delimited(*span, *delim, stream.to_tokenstream()),] + .into_iter() } - AttrAnnotatedTokenTree::Delimited(span, delim, stream) => smallvec![( - TokenTree::Delimited(*span, *delim, stream.to_tokenstream()), - tree.1, - )] - .into_iter(), AttrAnnotatedTokenTree::Attributes(data) => { let mut outer_attrs = Vec::new(); let mut inner_attrs = Vec::new(); @@ -226,7 +237,7 @@ impl AttrAnnotatedTokenStream { if !inner_attrs.is_empty() { let mut found = false; // Check the last two trees (to account for a trailing semi) - for (tree, _) in target_tokens.iter_mut().rev().take(2) { + for tree in target_tokens.iter_mut().rev().take(2) { if let TokenTree::Delimited(span, delim, delim_tokens) = tree { // Inner attributes are only supported on extern blocks, functions, impls, // and modules. All of these have their inner attributes placed at @@ -299,15 +310,13 @@ pub struct AttributesData { /// Today's `TokenTree`s can still contain AST via `token::Interpolated` for /// backwards compatibility. #[derive(Clone, Debug, Default, Encodable, Decodable)] -pub struct TokenStream(pub(crate) Lrc<Vec<TreeAndSpacing>>); - -pub type TreeAndSpacing = (TokenTree, Spacing); +pub struct TokenStream(pub(crate) Lrc<Vec<TokenTree>>); // `TokenStream` is used a lot. Make sure it doesn't unintentionally get bigger. #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))] rustc_data_structures::static_assert_size!(TokenStream, 8); -#[derive(Clone, Copy, Debug, PartialEq, Encodable, Decodable)] +#[derive(Clone, Copy, Debug, PartialEq, Encodable, Decodable, HashStable_Generic)] pub enum Spacing { Alone, Joint, @@ -323,10 +332,10 @@ impl TokenStream { while let Some((pos, ts)) = iter.next() { if let Some((_, next)) = iter.peek() { let sp = match (&ts, &next) { - (_, (TokenTree::Token(Token { kind: token::Comma, .. }), _)) => continue, + (_, TokenTree::Token(Token { kind: token::Comma, .. }, _)) => continue, ( - (TokenTree::Token(token_left), Spacing::Alone), - (TokenTree::Token(token_right), _), + TokenTree::Token(token_left, Spacing::Alone), + TokenTree::Token(token_right, _), ) if ((token_left.is_ident() && !token_left.is_reserved_ident()) || token_left.is_lit()) && ((token_right.is_ident() && !token_right.is_reserved_ident()) @@ -334,11 +343,11 @@ impl TokenStream { { token_left.span } - ((TokenTree::Delimited(sp, ..), Spacing::Alone), _) => sp.entire(), + (TokenTree::Delimited(sp, ..), _) => sp.entire(), _ => continue, }; let sp = sp.shrink_to_hi(); - let comma = (TokenTree::token(token::Comma, sp), Spacing::Alone); + let comma = TokenTree::token_alone(token::Comma, sp); suggestion = Some((pos, comma, sp)); } } @@ -360,21 +369,9 @@ impl From<(AttrAnnotatedTokenTree, Spacing)> for AttrAnnotatedTokenStream { } } -impl From<TokenTree> for TokenStream { - fn from(tree: TokenTree) -> TokenStream { - TokenStream::new(vec![(tree, Spacing::Alone)]) - } -} - -impl From<TokenTree> for TreeAndSpacing { - fn from(tree: TokenTree) -> TreeAndSpacing { - (tree, Spacing::Alone) - } -} - impl iter::FromIterator<TokenTree> for TokenStream { fn from_iter<I: IntoIterator<Item = TokenTree>>(iter: I) -> Self { - TokenStream::new(iter.into_iter().map(Into::into).collect::<Vec<TreeAndSpacing>>()) + TokenStream::new(iter.into_iter().collect::<Vec<TokenTree>>()) } } @@ -387,7 +384,7 @@ impl PartialEq<TokenStream> for TokenStream { } impl TokenStream { - pub fn new(streams: Vec<TreeAndSpacing>) -> TokenStream { + pub fn new(streams: Vec<TokenTree>) -> TokenStream { TokenStream(Lrc::new(streams)) } @@ -420,13 +417,7 @@ impl TokenStream { } pub fn map_enumerated<F: FnMut(usize, &TokenTree) -> TokenTree>(self, mut f: F) -> TokenStream { - TokenStream(Lrc::new( - self.0 - .iter() - .enumerate() - .map(|(i, (tree, is_joint))| (f(i, tree), *is_joint)) - .collect(), - )) + TokenStream(Lrc::new(self.0.iter().enumerate().map(|(i, tree)| f(i, tree)).collect())) } fn opt_from_ast(node: &(impl HasAttrs + HasTokens)) -> Option<TokenStream> { @@ -444,6 +435,21 @@ impl TokenStream { Some(attr_annotated.to_tokenstream()) } + // Create a token stream containing a single token with alone spacing. + pub fn token_alone(kind: TokenKind, span: Span) -> TokenStream { + TokenStream::new(vec![TokenTree::token_alone(kind, span)]) + } + + // Create a token stream containing a single token with joint spacing. + pub fn token_joint(kind: TokenKind, span: Span) -> TokenStream { + TokenStream::new(vec![TokenTree::token_joint(kind, span)]) + } + + // Create a token stream containing a single `Delimited`. + pub fn delimited(span: DelimSpan, delim: Delimiter, tts: TokenStream) -> TokenStream { + TokenStream::new(vec![TokenTree::Delimited(span, delim, tts)]) + } + pub fn from_ast(node: &(impl HasAttrs + HasSpan + HasTokens + fmt::Debug)) -> TokenStream { TokenStream::opt_from_ast(node) .unwrap_or_else(|| panic!("missing tokens for node at {:?}: {:?}", node.span(), node)) @@ -452,16 +458,16 @@ impl TokenStream { pub fn from_nonterminal_ast(nt: &Nonterminal) -> TokenStream { match nt { Nonterminal::NtIdent(ident, is_raw) => { - TokenTree::token(token::Ident(ident.name, *is_raw), ident.span).into() + TokenStream::token_alone(token::Ident(ident.name, *is_raw), ident.span) } Nonterminal::NtLifetime(ident) => { - TokenTree::token(token::Lifetime(ident.name), ident.span).into() + TokenStream::token_alone(token::Lifetime(ident.name), ident.span) } Nonterminal::NtItem(item) => TokenStream::from_ast(item), Nonterminal::NtBlock(block) => TokenStream::from_ast(block), Nonterminal::NtStmt(stmt) if let StmtKind::Empty = stmt.kind => { // FIXME: Properly collect tokens for empty statements. - TokenTree::token(token::Semi, stmt.span).into() + TokenStream::token_alone(token::Semi, stmt.span) } Nonterminal::NtStmt(stmt) => TokenStream::from_ast(stmt), Nonterminal::NtPat(pat) => TokenStream::from_ast(pat), @@ -473,23 +479,23 @@ impl TokenStream { } } - fn flatten_token(token: &Token) -> TokenTree { + fn flatten_token(token: &Token, spacing: Spacing) -> TokenTree { match &token.kind { token::Interpolated(nt) if let token::NtIdent(ident, is_raw) = **nt => { - TokenTree::token(token::Ident(ident.name, is_raw), ident.span) + TokenTree::Token(Token::new(token::Ident(ident.name, is_raw), ident.span), spacing) } token::Interpolated(nt) => TokenTree::Delimited( DelimSpan::from_single(token.span), Delimiter::Invisible, TokenStream::from_nonterminal_ast(&nt).flattened(), ), - _ => TokenTree::Token(token.clone()), + _ => TokenTree::Token(token.clone(), spacing), } } fn flatten_token_tree(tree: &TokenTree) -> TokenTree { match tree { - TokenTree::Token(token) => TokenStream::flatten_token(token), + TokenTree::Token(token, spacing) => TokenStream::flatten_token(token, *spacing), TokenTree::Delimited(span, delim, tts) => { TokenTree::Delimited(*span, *delim, tts.flattened()) } @@ -500,7 +506,7 @@ impl TokenStream { pub fn flattened(&self) -> TokenStream { fn can_skip(stream: &TokenStream) -> bool { stream.trees().all(|tree| match tree { - TokenTree::Token(token) => !matches!(token.kind, token::Interpolated(_)), + TokenTree::Token(token, _) => !matches!(token.kind, token::Interpolated(_)), TokenTree::Delimited(_, _, inner) => can_skip(inner), }) } @@ -522,8 +528,8 @@ impl TokenStreamBuilder { TokenStreamBuilder(SmallVec::new()) } - pub fn push<T: Into<TokenStream>>(&mut self, stream: T) { - self.0.push(stream.into()); + pub fn push(&mut self, stream: TokenStream) { + self.0.push(stream); } pub fn build(self) -> TokenStream { @@ -564,14 +570,14 @@ impl TokenStreamBuilder { // `stream` is not empty and the first tree within it is a // token tree, and (c) the two tokens can be glued // together... - if let Some((TokenTree::Token(last_tok), Spacing::Joint)) = res_vec_mut.last() - && let Some((TokenTree::Token(tok), spacing)) = stream.0.first() + if let Some(TokenTree::Token(last_tok, Spacing::Joint)) = res_vec_mut.last() + && let Some(TokenTree::Token(tok, spacing)) = stream.0.first() && let Some(glued_tok) = last_tok.glue(&tok) { // ...then overwrite the last token tree in // `res_vec_mut` with the glued token, and skip the // first token tree from `stream`. - *res_vec_mut.last_mut().unwrap() = (TokenTree::Token(glued_tok), *spacing); + *res_vec_mut.last_mut().unwrap() = TokenTree::Token(glued_tok, *spacing); res_vec_mut.extend(stream_iter.skip(1)); } else { // Append all of `stream`. @@ -597,16 +603,8 @@ impl<'t> CursorRef<'t> { CursorRef { stream, index: 0 } } - #[inline] - fn next_with_spacing(&mut self) -> Option<&'t TreeAndSpacing> { - self.stream.0.get(self.index).map(|tree| { - self.index += 1; - tree - }) - } - pub fn look_ahead(&self, n: usize) -> Option<&TokenTree> { - self.stream.0[self.index..].get(n).map(|(tree, _)| tree) + self.stream.0.get(self.index + n) } } @@ -614,7 +612,10 @@ impl<'t> Iterator for CursorRef<'t> { type Item = &'t TokenTree; fn next(&mut self) -> Option<&'t TokenTree> { - self.next_with_spacing().map(|(tree, _)| tree) + self.stream.0.get(self.index).map(|tree| { + self.index += 1; + tree + }) } } @@ -630,7 +631,10 @@ impl Iterator for Cursor { type Item = TokenTree; fn next(&mut self) -> Option<TokenTree> { - self.next_with_spacing().map(|(tree, _)| tree) + self.stream.0.get(self.index).map(|tree| { + self.index += 1; + tree.clone() + }) } } @@ -640,15 +644,7 @@ impl Cursor { } #[inline] - pub fn next_with_spacing(&mut self) -> Option<TreeAndSpacing> { - self.stream.0.get(self.index).map(|tree| { - self.index += 1; - tree.clone() - }) - } - - #[inline] - pub fn next_with_spacing_ref(&mut self) -> Option<&TreeAndSpacing> { + pub fn next_ref(&mut self) -> Option<&TokenTree> { self.stream.0.get(self.index).map(|tree| { self.index += 1; tree @@ -656,7 +652,7 @@ impl Cursor { } pub fn look_ahead(&self, n: usize) -> Option<&TokenTree> { - self.stream.0[self.index..].get(n).map(|(tree, _)| tree) + self.stream.0.get(self.index + n) } } diff --git a/compiler/rustc_ast_pretty/src/pprust/state.rs b/compiler/rustc_ast_pretty/src/pprust/state.rs index c9e3a7edfa6..55ddd24c48a 100644 --- a/compiler/rustc_ast_pretty/src/pprust/state.rs +++ b/compiler/rustc_ast_pretty/src/pprust/state.rs @@ -145,7 +145,7 @@ pub fn print_crate<'a>( /// This makes printed token streams look slightly nicer, /// and also addresses some specific regressions described in #63896 and #73345. fn tt_prepend_space(tt: &TokenTree, prev: &TokenTree) -> bool { - if let TokenTree::Token(token) = prev { + if let TokenTree::Token(token, _) = prev { if matches!(token.kind, token::Dot | token::Dollar) { return false; } @@ -154,12 +154,12 @@ fn tt_prepend_space(tt: &TokenTree, prev: &TokenTree) -> bool { } } match tt { - TokenTree::Token(token) => !matches!(token.kind, token::Comma | token::Not | token::Dot), + TokenTree::Token(token, _) => !matches!(token.kind, token::Comma | token::Not | token::Dot), TokenTree::Delimited(_, Delimiter::Parenthesis, _) => { - !matches!(prev, TokenTree::Token(Token { kind: token::Ident(..), .. })) + !matches!(prev, TokenTree::Token(Token { kind: token::Ident(..), .. }, _)) } TokenTree::Delimited(_, Delimiter::Bracket, _) => { - !matches!(prev, TokenTree::Token(Token { kind: token::Pound, .. })) + !matches!(prev, TokenTree::Token(Token { kind: token::Pound, .. }, _)) } TokenTree::Delimited(..) => true, } @@ -526,7 +526,7 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere /// expression arguments as expressions). It can be done! I think. fn print_tt(&mut self, tt: &TokenTree, convert_dollar_crate: bool) { match tt { - TokenTree::Token(token) => { + TokenTree::Token(token, _) => { let token_str = self.token_to_string_ext(&token, convert_dollar_crate); self.word(token_str); if let token::DocComment(..) = token.kind { diff --git a/compiler/rustc_builtin_macros/src/assert/context.rs b/compiler/rustc_builtin_macros/src/assert/context.rs index 01152ff7df5..dcea883a5a3 100644 --- a/compiler/rustc_builtin_macros/src/assert/context.rs +++ b/compiler/rustc_builtin_macros/src/assert/context.rs @@ -152,7 +152,7 @@ impl<'cx, 'a> Context<'cx, 'a> { fn build_panic(&self, expr_str: &str, panic_path: Path) -> P<Expr> { let escaped_expr_str = escape_to_fmt(expr_str); let initial = [ - TokenTree::token( + TokenTree::token_alone( token::Literal(token::Lit { kind: token::LitKind::Str, symbol: Symbol::intern(&if self.fmt_string.is_empty() { @@ -167,12 +167,12 @@ impl<'cx, 'a> Context<'cx, 'a> { }), self.span, ), - TokenTree::token(token::Comma, self.span), + TokenTree::token_alone(token::Comma, self.span), ]; let captures = self.capture_decls.iter().flat_map(|cap| { [ - TokenTree::token(token::Ident(cap.ident.name, false), cap.ident.span), - TokenTree::token(token::Comma, self.span), + TokenTree::token_alone(token::Ident(cap.ident.name, false), cap.ident.span), + TokenTree::token_alone(token::Comma, self.span), ] }); self.cx.expr( diff --git a/compiler/rustc_builtin_macros/src/concat_idents.rs b/compiler/rustc_builtin_macros/src/concat_idents.rs index 239bafb266a..297c604e020 100644 --- a/compiler/rustc_builtin_macros/src/concat_idents.rs +++ b/compiler/rustc_builtin_macros/src/concat_idents.rs @@ -20,14 +20,14 @@ pub fn expand_concat_idents<'cx>( for (i, e) in tts.into_trees().enumerate() { if i & 1 == 1 { match e { - TokenTree::Token(Token { kind: token::Comma, .. }) => {} + TokenTree::Token(Token { kind: token::Comma, .. }, _) => {} _ => { cx.span_err(sp, "concat_idents! expecting comma"); return DummyResult::any(sp); } } } else { - if let TokenTree::Token(token) = e { + if let TokenTree::Token(token, _) = e { if let Some((ident, _)) = token.ident() { res_str.push_str(ident.name.as_str()); continue; diff --git a/compiler/rustc_builtin_macros/src/trace_macros.rs b/compiler/rustc_builtin_macros/src/trace_macros.rs index c17f2afe494..cc5ae6894e6 100644 --- a/compiler/rustc_builtin_macros/src/trace_macros.rs +++ b/compiler/rustc_builtin_macros/src/trace_macros.rs @@ -11,8 +11,8 @@ pub fn expand_trace_macros( let mut cursor = tt.into_trees(); let mut err = false; let value = match &cursor.next() { - Some(TokenTree::Token(token)) if token.is_keyword(kw::True) => true, - Some(TokenTree::Token(token)) if token.is_keyword(kw::False) => false, + Some(TokenTree::Token(token, _)) if token.is_keyword(kw::True) => true, + Some(TokenTree::Token(token, _)) if token.is_keyword(kw::False) => false, _ => { err = true; false diff --git a/compiler/rustc_expand/src/config.rs b/compiler/rustc_expand/src/config.rs index ccc29adc015..3e1acf4382d 100644 --- a/compiler/rustc_expand/src/config.rs +++ b/compiler/rustc_expand/src/config.rs @@ -401,7 +401,7 @@ impl<'a> StripUnconfigured<'a> { // Use the `#` in `#[cfg_attr(pred, attr)]` as the `#` token // for `attr` when we expand it to `#[attr]` let mut orig_trees = orig_tokens.into_trees(); - let TokenTree::Token(pound_token @ Token { kind: TokenKind::Pound, .. }) = orig_trees.next().unwrap() else { + let TokenTree::Token(pound_token @ Token { kind: TokenKind::Pound, .. }, _) = orig_trees.next().unwrap() else { panic!("Bad tokens for attribute {:?}", attr); }; let pound_span = pound_token.span; @@ -409,7 +409,7 @@ impl<'a> StripUnconfigured<'a> { let mut trees = vec![(AttrAnnotatedTokenTree::Token(pound_token), Spacing::Alone)]; if attr.style == AttrStyle::Inner { // For inner attributes, we do the same thing for the `!` in `#![some_attr]` - let TokenTree::Token(bang_token @ Token { kind: TokenKind::Not, .. }) = orig_trees.next().unwrap() else { + let TokenTree::Token(bang_token @ Token { kind: TokenKind::Not, .. }, _) = orig_trees.next().unwrap() else { panic!("Bad tokens for attribute {:?}", attr); }; trees.push((AttrAnnotatedTokenTree::Token(bang_token), Spacing::Alone)); diff --git a/compiler/rustc_expand/src/mbe/macro_rules.rs b/compiler/rustc_expand/src/mbe/macro_rules.rs index 3d44c408d8f..f7e1575afbf 100644 --- a/compiler/rustc_expand/src/mbe/macro_rules.rs +++ b/compiler/rustc_expand/src/mbe/macro_rules.rs @@ -481,7 +481,7 @@ pub fn compile_declarative_macro( .map(|m| { if let MatchedTokenTree(ref tt) = *m { let tt = mbe::quoted::parse( - tt.clone().into(), + TokenStream::new(vec![tt.clone()]), true, &sess.parse_sess, def.id, @@ -505,7 +505,7 @@ pub fn compile_declarative_macro( .map(|m| { if let MatchedTokenTree(ref tt) = *m { return mbe::quoted::parse( - tt.clone().into(), + TokenStream::new(vec![tt.clone()]), false, &sess.parse_sess, def.id, diff --git a/compiler/rustc_expand/src/mbe/metavar_expr.rs b/compiler/rustc_expand/src/mbe/metavar_expr.rs index 45c462bc425..fc808401a5e 100644 --- a/compiler/rustc_expand/src/mbe/metavar_expr.rs +++ b/compiler/rustc_expand/src/mbe/metavar_expr.rs @@ -106,7 +106,7 @@ fn parse_depth<'sess>( let Some(tt) = iter.next() else { return Ok(0) }; let TokenTree::Token(token::Token { kind: token::TokenKind::Literal(lit), .. - }) = tt else { + }, _) = tt else { return Err(sess.span_diagnostic.struct_span_err( span, "meta-variable expression depth must be a literal" @@ -130,7 +130,7 @@ fn parse_ident<'sess>( sess: &'sess ParseSess, span: Span, ) -> PResult<'sess, Ident> { - if let Some(tt) = iter.next() && let TokenTree::Token(token) = tt { + if let Some(tt) = iter.next() && let TokenTree::Token(token, _) = tt { if let Some((elem, false)) = token.ident() { return Ok(elem); } @@ -153,7 +153,7 @@ fn parse_ident<'sess>( /// Tries to move the iterator forward returning `true` if there is a comma. If not, then the /// iterator is not modified and the result is `false`. fn try_eat_comma(iter: &mut CursorRef<'_>) -> bool { - if let Some(TokenTree::Token(token::Token { kind: token::Comma, .. })) = iter.look_ahead(0) { + if let Some(TokenTree::Token(token::Token { kind: token::Comma, .. }, _)) = iter.look_ahead(0) { let _ = iter.next(); return true; } diff --git a/compiler/rustc_expand/src/mbe/quoted.rs b/compiler/rustc_expand/src/mbe/quoted.rs index 707cb73f097..ee17d54f629 100644 --- a/compiler/rustc_expand/src/mbe/quoted.rs +++ b/compiler/rustc_expand/src/mbe/quoted.rs @@ -56,9 +56,9 @@ pub(super) fn parse( match tree { TokenTree::MetaVar(start_sp, ident) if parsing_patterns => { let span = match trees.next() { - Some(tokenstream::TokenTree::Token(Token { kind: token::Colon, span })) => { + Some(tokenstream::TokenTree::Token(Token { kind: token::Colon, span }, _)) => { match trees.next() { - Some(tokenstream::TokenTree::Token(token)) => match token.ident() { + Some(tokenstream::TokenTree::Token(token, _)) => match token.ident() { Some((frag, _)) => { let span = token.span.with_lo(start_sp.lo()); @@ -146,7 +146,7 @@ fn parse_tree( // Depending on what `tree` is, we could be parsing different parts of a macro match tree { // `tree` is a `$` token. Look at the next token in `trees` - tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }) => { + tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }, _) => { // FIXME: Handle `Invisible`-delimited groups in a more systematic way // during parsing. let mut next = outer_trees.next(); @@ -217,7 +217,7 @@ fn parse_tree( // `tree` is followed by an `ident`. This could be `$meta_var` or the `$crate` // special metavariable that names the crate of the invocation. - Some(tokenstream::TokenTree::Token(token)) if token.is_ident() => { + Some(tokenstream::TokenTree::Token(token, _)) if token.is_ident() => { let (ident, is_raw) = token.ident().unwrap(); let span = ident.span.with_lo(span.lo()); if ident.name == kw::Crate && !is_raw { @@ -228,7 +228,7 @@ fn parse_tree( } // `tree` is followed by another `$`. This is an escaped `$`. - Some(tokenstream::TokenTree::Token(Token { kind: token::Dollar, span })) => { + Some(tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }, _)) => { if parsing_patterns { span_dollar_dollar_or_metavar_in_the_lhs_err( sess, @@ -241,7 +241,7 @@ fn parse_tree( } // `tree` is followed by some other token. This is an error. - Some(tokenstream::TokenTree::Token(token)) => { + Some(tokenstream::TokenTree::Token(token, _)) => { let msg = format!( "expected identifier, found `{}`", pprust::token_to_string(&token), @@ -256,7 +256,7 @@ fn parse_tree( } // `tree` is an arbitrary token. Keep it. - tokenstream::TokenTree::Token(token) => TokenTree::Token(token), + tokenstream::TokenTree::Token(token, _) => TokenTree::Token(token), // `tree` is the beginning of a delimited set of tokens (e.g., `(` or `{`). We need to // descend into the delimited set and further parse it. @@ -291,7 +291,7 @@ fn parse_kleene_op( span: Span, ) -> Result<Result<(KleeneOp, Span), Token>, Span> { match input.next() { - Some(tokenstream::TokenTree::Token(token)) => match kleene_op(&token) { + Some(tokenstream::TokenTree::Token(token, _)) => match kleene_op(&token) { Some(op) => Ok(Ok((op, token.span))), None => Ok(Err(token)), }, diff --git a/compiler/rustc_expand/src/mbe/transcribe.rs b/compiler/rustc_expand/src/mbe/transcribe.rs index 3037855ae28..e47ea83ac38 100644 --- a/compiler/rustc_expand/src/mbe/transcribe.rs +++ b/compiler/rustc_expand/src/mbe/transcribe.rs @@ -3,7 +3,7 @@ use crate::mbe::macro_parser::{MatchedNonterminal, MatchedSeq, MatchedTokenTree, use crate::mbe::{self, MetaVarExpr}; use rustc_ast::mut_visit::{self, MutVisitor}; use rustc_ast::token::{self, Delimiter, Token, TokenKind}; -use rustc_ast::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndSpacing}; +use rustc_ast::tokenstream::{DelimSpan, Spacing, TokenStream, TokenTree}; use rustc_data_structures::fx::FxHashMap; use rustc_errors::{pluralize, PResult}; use rustc_errors::{DiagnosticBuilder, ErrorGuaranteed}; @@ -105,7 +105,7 @@ pub(super) fn transcribe<'a>( // // Thus, if we try to pop the `result_stack` and it is empty, we have reached the top-level // again, and we are done transcribing. - let mut result: Vec<TreeAndSpacing> = Vec::new(); + let mut result: Vec<TokenTree> = Vec::new(); let mut result_stack = Vec::new(); let mut marker = Marker(cx.current_expansion.id, transparency); @@ -123,7 +123,7 @@ pub(super) fn transcribe<'a>( if repeat_idx < repeat_len { *idx = 0; if let Some(sep) = sep { - result.push(TokenTree::Token(sep.clone()).into()); + result.push(TokenTree::Token(sep.clone(), Spacing::Alone)); } continue; } @@ -150,7 +150,7 @@ pub(super) fn transcribe<'a>( // Step back into the parent Delimited. let tree = TokenTree::Delimited(span, delim, TokenStream::new(result)); result = result_stack.pop().unwrap(); - result.push(tree.into()); + result.push(tree); } } continue; @@ -227,15 +227,15 @@ pub(super) fn transcribe<'a>( // `tt`s are emitted into the output stream directly as "raw tokens", // without wrapping them into groups. let token = tt.clone(); - result.push(token.into()); + result.push(token); } MatchedNonterminal(ref nt) => { // Other variables are emitted into the output stream as groups with // `Delimiter::Invisible` to maintain parsing priorities. // `Interpolated` is currently used for such groups in rustc parser. marker.visit_span(&mut sp); - let token = TokenTree::token(token::Interpolated(nt.clone()), sp); - result.push(token.into()); + let token = TokenTree::token_alone(token::Interpolated(nt.clone()), sp); + result.push(token); } MatchedSeq(..) => { // We were unable to descend far enough. This is an error. @@ -250,8 +250,11 @@ pub(super) fn transcribe<'a>( // with modified syntax context. (I believe this supports nested macros). marker.visit_span(&mut sp); marker.visit_ident(&mut original_ident); - result.push(TokenTree::token(token::Dollar, sp).into()); - result.push(TokenTree::Token(Token::from_ast_ident(original_ident)).into()); + result.push(TokenTree::token_alone(token::Dollar, sp)); + result.push(TokenTree::Token( + Token::from_ast_ident(original_ident), + Spacing::Alone, + )); } } @@ -281,8 +284,8 @@ pub(super) fn transcribe<'a>( mbe::TokenTree::Token(token) => { let mut token = token.clone(); mut_visit::visit_token(&mut token, &mut marker); - let tt = TokenTree::Token(token); - result.push(tt.into()); + let tt = TokenTree::Token(token, Spacing::Alone); + result.push(tt); } // There should be no meta-var declarations in the invocation of a macro. @@ -532,7 +535,7 @@ fn transcribe_metavar_expr<'a>( interp: &FxHashMap<MacroRulesNormalizedIdent, NamedMatch>, marker: &mut Marker, repeats: &[(usize, usize)], - result: &mut Vec<TreeAndSpacing>, + result: &mut Vec<TokenTree>, sp: &DelimSpan, ) -> PResult<'a, ()> { let mut visited_span = || { @@ -544,11 +547,11 @@ fn transcribe_metavar_expr<'a>( MetaVarExpr::Count(original_ident, depth_opt) => { let matched = matched_from_ident(cx, original_ident, interp)?; let count = count_repetitions(cx, depth_opt, matched, &repeats, sp)?; - let tt = TokenTree::token( + let tt = TokenTree::token_alone( TokenKind::lit(token::Integer, sym::integer(count), None), visited_span(), ); - result.push(tt.into()); + result.push(tt); } MetaVarExpr::Ignore(original_ident) => { // Used to ensure that `original_ident` is present in the LHS @@ -556,25 +559,19 @@ fn transcribe_metavar_expr<'a>( } MetaVarExpr::Index(depth) => match repeats.iter().nth_back(depth) { Some((index, _)) => { - result.push( - TokenTree::token( - TokenKind::lit(token::Integer, sym::integer(*index), None), - visited_span(), - ) - .into(), - ); + result.push(TokenTree::token_alone( + TokenKind::lit(token::Integer, sym::integer(*index), None), + visited_span(), + )); } None => return Err(out_of_bounds_err(cx, repeats.len(), sp.entire(), "index")), }, MetaVarExpr::Length(depth) => match repeats.iter().nth_back(depth) { Some((_, length)) => { - result.push( - TokenTree::token( - TokenKind::lit(token::Integer, sym::integer(*length), None), - visited_span(), - ) - .into(), - ); + result.push(TokenTree::token_alone( + TokenKind::lit(token::Integer, sym::integer(*length), None), + visited_span(), + )); } None => return Err(out_of_bounds_err(cx, repeats.len(), sp.entire(), "length")), }, diff --git a/compiler/rustc_expand/src/parse/tests.rs b/compiler/rustc_expand/src/parse/tests.rs index 8da78792758..a3c631d3318 100644 --- a/compiler/rustc_expand/src/parse/tests.rs +++ b/compiler/rustc_expand/src/parse/tests.rs @@ -66,23 +66,23 @@ fn string_to_tts_macro() { match tts { [ - TokenTree::Token(Token { kind: token::Ident(name_macro_rules, false), .. }), - TokenTree::Token(Token { kind: token::Not, .. }), - TokenTree::Token(Token { kind: token::Ident(name_zip, false), .. }), + TokenTree::Token(Token { kind: token::Ident(name_macro_rules, false), .. }, _), + TokenTree::Token(Token { kind: token::Not, .. }, _), + TokenTree::Token(Token { kind: token::Ident(name_zip, false), .. }, _), TokenTree::Delimited(_, macro_delim, macro_tts), ] if name_macro_rules == &kw::MacroRules && name_zip.as_str() == "zip" => { let tts = ¯o_tts.trees().collect::<Vec<_>>(); match &tts[..] { [ TokenTree::Delimited(_, first_delim, first_tts), - TokenTree::Token(Token { kind: token::FatArrow, .. }), + TokenTree::Token(Token { kind: token::FatArrow, .. }, _), TokenTree::Delimited(_, second_delim, second_tts), ] if macro_delim == &Delimiter::Parenthesis => { let tts = &first_tts.trees().collect::<Vec<_>>(); match &tts[..] { [ - TokenTree::Token(Token { kind: token::Dollar, .. }), - TokenTree::Token(Token { kind: token::Ident(name, false), .. }), + TokenTree::Token(Token { kind: token::Dollar, .. }, _), + TokenTree::Token(Token { kind: token::Ident(name, false), .. }, _), ] if first_delim == &Delimiter::Parenthesis && name.as_str() == "a" => { } _ => panic!("value 3: {:?} {:?}", first_delim, first_tts), @@ -90,8 +90,8 @@ fn string_to_tts_macro() { let tts = &second_tts.trees().collect::<Vec<_>>(); match &tts[..] { [ - TokenTree::Token(Token { kind: token::Dollar, .. }), - TokenTree::Token(Token { kind: token::Ident(name, false), .. }), + TokenTree::Token(Token { kind: token::Dollar, .. }, _), + TokenTree::Token(Token { kind: token::Ident(name, false), .. }, _), ] if second_delim == &Delimiter::Parenthesis && name.as_str() == "a" => {} _ => panic!("value 4: {:?} {:?}", second_delim, second_tts), @@ -111,29 +111,27 @@ fn string_to_tts_1() { let tts = string_to_stream("fn a (b : i32) { b; }".to_string()); let expected = TokenStream::new(vec![ - TokenTree::token(token::Ident(kw::Fn, false), sp(0, 2)).into(), - TokenTree::token(token::Ident(Symbol::intern("a"), false), sp(3, 4)).into(), + TokenTree::token_alone(token::Ident(kw::Fn, false), sp(0, 2)), + TokenTree::token_alone(token::Ident(Symbol::intern("a"), false), sp(3, 4)), TokenTree::Delimited( DelimSpan::from_pair(sp(5, 6), sp(13, 14)), Delimiter::Parenthesis, TokenStream::new(vec![ - TokenTree::token(token::Ident(Symbol::intern("b"), false), sp(6, 7)).into(), - TokenTree::token(token::Colon, sp(8, 9)).into(), - TokenTree::token(token::Ident(sym::i32, false), sp(10, 13)).into(), + TokenTree::token_alone(token::Ident(Symbol::intern("b"), false), sp(6, 7)), + TokenTree::token_alone(token::Colon, sp(8, 9)), + TokenTree::token_alone(token::Ident(sym::i32, false), sp(10, 13)), ]) .into(), - ) - .into(), + ), TokenTree::Delimited( DelimSpan::from_pair(sp(15, 16), sp(20, 21)), Delimiter::Brace, TokenStream::new(vec![ - TokenTree::token(token::Ident(Symbol::intern("b"), false), sp(17, 18)).into(), - TokenTree::token(token::Semi, sp(18, 19)).into(), + TokenTree::token_joint(token::Ident(Symbol::intern("b"), false), sp(17, 18)), + TokenTree::token_alone(token::Semi, sp(18, 19)), ]) .into(), - ) - .into(), + ), ]); assert_eq!(tts, expected); diff --git a/compiler/rustc_expand/src/proc_macro.rs b/compiler/rustc_expand/src/proc_macro.rs index 1b1078a67ee..1a2ab9d190e 100644 --- a/compiler/rustc_expand/src/proc_macro.rs +++ b/compiler/rustc_expand/src/proc_macro.rs @@ -4,7 +4,7 @@ use crate::proc_macro_server; use rustc_ast as ast; use rustc_ast::ptr::P; use rustc_ast::token; -use rustc_ast::tokenstream::{TokenStream, TokenTree}; +use rustc_ast::tokenstream::TokenStream; use rustc_data_structures::sync::Lrc; use rustc_errors::ErrorGuaranteed; use rustc_parse::parser::ForceCollect; @@ -123,7 +123,7 @@ impl MultiItemModifier for DeriveProcMacro { Annotatable::Stmt(stmt) => token::NtStmt(stmt), _ => unreachable!(), }; - TokenTree::token(token::Interpolated(Lrc::new(nt)), DUMMY_SP).into() + TokenStream::token_alone(token::Interpolated(Lrc::new(nt)), DUMMY_SP) } else { item.to_tokens() }; diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index 176c77ca6ed..7d9a4aed0bf 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -92,9 +92,8 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre let mut trees = Vec::with_capacity(stream.len().next_power_of_two()); let mut cursor = stream.into_trees(); - while let Some((tree, spacing)) = cursor.next_with_spacing() { - let joint = spacing == Joint; - let Token { kind, span } = match tree { + while let Some(tree) = cursor.next() { + let (Token { kind, span }, joint) = match tree { tokenstream::TokenTree::Delimited(span, delim, tts) => { let delimiter = pm::Delimiter::from_internal(delim); trees.push(TokenTree::Group(Group { @@ -108,7 +107,7 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre })); continue; } - tokenstream::TokenTree::Token(token) => token, + tokenstream::TokenTree::Token(token, spacing) => (token, spacing == Joint), }; let mut op = |s: &str| { @@ -194,7 +193,7 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre TokenKind::lit(token::Str, Symbol::intern(&escaped), None), ] .into_iter() - .map(|kind| tokenstream::TokenTree::token(kind, span)) + .map(|kind| tokenstream::TokenTree::token_alone(kind, span)) .collect(); trees.push(TokenTree::Punct(Punct { ch: b'#', joint: false, span })); if attr_style == ast::AttrStyle::Inner { @@ -246,16 +245,15 @@ impl ToInternal<TokenStream> for (TokenTree<TokenStream, Span, Symbol>, &mut Rus let (ch, joint, span) = match tree { TokenTree::Punct(Punct { ch, joint, span }) => (ch, joint, span), TokenTree::Group(Group { delimiter, stream, span: DelimSpan { open, close, .. } }) => { - return tokenstream::TokenTree::Delimited( + return tokenstream::TokenStream::delimited( tokenstream::DelimSpan { open, close }, delimiter.to_internal(), stream.unwrap_or_default(), - ) - .into(); + ); } TokenTree::Ident(self::Ident { sym, is_raw, span }) => { rustc.sess().symbol_gallery.insert(sym, span); - return tokenstream::TokenTree::token(Ident(sym, is_raw), span).into(); + return tokenstream::TokenStream::token_alone(Ident(sym, is_raw), span); } TokenTree::Literal(self::Literal { kind: self::LitKind::Integer, @@ -266,8 +264,8 @@ impl ToInternal<TokenStream> for (TokenTree<TokenStream, Span, Symbol>, &mut Rus let minus = BinOp(BinOpToken::Minus); let symbol = Symbol::intern(&symbol.as_str()[1..]); let integer = TokenKind::lit(token::Integer, symbol, suffix); - let a = tokenstream::TokenTree::token(minus, span); - let b = tokenstream::TokenTree::token(integer, span); + let a = tokenstream::TokenTree::token_alone(minus, span); + let b = tokenstream::TokenTree::token_alone(integer, span); return [a, b].into_iter().collect(); } TokenTree::Literal(self::Literal { @@ -279,16 +277,15 @@ impl ToInternal<TokenStream> for (TokenTree<TokenStream, Span, Symbol>, &mut Rus let minus = BinOp(BinOpToken::Minus); let symbol = Symbol::intern(&symbol.as_str()[1..]); let float = TokenKind::lit(token::Float, symbol, suffix); - let a = tokenstream::TokenTree::token(minus, span); - let b = tokenstream::TokenTree::token(float, span); + let a = tokenstream::TokenTree::token_alone(minus, span); + let b = tokenstream::TokenTree::token_alone(float, span); return [a, b].into_iter().collect(); } TokenTree::Literal(self::Literal { kind, symbol, suffix, span }) => { - return tokenstream::TokenTree::token( + return tokenstream::TokenStream::token_alone( TokenKind::lit(kind.to_internal(), symbol, suffix), span, - ) - .into(); + ); } }; @@ -318,8 +315,11 @@ impl ToInternal<TokenStream> for (TokenTree<TokenStream, Span, Symbol>, &mut Rus _ => unreachable!(), }; - let tree = tokenstream::TokenTree::token(kind, span); - TokenStream::new(vec![(tree, if joint { Joint } else { Alone })]) + if joint { + tokenstream::TokenStream::token_joint(kind, span) + } else { + tokenstream::TokenStream::token_alone(kind, span) + } } } @@ -486,12 +486,11 @@ impl server::TokenStream for Rustc<'_, '_> { // We don't use `TokenStream::from_ast` as the tokenstream currently cannot // be recovered in the general case. match &expr.kind { - ast::ExprKind::Lit(l) if l.token.kind == token::Bool => { - Ok(tokenstream::TokenTree::token(token::Ident(l.token.symbol, false), l.span) - .into()) - } + ast::ExprKind::Lit(l) if l.token.kind == token::Bool => Ok( + tokenstream::TokenStream::token_alone(token::Ident(l.token.symbol, false), l.span), + ), ast::ExprKind::Lit(l) => { - Ok(tokenstream::TokenTree::token(token::Literal(l.token), l.span).into()) + Ok(tokenstream::TokenStream::token_alone(token::Literal(l.token), l.span)) } ast::ExprKind::Unary(ast::UnOp::Neg, e) => match &e.kind { ast::ExprKind::Lit(l) => match l.token { @@ -499,8 +498,8 @@ impl server::TokenStream for Rustc<'_, '_> { Ok(Self::TokenStream::from_iter([ // FIXME: The span of the `-` token is lost when // parsing, so we cannot faithfully recover it here. - tokenstream::TokenTree::token(token::BinOp(token::Minus), e.span), - tokenstream::TokenTree::token(token::Literal(l.token), l.span), + tokenstream::TokenTree::token_alone(token::BinOp(token::Minus), e.span), + tokenstream::TokenTree::token_alone(token::Literal(l.token), l.span), ])) } _ => Err(()), diff --git a/compiler/rustc_expand/src/tokenstream/tests.rs b/compiler/rustc_expand/src/tokenstream/tests.rs index e4a4db204d9..eed69681011 100644 --- a/compiler/rustc_expand/src/tokenstream/tests.rs +++ b/compiler/rustc_expand/src/tokenstream/tests.rs @@ -1,7 +1,7 @@ use crate::tests::string_to_stream; use rustc_ast::token; -use rustc_ast::tokenstream::{Spacing, TokenStream, TokenStreamBuilder, TokenTree}; +use rustc_ast::tokenstream::{TokenStream, TokenStreamBuilder}; use rustc_span::create_default_session_globals_then; use rustc_span::{BytePos, Span, Symbol}; @@ -13,10 +13,6 @@ fn sp(a: u32, b: u32) -> Span { Span::with_root_ctxt(BytePos(a), BytePos(b)) } -fn joint(tree: TokenTree) -> TokenStream { - TokenStream::new(vec![(tree, Spacing::Joint)]) -} - #[test] fn test_concat() { create_default_session_globals_then(|| { @@ -90,9 +86,8 @@ fn test_diseq_1() { #[test] fn test_is_empty() { create_default_session_globals_then(|| { - let test0: TokenStream = Vec::<TokenTree>::new().into_iter().collect(); - let test1: TokenStream = - TokenTree::token(token::Ident(Symbol::intern("a"), false), sp(0, 1)).into(); + let test0 = TokenStream::default(); + let test1 = TokenStream::token_alone(token::Ident(Symbol::intern("a"), false), sp(0, 1)); let test2 = string_to_ts("foo(bar::baz)"); assert_eq!(test0.is_empty(), true); @@ -105,9 +100,9 @@ fn test_is_empty() { fn test_dotdotdot() { create_default_session_globals_then(|| { let mut builder = TokenStreamBuilder::new(); - builder.push(joint(TokenTree::token(token::Dot, sp(0, 1)))); - builder.push(joint(TokenTree::token(token::Dot, sp(1, 2)))); - builder.push(TokenTree::token(token::Dot, sp(2, 3))); + builder.push(TokenStream::token_joint(token::Dot, sp(0, 1))); + builder.push(TokenStream::token_joint(token::Dot, sp(1, 2))); + builder.push(TokenStream::token_alone(token::Dot, sp(2, 3))); let stream = builder.build(); assert!(stream.eq_unspanned(&string_to_ts("..."))); assert_eq!(stream.trees().count(), 1); diff --git a/compiler/rustc_lint/src/builtin.rs b/compiler/rustc_lint/src/builtin.rs index 9e4dc702f07..ca3e6ce4d60 100644 --- a/compiler/rustc_lint/src/builtin.rs +++ b/compiler/rustc_lint/src/builtin.rs @@ -1974,7 +1974,7 @@ impl KeywordIdents { for tt in tokens.into_trees() { match tt { // Only report non-raw idents. - TokenTree::Token(token) => { + TokenTree::Token(token, _) => { if let Some((ident, false)) = token.ident() { self.check_ident_token(cx, UnderMacro(true), ident); } diff --git a/compiler/rustc_parse/src/lexer/tokentrees.rs b/compiler/rustc_parse/src/lexer/tokentrees.rs index ef84f95ec83..0816bc8deb6 100644 --- a/compiler/rustc_parse/src/lexer/tokentrees.rs +++ b/compiler/rustc_parse/src/lexer/tokentrees.rs @@ -1,11 +1,7 @@ use super::{StringReader, UnmatchedBrace}; use rustc_ast::token::{self, Delimiter, Token}; -use rustc_ast::tokenstream::{ - DelimSpan, - Spacing::{self, *}, - TokenStream, TokenTree, TreeAndSpacing, -}; +use rustc_ast::tokenstream::{DelimSpan, Spacing, TokenStream, TokenTree}; use rustc_ast_pretty::pprust::token_to_string; use rustc_data_structures::fx::FxHashMap; use rustc_errors::PResult; @@ -77,7 +73,7 @@ impl<'a> TokenTreesReader<'a> { } } - fn parse_token_tree(&mut self) -> PResult<'a, TreeAndSpacing> { + fn parse_token_tree(&mut self) -> PResult<'a, TokenTree> { let sm = self.string_reader.sess.source_map(); match self.token.kind { @@ -223,7 +219,7 @@ impl<'a> TokenTreesReader<'a> { _ => {} } - Ok(TokenTree::Delimited(delim_span, delim, tts).into()) + Ok(TokenTree::Delimited(delim_span, delim, tts)) } token::CloseDelim(delim) => { // An unexpected closing delimiter (i.e., there is no @@ -258,12 +254,12 @@ impl<'a> TokenTreesReader<'a> { Err(err) } _ => { - let tt = TokenTree::Token(self.token.take()); + let tok = self.token.take(); let mut spacing = self.bump(); if !self.token.is_op() { - spacing = Alone; + spacing = Spacing::Alone; } - Ok((tt, spacing)) + Ok(TokenTree::Token(tok, spacing)) } } } @@ -277,20 +273,20 @@ impl<'a> TokenTreesReader<'a> { #[derive(Default)] struct TokenStreamBuilder { - buf: Vec<TreeAndSpacing>, + buf: Vec<TokenTree>, } impl TokenStreamBuilder { - fn push(&mut self, (tree, joint): TreeAndSpacing) { - if let Some((TokenTree::Token(prev_token), Joint)) = self.buf.last() - && let TokenTree::Token(token) = &tree + fn push(&mut self, tree: TokenTree) { + if let Some(TokenTree::Token(prev_token, Spacing::Joint)) = self.buf.last() + && let TokenTree::Token(token, joint) = &tree && let Some(glued) = prev_token.glue(token) { self.buf.pop(); - self.buf.push((TokenTree::Token(glued), joint)); + self.buf.push(TokenTree::Token(glued, *joint)); return; } - self.buf.push((tree, joint)) + self.buf.push(tree); } fn into_token_stream(self) -> TokenStream { diff --git a/compiler/rustc_parse/src/parser/item.rs b/compiler/rustc_parse/src/parser/item.rs index a14a7fc0610..2c1e5807aa7 100644 --- a/compiler/rustc_parse/src/parser/item.rs +++ b/compiler/rustc_parse/src/parser/item.rs @@ -1664,8 +1664,8 @@ impl<'a> Parser<'a> { let body = self.parse_token_tree(); // `MacBody` // Convert `MacParams MacBody` into `{ MacParams => MacBody }`. let bspan = body.span(); - let arrow = TokenTree::token(token::FatArrow, pspan.between(bspan)); // `=>` - let tokens = TokenStream::new(vec![params.into(), arrow.into(), body.into()]); + let arrow = TokenTree::token_alone(token::FatArrow, pspan.between(bspan)); // `=>` + let tokens = TokenStream::new(vec![params, arrow, body]); let dspan = DelimSpan::from_pair(pspan.shrink_to_lo(), bspan.shrink_to_hi()); P(MacArgs::Delimited(dspan, MacDelimiter::Brace, tokens)) } else { diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index 67e6402c0ae..1ac8b224248 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -268,13 +268,13 @@ impl TokenCursor { // FIXME: we currently don't return `Delimiter` open/close delims. To fix #67062 we will // need to, whereupon the `delim != Delimiter::Invisible` conditions below can be // removed. - if let Some((tree, spacing)) = self.frame.tree_cursor.next_with_spacing_ref() { + if let Some(tree) = self.frame.tree_cursor.next_ref() { match tree { - &TokenTree::Token(ref token) => match (desugar_doc_comments, token) { + &TokenTree::Token(ref token, spacing) => match (desugar_doc_comments, token) { (true, &Token { kind: token::DocComment(_, attr_style, data), span }) => { return self.desugar(attr_style, data, span); } - _ => return (token.clone(), *spacing), + _ => return (token.clone(), spacing), }, &TokenTree::Delimited(sp, delim, ref tts) => { // Set `open_delim` to true here because we deal with it immediately. @@ -318,12 +318,14 @@ impl TokenCursor { delim_span, Delimiter::Bracket, [ - TokenTree::token(token::Ident(sym::doc, false), span), - TokenTree::token(token::Eq, span), - TokenTree::token(TokenKind::lit(token::StrRaw(num_of_hashes), data, None), span), + TokenTree::token_alone(token::Ident(sym::doc, false), span), + TokenTree::token_alone(token::Eq, span), + TokenTree::token_alone( + TokenKind::lit(token::StrRaw(num_of_hashes), data, None), + span, + ), ] - .iter() - .cloned() + .into_iter() .collect::<TokenStream>(), ); @@ -332,14 +334,16 @@ impl TokenCursor { TokenCursorFrame::new( None, if attr_style == AttrStyle::Inner { - [TokenTree::token(token::Pound, span), TokenTree::token(token::Not, span), body] - .iter() - .cloned() - .collect::<TokenStream>() + [ + TokenTree::token_alone(token::Pound, span), + TokenTree::token_alone(token::Not, span), + body, + ] + .into_iter() + .collect::<TokenStream>() } else { - [TokenTree::token(token::Pound, span), body] - .iter() - .cloned() + [TokenTree::token_alone(token::Pound, span), body] + .into_iter() .collect::<TokenStream>() }, ), @@ -1042,7 +1046,7 @@ impl<'a> Parser<'a> { if all_normal { return match frame.tree_cursor.look_ahead(dist - 1) { Some(tree) => match tree { - TokenTree::Token(token) => looker(token), + TokenTree::Token(token, _) => looker(token), TokenTree::Delimited(dspan, delim, _) => { looker(&Token::new(token::OpenDelim(*delim), dspan.open)) } @@ -1226,7 +1230,7 @@ impl<'a> Parser<'a> { token::CloseDelim(_) | token::Eof => unreachable!(), _ => { self.bump(); - TokenTree::Token(self.prev_token.clone()) + TokenTree::Token(self.prev_token.clone(), Spacing::Alone) } } } @@ -1245,7 +1249,7 @@ impl<'a> Parser<'a> { loop { match self.token.kind { token::Eof | token::CloseDelim(..) => break, - _ => result.push(self.parse_token_tree().into()), + _ => result.push(self.parse_token_tree()), } } TokenStream::new(result) diff --git a/src/librustdoc/clean/render_macro_matchers.rs b/src/librustdoc/clean/render_macro_matchers.rs index f7700c43353..ed7683e36fd 100644 --- a/src/librustdoc/clean/render_macro_matchers.rs +++ b/src/librustdoc/clean/render_macro_matchers.rs @@ -43,7 +43,7 @@ pub(super) fn render_macro_matcher(tcx: TyCtxt<'_>, matcher: &TokenTree) -> Stri TokenTree::Delimited(_span, _delim, tts) => print_tts(&mut printer, tts), // Matcher which is not a Delimited is unexpected and should've failed // to compile, but we render whatever it is wrapped in parens. - TokenTree::Token(_) => print_tt(&mut printer, matcher), + TokenTree::Token(..) => print_tt(&mut printer, matcher), } printer.end(); printer.break_offset_if_not_bol(0, -4); @@ -93,7 +93,7 @@ fn snippet_equal_to_token(tcx: TyCtxt<'_>, matcher: &TokenTree) -> Option<String fn print_tt(printer: &mut Printer<'_>, tt: &TokenTree) { match tt { - TokenTree::Token(token) => { + TokenTree::Token(token, _) => { let token_str = printer.token_to_string(token); printer.word(token_str); if let token::DocComment(..) = token.kind { @@ -138,7 +138,7 @@ fn print_tts(printer: &mut Printer<'_>, tts: &TokenStream) { let mut state = Start; for tt in tts.trees() { let (needs_space, next_state) = match &tt { - TokenTree::Token(tt) => match (state, &tt.kind) { + TokenTree::Token(tt, _) => match (state, &tt.kind) { (Dollar, token::Ident(..)) => (false, DollarIdent), (DollarIdent, token::Colon) => (false, DollarIdentColon), (DollarIdentColon, token::Ident(..)) => (false, Other), diff --git a/src/tools/clippy/clippy_lints/src/crate_in_macro_def.rs b/src/tools/clippy/clippy_lints/src/crate_in_macro_def.rs index f6ec8fe7edc..454ec23388a 100644 --- a/src/tools/clippy/clippy_lints/src/crate_in_macro_def.rs +++ b/src/tools/clippy/clippy_lints/src/crate_in_macro_def.rs @@ -110,14 +110,14 @@ fn contains_unhygienic_crate_reference(tts: &TokenStream) -> Option<Span> { fn is_crate_keyword(tt: &TokenTree) -> Option<Span> { if_chain! { - if let TokenTree::Token(Token { kind: TokenKind::Ident(symbol, _), span }) = tt; + if let TokenTree::Token(Token { kind: TokenKind::Ident(symbol, _), span }, _) = tt; if symbol.as_str() == "crate"; then { Some(*span) } else { None } } } fn is_token(tt: &TokenTree, kind: &TokenKind) -> bool { - if let TokenTree::Token(Token { kind: other, .. }) = tt { + if let TokenTree::Token(Token { kind: other, .. }, _) = tt { kind == other } else { false diff --git a/src/tools/rustfmt/src/macros.rs b/src/tools/rustfmt/src/macros.rs index f4b2bcf2815..3a641fab5d6 100644 --- a/src/tools/rustfmt/src/macros.rs +++ b/src/tools/rustfmt/src/macros.rs @@ -13,7 +13,7 @@ use std::collections::HashMap; use std::panic::{catch_unwind, AssertUnwindSafe}; use rustc_ast::token::{BinOpToken, Delimiter, Token, TokenKind}; -use rustc_ast::tokenstream::{Cursor, Spacing, TokenStream, TokenTree}; +use rustc_ast::tokenstream::{Cursor, TokenStream, TokenTree}; use rustc_ast::{ast, ptr}; use rustc_ast_pretty::pprust; use rustc_span::{ @@ -682,7 +682,7 @@ struct MacroArgParser { fn last_tok(tt: &TokenTree) -> Token { match *tt { - TokenTree::Token(ref t) => t.clone(), + TokenTree::Token(ref t, _) => t.clone(), TokenTree::Delimited(delim_span, delim, _) => Token { kind: TokenKind::CloseDelim(delim), span: delim_span.close, @@ -737,10 +737,13 @@ impl MacroArgParser { fn add_meta_variable(&mut self, iter: &mut Cursor) -> Option<()> { match iter.next() { - Some(TokenTree::Token(Token { - kind: TokenKind::Ident(name, _), - .. - })) => { + Some(TokenTree::Token( + Token { + kind: TokenKind::Ident(name, _), + .. + }, + _, + )) => { self.result.push(ParsedMacroArg { kind: MacroArgKind::MetaVariable(name, self.buf.clone()), }); @@ -777,21 +780,30 @@ impl MacroArgParser { } match tok { - TokenTree::Token(Token { - kind: TokenKind::BinOp(BinOpToken::Plus), - .. - }) - | TokenTree::Token(Token { - kind: TokenKind::Question, - .. - }) - | TokenTree::Token(Token { - kind: TokenKind::BinOp(BinOpToken::Star), - .. - }) => { + TokenTree::Token( + Token { + kind: TokenKind::BinOp(BinOpToken::Plus), + .. + }, + _, + ) + | TokenTree::Token( + Token { + kind: TokenKind::Question, + .. + }, + _, + ) + | TokenTree::Token( + Token { + kind: TokenKind::BinOp(BinOpToken::Star), + .. + }, + _, + ) => { break; } - TokenTree::Token(ref t) => { + TokenTree::Token(ref t, _) => { buffer.push_str(&pprust::token_to_string(t)); } _ => return None, @@ -859,10 +871,13 @@ impl MacroArgParser { while let Some(tok) = iter.next() { match tok { - TokenTree::Token(Token { - kind: TokenKind::Dollar, - span, - }) => { + TokenTree::Token( + Token { + kind: TokenKind::Dollar, + span, + }, + _, + ) => { // We always want to add a separator before meta variables. if !self.buf.is_empty() { self.add_separator(); @@ -875,13 +890,16 @@ impl MacroArgParser { span, }; } - TokenTree::Token(Token { - kind: TokenKind::Colon, - .. - }) if self.is_meta_var => { + TokenTree::Token( + Token { + kind: TokenKind::Colon, + .. + }, + _, + ) if self.is_meta_var => { self.add_meta_variable(&mut iter)?; } - TokenTree::Token(ref t) => self.update_buffer(t), + TokenTree::Token(ref t, _) => self.update_buffer(t), TokenTree::Delimited(_delimited_span, delimited, ref tts) => { if !self.buf.is_empty() { if next_space(&self.last_tok.kind) == SpaceState::Always { @@ -1123,12 +1141,15 @@ impl MacroParser { TokenTree::Token(..) => return None, TokenTree::Delimited(delimited_span, d, _) => (delimited_span.open.lo(), d), }; - let args = TokenStream::new(vec![(tok, Spacing::Joint)]); + let args = TokenStream::new(vec![tok]); match self.toks.next()? { - TokenTree::Token(Token { - kind: TokenKind::FatArrow, - .. - }) => {} + TokenTree::Token( + Token { + kind: TokenKind::FatArrow, + .. + }, + _, + ) => {} _ => return None, } let (mut hi, body, whole_body) = match self.toks.next()? { @@ -1147,10 +1168,13 @@ impl MacroParser { ) } }; - if let Some(TokenTree::Token(Token { - kind: TokenKind::Semi, - span, - })) = self.toks.look_ahead(0) + if let Some(TokenTree::Token( + Token { + kind: TokenKind::Semi, + span, + }, + _, + )) = self.toks.look_ahead(0) { hi = span.hi(); self.toks.next(); |