diff options
Diffstat (limited to 'compiler/rustc_ast/src/token.rs')
| -rw-r--r-- | compiler/rustc_ast/src/token.rs | 110 |
1 files changed, 53 insertions, 57 deletions
diff --git a/compiler/rustc_ast/src/token.rs b/compiler/rustc_ast/src/token.rs index 5b41ac8a69e..099a6096d0b 100644 --- a/compiler/rustc_ast/src/token.rs +++ b/compiler/rustc_ast/src/token.rs @@ -9,7 +9,7 @@ use crate::util::case::Case; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_data_structures::sync::Lrc; -use rustc_macros::HashStable_Generic; +use rustc_macros::{Decodable, Encodable, HashStable_Generic}; use rustc_span::symbol::{kw, sym}; #[allow(clippy::useless_attribute)] // FIXME: following use of `hidden_glob_reexports` incorrectly triggers `useless_attribute` lint. #[allow(hidden_glob_reexports)] @@ -111,7 +111,7 @@ impl Lit { Ident(name, IdentIsRaw::No) if name.is_bool_lit() => Some(Lit::new(Bool, name, None)), Literal(token_lit) => Some(token_lit), Interpolated(ref nt) - if let NtExpr(expr) | NtLiteral(expr) = &nt.0 + if let NtExpr(expr) | NtLiteral(expr) = &**nt && let ast::ExprKind::Lit(token_lit) = expr.kind => { Some(token_lit) @@ -167,7 +167,7 @@ impl LitKind { pub fn descr(self) -> &'static str { match self { - Bool => panic!("literal token contains `Lit::Bool`"), + Bool => "boolean", Byte => "byte", Char => "char", Integer => "integer", @@ -318,11 +318,20 @@ pub enum TokenKind { /// It's recommended to use `Token::(ident,uninterpolate,uninterpolated_span)` to /// treat regular and interpolated identifiers in the same way. Ident(Symbol, IdentIsRaw), + /// This identifier (and its span) is the identifier passed to the + /// declarative macro. The span in the surrounding `Token` is the span of + /// the `ident` metavariable in the macro's RHS. + NtIdent(Ident, IdentIsRaw), + /// Lifetime identifier token. /// Do not forget about `NtLifetime` when you want to match on lifetime identifiers. /// It's recommended to use `Token::(lifetime,uninterpolate,uninterpolated_span)` to /// treat regular and interpolated lifetime identifiers in the same way. Lifetime(Symbol), + /// This identifier (and its span) is the lifetime passed to the + /// declarative macro. The span in the surrounding `Token` is the span of + /// the `lifetime` metavariable in the macro's RHS. + NtLifetime(Ident), /// An embedded AST node, as produced by a macro. This only exists for /// historical reasons. We'd like to get rid of it, for multiple reasons. @@ -333,7 +342,11 @@ pub enum TokenKind { /// - It prevents `Token` from implementing `Copy`. /// It adds complexity and likely slows things down. Please don't add new /// occurrences of this token kind! - Interpolated(Lrc<(Nonterminal, Span)>), + /// + /// The span in the surrounding `Token` is that of the metavariable in the + /// macro's RHS. The span within the Nonterminal is that of the fragment + /// passed to the macro at the call site. + Interpolated(Lrc<Nonterminal>), /// A doc comment token. /// `Symbol` is the doc comment's data excluding its "quotes" (`///`, `/**`, etc) @@ -440,8 +453,9 @@ impl Token { /// Note that keywords are also identifiers, so they should use this /// if they keep spans or perform edition checks. pub fn uninterpolated_span(&self) -> Span { - match &self.kind { - Interpolated(nt) => nt.0.use_span(), + match self.kind { + NtIdent(ident, _) | NtLifetime(ident) => ident.span, + Interpolated(ref nt) => nt.use_span(), _ => self.span, } } @@ -459,7 +473,7 @@ impl Token { } OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | Ident(..) - | Lifetime(..) | Interpolated(..) | Eof => false, + | NtIdent(..) | Lifetime(..) | NtLifetime(..) | Interpolated(..) | Eof => false, } } @@ -486,7 +500,7 @@ impl Token { PathSep | // global path Lifetime(..) | // labeled loop Pound => true, // expression attributes - Interpolated(ref nt) => matches!(&nt.0, NtLiteral(..) | + Interpolated(ref nt) => matches!(&**nt, NtLiteral(..) | NtExpr(..) | NtBlock(..) | NtPath(..)), @@ -510,7 +524,7 @@ impl Token { | DotDot | DotDotDot | DotDotEq // ranges | Lt | BinOp(Shl) // associated path | PathSep => true, // global path - Interpolated(ref nt) => matches!(&nt.0, NtLiteral(..) | + Interpolated(ref nt) => matches!(&**nt, NtLiteral(..) | NtPat(..) | NtBlock(..) | NtPath(..)), @@ -533,7 +547,7 @@ impl Token { Lifetime(..) | // lifetime bound in trait object Lt | BinOp(Shl) | // associated path PathSep => true, // global path - Interpolated(ref nt) => matches!(&nt.0, NtTy(..) | NtPath(..)), + Interpolated(ref nt) => matches!(&**nt, NtTy(..) | NtPath(..)), // For anonymous structs or unions, which only appear in specific positions // (type of struct fields or union fields), we don't consider them as regular types _ => false, @@ -544,7 +558,7 @@ impl Token { pub fn can_begin_const_arg(&self) -> bool { match self.kind { OpenDelim(Delimiter::Brace) => true, - Interpolated(ref nt) => matches!(&nt.0, NtExpr(..) | NtBlock(..) | NtLiteral(..)), + Interpolated(ref nt) => matches!(&**nt, NtExpr(..) | NtBlock(..) | NtLiteral(..)), _ => self.can_begin_literal_maybe_minus(), } } @@ -589,7 +603,7 @@ impl Token { match self.uninterpolate().kind { Literal(..) | BinOp(Minus) => true, Ident(name, IdentIsRaw::No) if name.is_bool_lit() => true, - Interpolated(ref nt) => match &nt.0 { + Interpolated(ref nt) => match &**nt { NtLiteral(_) => true, NtExpr(e) => match &e.kind { ast::ExprKind::Lit(_) => true, @@ -609,14 +623,9 @@ impl Token { /// into the regular identifier or lifetime token it refers to, /// otherwise returns the original token. pub fn uninterpolate(&self) -> Cow<'_, Token> { - match &self.kind { - Interpolated(nt) => match &nt.0 { - NtIdent(ident, is_raw) => { - Cow::Owned(Token::new(Ident(ident.name, *is_raw), ident.span)) - } - NtLifetime(ident) => Cow::Owned(Token::new(Lifetime(ident.name), ident.span)), - _ => Cow::Borrowed(self), - }, + match self.kind { + NtIdent(ident, is_raw) => Cow::Owned(Token::new(Ident(ident.name, is_raw), ident.span)), + NtLifetime(ident) => Cow::Owned(Token::new(Lifetime(ident.name), ident.span)), _ => Cow::Borrowed(self), } } @@ -625,12 +634,9 @@ impl Token { #[inline] pub fn ident(&self) -> Option<(Ident, IdentIsRaw)> { // We avoid using `Token::uninterpolate` here because it's slow. - match &self.kind { - &Ident(name, is_raw) => Some((Ident::new(name, self.span), is_raw)), - Interpolated(nt) => match &nt.0 { - NtIdent(ident, is_raw) => Some((*ident, *is_raw)), - _ => None, - }, + match self.kind { + Ident(name, is_raw) => Some((Ident::new(name, self.span), is_raw)), + NtIdent(ident, is_raw) => Some((ident, is_raw)), _ => None, } } @@ -639,12 +645,9 @@ impl Token { #[inline] pub fn lifetime(&self) -> Option<Ident> { // We avoid using `Token::uninterpolate` here because it's slow. - match &self.kind { - &Lifetime(name) => Some(Ident::new(name, self.span)), - Interpolated(nt) => match &nt.0 { - NtLifetime(ident) => Some(*ident), - _ => None, - }, + match self.kind { + Lifetime(name) => Some(Ident::new(name, self.span)), + NtLifetime(ident) => Some(ident), _ => None, } } @@ -668,7 +671,7 @@ impl Token { /// Returns `true` if the token is an interpolated path. fn is_whole_path(&self) -> bool { if let Interpolated(nt) = &self.kind - && let NtPath(..) = &nt.0 + && let NtPath(..) = &**nt { return true; } @@ -681,7 +684,7 @@ impl Token { /// (which happens while parsing the result of macro expansion)? pub fn is_whole_expr(&self) -> bool { if let Interpolated(nt) = &self.kind - && let NtExpr(_) | NtLiteral(_) | NtPath(_) | NtBlock(_) = &nt.0 + && let NtExpr(_) | NtLiteral(_) | NtPath(_) | NtBlock(_) = &**nt { return true; } @@ -692,7 +695,7 @@ impl Token { /// Is the token an interpolated block (`$b:block`)? pub fn is_whole_block(&self) -> bool { if let Interpolated(nt) = &self.kind - && let NtBlock(..) = &nt.0 + && let NtBlock(..) = &**nt { return true; } @@ -833,8 +836,10 @@ impl Token { Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | BinOpEq(..) | At | DotDotDot | DotDotEq | Comma | Semi | PathSep | RArrow | LArrow | FatArrow | Pound | Dollar - | Question | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..) - | Lifetime(..) | Interpolated(..) | DocComment(..) | Eof => return None, + | Question | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..) | NtIdent(..) + | Lifetime(..) | NtLifetime(..) | Interpolated(..) | DocComment(..) | Eof => { + return None; + } }; Some(Token::new(kind, self.span.to(joint.span))) @@ -857,8 +862,6 @@ pub enum Nonterminal { NtPat(P<ast::Pat>), NtExpr(P<ast::Expr>), NtTy(P<ast::Ty>), - NtIdent(Ident, IdentIsRaw), - NtLifetime(Ident), NtLiteral(P<ast::Expr>), /// Stuff inside brackets for attributes NtMeta(P<ast::AttrItem>), @@ -878,6 +881,8 @@ pub enum NonterminalKind { }, PatWithOr, Expr, + /// Matches an expression using the rules from edition 2021 and earlier. + Expr2021, Ty, Ident, Lifetime, @@ -907,6 +912,7 @@ impl NonterminalKind { }, sym::pat_param => NonterminalKind::PatParam { inferred: false }, sym::expr => NonterminalKind::Expr, + sym::expr_2021 if edition().at_least_rust_2021() => NonterminalKind::Expr2021, sym::ty => NonterminalKind::Ty, sym::ident => NonterminalKind::Ident, sym::lifetime => NonterminalKind::Lifetime, @@ -926,6 +932,7 @@ impl NonterminalKind { NonterminalKind::PatParam { inferred: false } => sym::pat_param, NonterminalKind::PatParam { inferred: true } | NonterminalKind::PatWithOr => sym::pat, NonterminalKind::Expr => sym::expr, + NonterminalKind::Expr2021 => sym::expr_2021, NonterminalKind::Ty => sym::ty, NonterminalKind::Ident => sym::ident, NonterminalKind::Lifetime => sym::lifetime, @@ -953,7 +960,6 @@ impl Nonterminal { NtPat(pat) => pat.span, NtExpr(expr) | NtLiteral(expr) => expr.span, NtTy(ty) => ty.span, - NtIdent(ident, _) | NtLifetime(ident) => ident.span, NtMeta(attr_item) => attr_item.span(), NtPath(path) => path.span, NtVis(vis) => vis.span, @@ -969,8 +975,6 @@ impl Nonterminal { NtExpr(..) => "expression", NtLiteral(..) => "literal", NtTy(..) => "type", - NtIdent(..) => "identifier", - NtLifetime(..) => "lifetime", NtMeta(..) => "attribute", NtPath(..) => "path", NtVis(..) => "visibility", @@ -979,18 +983,12 @@ impl Nonterminal { } impl PartialEq for Nonterminal { - fn eq(&self, rhs: &Self) -> bool { - match (self, rhs) { - (NtIdent(ident_lhs, is_raw_lhs), NtIdent(ident_rhs, is_raw_rhs)) => { - ident_lhs == ident_rhs && is_raw_lhs == is_raw_rhs - } - (NtLifetime(ident_lhs), NtLifetime(ident_rhs)) => ident_lhs == ident_rhs, - // FIXME: Assume that all "complex" nonterminal are not equal, we can't compare them - // correctly based on data from AST. This will prevent them from matching each other - // in macros. The comparison will become possible only when each nonterminal has an - // attached token stream from which it was parsed. - _ => false, - } + fn eq(&self, _rhs: &Self) -> bool { + // FIXME: Assume that all nonterminals are not equal, we can't compare them + // correctly based on data from AST. This will prevent them from matching each other + // in macros. The comparison will become possible only when each nonterminal has an + // attached token stream from which it was parsed. + false } } @@ -1003,12 +1001,10 @@ impl fmt::Debug for Nonterminal { NtPat(..) => f.pad("NtPat(..)"), NtExpr(..) => f.pad("NtExpr(..)"), NtTy(..) => f.pad("NtTy(..)"), - NtIdent(..) => f.pad("NtIdent(..)"), NtLiteral(..) => f.pad("NtLiteral(..)"), NtMeta(..) => f.pad("NtMeta(..)"), NtPath(..) => f.pad("NtPath(..)"), NtVis(..) => f.pad("NtVis(..)"), - NtLifetime(..) => f.pad("NtLifetime(..)"), } } } @@ -1023,7 +1019,7 @@ where } // Some types are used a lot. Make sure they don't unintentionally get bigger. -#[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64"), target_pointer_width = "64"))] +#[cfg(target_pointer_width = "64")] mod size_asserts { use super::*; use rustc_data_structures::static_assert_size; |