about summary refs log tree commit diff
path: root/src/libsyntax
diff options
context:
space:
mode:
Diffstat (limited to 'src/libsyntax')
-rw-r--r--src/libsyntax/attr/mod.rs5
-rw-r--r--src/libsyntax/ext/tt/macro_parser.rs28
-rw-r--r--src/libsyntax/ext/tt/macro_rules.rs34
-rw-r--r--src/libsyntax/ext/tt/quoted.rs34
-rw-r--r--src/libsyntax/ext/tt/transcribe.rs21
-rw-r--r--src/libsyntax/parse/diagnostics.rs2
-rw-r--r--src/libsyntax/parse/lexer/mod.rs6
-rw-r--r--src/libsyntax/parse/lexer/tokentrees.rs2
-rw-r--r--src/libsyntax/parse/mod.rs4
-rw-r--r--src/libsyntax/parse/parser.rs26
-rw-r--r--src/libsyntax/parse/token.rs195
-rw-r--r--src/libsyntax/print/pprust.rs12
-rw-r--r--src/libsyntax/tokenstream.rs13
-rw-r--r--src/libsyntax/util/parser.rs6
14 files changed, 160 insertions, 228 deletions
diff --git a/src/libsyntax/attr/mod.rs b/src/libsyntax/attr/mod.rs
index b5d9b761773..d7e43f645df 100644
--- a/src/libsyntax/attr/mod.rs
+++ b/src/libsyntax/attr/mod.rs
@@ -20,7 +20,7 @@ use crate::source_map::{BytePos, Spanned, dummy_spanned};
 use crate::parse::lexer::comments::{doc_comment_style, strip_doc_comment_decoration};
 use crate::parse::parser::Parser;
 use crate::parse::{self, ParseSess, PResult};
-use crate::parse::token::{self, Token, TokenKind};
+use crate::parse::token::{self, Token};
 use crate::ptr::P;
 use crate::symbol::{sym, Symbol};
 use crate::ThinVec;
@@ -467,8 +467,7 @@ impl MetaItem {
                                              segment.ident.span.ctxt());
                 idents.push(TokenTree::token(token::ModSep, mod_sep_span).into());
             }
-            idents.push(TokenTree::token(TokenKind::from_ast_ident(segment.ident),
-                                         segment.ident.span).into());
+            idents.push(TokenTree::Token(Token::from_ast_ident(segment.ident)).into());
             last_pos = segment.ident.span.hi();
         }
         self.node.tokens(self.span).append_to_tree_and_joint_vec(&mut idents);
diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs
index f98e1433356..4758b6a50e5 100644
--- a/src/libsyntax/ext/tt/macro_parser.rs
+++ b/src/libsyntax/ext/tt/macro_parser.rs
@@ -78,7 +78,7 @@ use crate::ast::{Ident, Name};
 use crate::ext::tt::quoted::{self, TokenTree};
 use crate::parse::{Directory, ParseSess};
 use crate::parse::parser::{Parser, PathStyle};
-use crate::parse::token::{self, DocComment, Nonterminal, Token, TokenKind};
+use crate::parse::token::{self, DocComment, Nonterminal, Token};
 use crate::print::pprust;
 use crate::symbol::{kw, sym, Symbol};
 use crate::tokenstream::{DelimSpan, TokenStream};
@@ -199,7 +199,7 @@ struct MatcherPos<'root, 'tt: 'root> {
     seq_op: Option<quoted::KleeneOp>,
 
     /// The separator if we are in a repetition.
-    sep: Option<TokenKind>,
+    sep: Option<Token>,
 
     /// The "parent" matcher position if we are in a repetition. That is, the matcher position just
     /// before we enter the sequence.
@@ -417,24 +417,24 @@ fn nameize<I: Iterator<Item = NamedMatch>>(
 
 /// Generates an appropriate parsing failure message. For EOF, this is "unexpected end...". For
 /// other tokens, this is "unexpected token...".
-pub fn parse_failure_msg(tok: TokenKind) -> String {
-    match tok {
+pub fn parse_failure_msg(tok: &Token) -> String {
+    match tok.kind {
         token::Eof => "unexpected end of macro invocation".to_string(),
         _ => format!(
             "no rules expected the token `{}`",
-            pprust::token_to_string(&tok)
+            pprust::token_to_string(tok)
         ),
     }
 }
 
 /// Performs a token equality check, ignoring syntax context (that is, an unhygienic comparison)
-fn token_name_eq(t1: &TokenKind, t2: &TokenKind) -> bool {
-    if let (Some((name1, is_raw1)), Some((name2, is_raw2))) = (t1.ident_name(), t2.ident_name()) {
-        name1 == name2 && is_raw1 == is_raw2
-    } else if let (Some(name1), Some(name2)) = (t1.lifetime_name(), t2.lifetime_name()) {
-        name1 == name2
+fn token_name_eq(t1: &Token, t2: &Token) -> bool {
+    if let (Some((ident1, is_raw1)), Some((ident2, is_raw2))) = (t1.ident(), t2.ident()) {
+        ident1.name == ident2.name && is_raw1 == is_raw2
+    } else if let (Some(ident1), Some(ident2)) = (t1.lifetime(), t2.lifetime()) {
+        ident1.name == ident2.name
     } else {
-        *t1 == *t2
+        t1.kind == t2.kind
     }
 }
 
@@ -712,7 +712,7 @@ pub fn parse(
 
         // If we reached the EOF, check that there is EXACTLY ONE possible matcher. Otherwise,
         // either the parse is ambiguous (which should never happen) or there is a syntax error.
-        if token_name_eq(&parser.token, &token::Eof) {
+        if parser.token == token::Eof {
             if eof_items.len() == 1 {
                 let matches = eof_items[0]
                     .matches
@@ -804,8 +804,8 @@ pub fn parse(
 
 /// The token is an identifier, but not `_`.
 /// We prohibit passing `_` to macros expecting `ident` for now.
-fn get_macro_name(token: &TokenKind) -> Option<(Name, bool)> {
-    match *token {
+fn get_macro_name(token: &Token) -> Option<(Name, bool)> {
+    match token.kind {
         token::Ident(name, is_raw) if name != kw::Underscore => Some((name, is_raw)),
         _ => None,
     }
diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs
index b9b93b954fe..6f82f509465 100644
--- a/src/libsyntax/ext/tt/macro_rules.rs
+++ b/src/libsyntax/ext/tt/macro_rules.rs
@@ -17,7 +17,7 @@ use crate::symbol::{Symbol, kw, sym};
 use crate::tokenstream::{DelimSpan, TokenStream, TokenTree};
 
 use errors::FatalError;
-use syntax_pos::{Span, DUMMY_SP, symbol::Ident};
+use syntax_pos::{Span, symbol::Ident};
 use log::debug;
 
 use rustc_data_structures::fx::{FxHashMap};
@@ -200,7 +200,7 @@ fn generic_extension<'cx>(cx: &'cx mut ExtCtxt<'_>,
 
     let (token, label) = best_failure.expect("ran no matchers");
     let span = token.span.substitute_dummy(sp);
-    let mut err = cx.struct_span_err(span, &parse_failure_msg(token.kind));
+    let mut err = cx.struct_span_err(span, &parse_failure_msg(&token));
     err.span_label(span, label);
     if let Some(sp) = def_span {
         if cx.source_map().span_to_filename(sp).is_real() && !sp.is_dummy() {
@@ -266,17 +266,19 @@ pub fn compile(
     let argument_gram = vec![
         quoted::TokenTree::Sequence(DelimSpan::dummy(), Lrc::new(quoted::SequenceRepetition {
             tts: vec![
-                quoted::TokenTree::MetaVarDecl(DUMMY_SP, lhs_nm, ast::Ident::from_str("tt")),
-                quoted::TokenTree::token(token::FatArrow, DUMMY_SP),
-                quoted::TokenTree::MetaVarDecl(DUMMY_SP, rhs_nm, ast::Ident::from_str("tt")),
+                quoted::TokenTree::MetaVarDecl(def.span, lhs_nm, ast::Ident::from_str("tt")),
+                quoted::TokenTree::token(token::FatArrow, def.span),
+                quoted::TokenTree::MetaVarDecl(def.span, rhs_nm, ast::Ident::from_str("tt")),
             ],
-            separator: Some(if body.legacy { token::Semi } else { token::Comma }),
+            separator: Some(Token::new(
+                if body.legacy { token::Semi } else { token::Comma }, def.span
+            )),
             op: quoted::KleeneOp::OneOrMore,
             num_captures: 2,
         })),
         // to phase into semicolon-termination instead of semicolon-separation
         quoted::TokenTree::Sequence(DelimSpan::dummy(), Lrc::new(quoted::SequenceRepetition {
-            tts: vec![quoted::TokenTree::token(token::Semi, DUMMY_SP)],
+            tts: vec![quoted::TokenTree::token(token::Semi, def.span)],
             separator: None,
             op: quoted::KleeneOp::ZeroOrMore,
             num_captures: 0
@@ -286,7 +288,7 @@ pub fn compile(
     let argument_map = match parse(sess, body.stream(), &argument_gram, None, true) {
         Success(m) => m,
         Failure(token, msg) => {
-            let s = parse_failure_msg(token.kind);
+            let s = parse_failure_msg(&token);
             let sp = token.span.substitute_dummy(def.span);
             let mut err = sess.span_diagnostic.struct_span_fatal(sp, &s);
             err.span_label(sp, msg);
@@ -608,9 +610,8 @@ impl FirstSets {
                         // If the sequence contents can be empty, then the first
                         // token could be the separator token itself.
 
-                        if let (Some(ref sep), true) = (seq_rep.separator.clone(),
-                                                        subfirst.maybe_empty) {
-                            first.add_one_maybe(TokenTree::token(sep.clone(), sp.entire()));
+                        if let (Some(sep), true) = (&seq_rep.separator, subfirst.maybe_empty) {
+                            first.add_one_maybe(TokenTree::Token(sep.clone()));
                         }
 
                         // Reverse scan: Sequence comes before `first`.
@@ -658,9 +659,8 @@ impl FirstSets {
                             // If the sequence contents can be empty, then the first
                             // token could be the separator token itself.
 
-                            if let (Some(ref sep), true) = (seq_rep.separator.clone(),
-                                                            subfirst.maybe_empty) {
-                                first.add_one_maybe(TokenTree::token(sep.clone(), sp.entire()));
+                            if let (Some(sep), true) = (&seq_rep.separator, subfirst.maybe_empty) {
+                                first.add_one_maybe(TokenTree::Token(sep.clone()));
                             }
 
                             assert!(first.maybe_empty);
@@ -851,7 +851,7 @@ fn check_matcher_core(sess: &ParseSess,
                 // against SUFFIX
                 continue 'each_token;
             }
-            TokenTree::Sequence(sp, ref seq_rep) => {
+            TokenTree::Sequence(_, ref seq_rep) => {
                 suffix_first = build_suffix_first();
                 // The trick here: when we check the interior, we want
                 // to include the separator (if any) as a potential
@@ -864,9 +864,9 @@ fn check_matcher_core(sess: &ParseSess,
                 // work of cloning it? But then again, this way I may
                 // get a "tighter" span?
                 let mut new;
-                let my_suffix = if let Some(ref u) = seq_rep.separator {
+                let my_suffix = if let Some(sep) = &seq_rep.separator {
                     new = suffix_first.clone();
-                    new.add_one_maybe(TokenTree::token(u.clone(), sp.entire()));
+                    new.add_one_maybe(TokenTree::Token(sep.clone()));
                     &new
                 } else {
                     &suffix_first
diff --git a/src/libsyntax/ext/tt/quoted.rs b/src/libsyntax/ext/tt/quoted.rs
index ec7d7f705d8..707fb65bcc5 100644
--- a/src/libsyntax/ext/tt/quoted.rs
+++ b/src/libsyntax/ext/tt/quoted.rs
@@ -23,16 +23,6 @@ pub struct Delimited {
 }
 
 impl Delimited {
-    /// Returns the opening delimiter (possibly `NoDelim`).
-    pub fn open_token(&self) -> TokenKind {
-        token::OpenDelim(self.delim)
-    }
-
-    /// Returns the closing delimiter (possibly `NoDelim`).
-    pub fn close_token(&self) -> TokenKind {
-        token::CloseDelim(self.delim)
-    }
-
     /// Returns a `self::TokenTree` with a `Span` corresponding to the opening delimiter.
     pub fn open_tt(&self, span: Span) -> TokenTree {
         let open_span = if span.is_dummy() {
@@ -40,7 +30,7 @@ impl Delimited {
         } else {
             span.with_lo(span.lo() + BytePos(self.delim.len() as u32))
         };
-        TokenTree::token(self.open_token(), open_span)
+        TokenTree::token(token::OpenDelim(self.delim), open_span)
     }
 
     /// Returns a `self::TokenTree` with a `Span` corresponding to the closing delimiter.
@@ -50,7 +40,7 @@ impl Delimited {
         } else {
             span.with_lo(span.hi() - BytePos(self.delim.len() as u32))
         };
-        TokenTree::token(self.close_token(), close_span)
+        TokenTree::token(token::CloseDelim(self.delim), close_span)
     }
 }
 
@@ -59,7 +49,7 @@ pub struct SequenceRepetition {
     /// The sequence of token trees
     pub tts: Vec<TokenTree>,
     /// The optional separator
-    pub separator: Option<TokenKind>,
+    pub separator: Option<Token>,
     /// Whether the sequence can be repeated zero (*), or one or more times (+)
     pub op: KleeneOp,
     /// The number of `Match`s that appear in the sequence (and subsequences)
@@ -282,7 +272,7 @@ where
             Some(tokenstream::TokenTree::Delimited(span, delim, tts)) => {
                 // Must have `(` not `{` or `[`
                 if delim != token::Paren {
-                    let tok = pprust::token_to_string(&token::OpenDelim(delim));
+                    let tok = pprust::token_kind_to_string(&token::OpenDelim(delim));
                     let msg = format!("expected `(`, found `{}`", tok);
                     sess.span_diagnostic.span_err(span.entire(), &msg);
                 }
@@ -371,8 +361,8 @@ where
 
 /// Takes a token and returns `Some(KleeneOp)` if the token is `+` `*` or `?`. Otherwise, return
 /// `None`.
-fn kleene_op(token: &TokenKind) -> Option<KleeneOp> {
-    match *token {
+fn kleene_op(token: &Token) -> Option<KleeneOp> {
+    match token.kind {
         token::BinOp(token::Star) => Some(KleeneOp::ZeroOrMore),
         token::BinOp(token::Plus) => Some(KleeneOp::OneOrMore),
         token::Question => Some(KleeneOp::ZeroOrOne),
@@ -424,7 +414,7 @@ fn parse_sep_and_kleene_op<I>(
     attrs: &[ast::Attribute],
     edition: Edition,
     macro_node_id: NodeId,
-) -> (Option<TokenKind>, KleeneOp)
+) -> (Option<Token>, KleeneOp)
 where
     I: Iterator<Item = tokenstream::TokenTree>,
 {
@@ -449,7 +439,7 @@ fn parse_sep_and_kleene_op_2015<I>(
     _features: &Features,
     _attrs: &[ast::Attribute],
     macro_node_id: NodeId,
-) -> (Option<TokenKind>, KleeneOp)
+) -> (Option<Token>, KleeneOp)
 where
     I: Iterator<Item = tokenstream::TokenTree>,
 {
@@ -502,7 +492,7 @@ where
                              a hard error in an upcoming edition",
                         );
 
-                        return (Some(token::Question), op);
+                        return (Some(Token::new(token::Question, op1_span)), op);
                     }
 
                     // #2 is a random token (this is an error) :(
@@ -541,7 +531,7 @@ where
             }
 
             // #2 is a KleeneOp :D
-            Ok(Ok((op, _))) => return (Some(token.kind), op),
+            Ok(Ok((op, _))) => return (Some(token), op),
 
             // #2 is a random token :(
             Ok(Err(token)) => token.span,
@@ -567,7 +557,7 @@ fn parse_sep_and_kleene_op_2018<I>(
     sess: &ParseSess,
     _features: &Features,
     _attrs: &[ast::Attribute],
-) -> (Option<TokenKind>, KleeneOp)
+) -> (Option<Token>, KleeneOp)
 where
     I: Iterator<Item = tokenstream::TokenTree>,
 {
@@ -596,7 +586,7 @@ where
             }
 
             // #2 is a KleeneOp :D
-            Ok(Ok((op, _))) => return (Some(token.kind), op),
+            Ok(Ok((op, _))) => return (Some(token), op),
 
             // #2 is a random token :(
             Ok(Err(token)) => token.span,
diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/tt/transcribe.rs
index 90a9cc8f34d..c51f4b20c31 100644
--- a/src/libsyntax/ext/tt/transcribe.rs
+++ b/src/libsyntax/ext/tt/transcribe.rs
@@ -4,11 +4,10 @@ use crate::ext::expand::Marker;
 use crate::ext::tt::macro_parser::{MatchedNonterminal, MatchedSeq, NamedMatch};
 use crate::ext::tt::quoted;
 use crate::mut_visit::noop_visit_tt;
-use crate::parse::token::{self, NtTT, TokenKind};
+use crate::parse::token::{self, NtTT, Token};
 use crate::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndJoint};
 
 use smallvec::{smallvec, SmallVec};
-use syntax_pos::DUMMY_SP;
 
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::sync::Lrc;
@@ -18,7 +17,7 @@ use std::rc::Rc;
 /// An iterator over the token trees in a delimited token tree (`{ ... }`) or a sequence (`$(...)`).
 enum Frame {
     Delimited { forest: Lrc<quoted::Delimited>, idx: usize, span: DelimSpan },
-    Sequence { forest: Lrc<quoted::SequenceRepetition>, idx: usize, sep: Option<TokenKind> },
+    Sequence { forest: Lrc<quoted::SequenceRepetition>, idx: usize, sep: Option<Token> },
 }
 
 impl Frame {
@@ -109,17 +108,13 @@ pub fn transcribe(
         else {
             // Otherwise, if we have just reached the end of a sequence and we can keep repeating,
             // go back to the beginning of the sequence.
-            if let Frame::Sequence { ref mut idx, ref sep, .. } = *stack.last_mut().unwrap() {
-                let (ref mut repeat_idx, repeat_len) = *repeats.last_mut().unwrap();
+            if let Frame::Sequence { idx, sep, .. } = stack.last_mut().unwrap() {
+                let (repeat_idx, repeat_len) = repeats.last_mut().unwrap();
                 *repeat_idx += 1;
-                if *repeat_idx < repeat_len {
+                if repeat_idx < repeat_len {
                     *idx = 0;
-                    if let Some(sep) = sep.clone() {
-                        let prev_span = match result.last() {
-                            Some((tt, _)) => tt.span(),
-                            None => DUMMY_SP,
-                        };
-                        result.push(TokenTree::token(sep, prev_span).into());
+                    if let Some(sep) = sep {
+                        result.push(TokenTree::Token(sep.clone()).into());
                     }
                     continue;
                 }
@@ -242,7 +237,7 @@ pub fn transcribe(
                         Ident::new(ident.name, ident.span.apply_mark(cx.current_expansion.mark));
                     sp = sp.apply_mark(cx.current_expansion.mark);
                     result.push(TokenTree::token(token::Dollar, sp).into());
-                    result.push(TokenTree::token(TokenKind::from_ast_ident(ident), sp).into());
+                    result.push(TokenTree::Token(Token::from_ast_ident(ident)).into());
                 }
             }
 
diff --git a/src/libsyntax/parse/diagnostics.rs b/src/libsyntax/parse/diagnostics.rs
index c4db9a9df45..9d2ac5b4b51 100644
--- a/src/libsyntax/parse/diagnostics.rs
+++ b/src/libsyntax/parse/diagnostics.rs
@@ -729,7 +729,7 @@ impl<'a> Parser<'a> {
         &mut self,
         t: &TokenKind,
     ) -> PResult<'a, bool /* recovered */> {
-        let token_str = pprust::token_to_string(t);
+        let token_str = pprust::token_kind_to_string(t);
         let this_token_str = self.this_token_descr();
         let (prev_sp, sp) = match (&self.token.kind, self.subparser_name) {
             // Point at the end of the macro call when reaching end of macro arguments.
diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs
index e3d959c2c54..2f4c48d4bf9 100644
--- a/src/libsyntax/parse/lexer/mod.rs
+++ b/src/libsyntax/parse/lexer/mod.rs
@@ -1501,7 +1501,7 @@ fn char_at(s: &str, byte: usize) -> char {
 mod tests {
     use super::*;
 
-    use crate::ast::{Ident, CrateConfig};
+    use crate::ast::CrateConfig;
     use crate::symbol::Symbol;
     use crate::source_map::{SourceMap, FilePathMapping};
     use crate::feature_gate::UnstableFeatures;
@@ -1562,7 +1562,7 @@ mod tests {
             assert_eq!(string_reader.next_token(), token::Whitespace);
             let tok1 = string_reader.next_token();
             let tok2 = Token::new(
-                token::Ident(Symbol::intern("fn"), false),
+                mk_ident("fn"),
                 Span::new(BytePos(21), BytePos(23), NO_EXPANSION),
             );
             assert_eq!(tok1.kind, tok2.kind);
@@ -1593,7 +1593,7 @@ mod tests {
 
     // make the identifier by looking up the string in the interner
     fn mk_ident(id: &str) -> TokenKind {
-        TokenKind::from_ast_ident(Ident::from_str(id))
+        token::Ident(Symbol::intern(id), false)
     }
 
     fn mk_lit(kind: token::LitKind, symbol: &str, suffix: Option<&str>) -> TokenKind {
diff --git a/src/libsyntax/parse/lexer/tokentrees.rs b/src/libsyntax/parse/lexer/tokentrees.rs
index b809f99beba..99d9d40a45b 100644
--- a/src/libsyntax/parse/lexer/tokentrees.rs
+++ b/src/libsyntax/parse/lexer/tokentrees.rs
@@ -211,7 +211,7 @@ impl<'a> TokenTreesReader<'a> {
                 let raw = self.string_reader.peek_span_src_raw;
                 self.real_token();
                 let is_joint = raw.hi() == self.string_reader.peek_span_src_raw.lo()
-                    && token::is_op(&self.token);
+                    && self.token.is_op();
                 Ok((tt, if is_joint { Joint } else { NonJoint }))
             }
         }
diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs
index 1d708d39a13..cde35681988 100644
--- a/src/libsyntax/parse/mod.rs
+++ b/src/libsyntax/parse/mod.rs
@@ -9,7 +9,7 @@ use crate::parse::parser::emit_unclosed_delims;
 use crate::parse::token::TokenKind;
 use crate::tokenstream::{TokenStream, TokenTree};
 use crate::diagnostics::plugin::ErrorMap;
-use crate::print::pprust::token_to_string;
+use crate::print::pprust;
 
 use errors::{Applicability, FatalError, Level, Handler, ColorConfig, Diagnostic, DiagnosticBuilder};
 use rustc_data_structures::sync::{Lrc, Lock};
@@ -312,7 +312,7 @@ pub fn maybe_file_to_stream(
             for unmatched in unmatched_braces {
                 let mut db = sess.span_diagnostic.struct_span_err(unmatched.found_span, &format!(
                     "incorrect close delimiter: `{}`",
-                    token_to_string(&token::CloseDelim(unmatched.found_delim)),
+                    pprust::token_kind_to_string(&token::CloseDelim(unmatched.found_delim)),
                 ));
                 db.span_label(unmatched.found_span, "incorrect close delimiter");
                 if let Some(sp) = unmatched.candidate_span {
diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs
index 3acd7088145..d9eba3bbadb 100644
--- a/src/libsyntax/parse/parser.rs
+++ b/src/libsyntax/parse/parser.rs
@@ -401,7 +401,7 @@ crate enum TokenType {
 impl TokenType {
     crate fn to_string(&self) -> String {
         match *self {
-            TokenType::Token(ref t) => format!("`{}`", pprust::token_to_string(t)),
+            TokenType::Token(ref t) => format!("`{}`", pprust::token_kind_to_string(t)),
             TokenType::Keyword(kw) => format!("`{}`", kw),
             TokenType::Operator => "an operator".to_string(),
             TokenType::Lifetime => "lifetime".to_string(),
@@ -418,7 +418,7 @@ impl TokenType {
 ///
 /// Types can also be of the form `IDENT(u8, u8) -> u8`, however this assumes
 /// that `IDENT` is not the ident of a fn trait.
-fn can_continue_type_after_non_fn_ident(t: &TokenKind) -> bool {
+fn can_continue_type_after_non_fn_ident(t: &Token) -> bool {
     t == &token::ModSep || t == &token::Lt ||
     t == &token::BinOp(token::Shl)
 }
@@ -586,10 +586,10 @@ impl<'a> Parser<'a> {
         edible: &[TokenKind],
         inedible: &[TokenKind],
     ) -> PResult<'a, bool /* recovered */> {
-        if edible.contains(&self.token) {
+        if edible.contains(&self.token.kind) {
             self.bump();
             Ok(false)
-        } else if inedible.contains(&self.token) {
+        } else if inedible.contains(&self.token.kind) {
             // leave it in the input
             Ok(false)
         } else if self.last_unexpected_token_span == Some(self.token.span) {
@@ -951,7 +951,7 @@ impl<'a> Parser<'a> {
                         Err(mut e) => {
                             // Attempt to keep parsing if it was a similar separator
                             if let Some(ref tokens) = t.similar_tokens() {
-                                if tokens.contains(&self.token) {
+                                if tokens.contains(&self.token.kind) {
                                     self.bump();
                                 }
                             }
@@ -1756,7 +1756,7 @@ impl<'a> Parser<'a> {
     fn parse_path_segment(&mut self, style: PathStyle) -> PResult<'a, PathSegment> {
         let ident = self.parse_path_segment_ident()?;
 
-        let is_args_start = |token: &TokenKind| match *token {
+        let is_args_start = |token: &Token| match token.kind {
             token::Lt | token::BinOp(token::Shl) | token::OpenDelim(token::Paren)
             | token::LArrow => true,
             _ => false,
@@ -2627,9 +2627,11 @@ impl<'a> Parser<'a> {
                     token::Ident(name, _) => name,
                     _ => unreachable!()
                 };
-                let mut err = self.fatal(&format!("unknown macro variable `{}`", name));
-                err.span_label(self.token.span, "unknown macro variable");
-                err.emit();
+                let span = self.prev_span.to(self.token.span);
+                self.diagnostic()
+                    .struct_span_fatal(span, &format!("unknown macro variable `{}`", name))
+                    .span_label(span, "unknown macro variable")
+                    .emit();
                 self.bump();
                 return
             }
@@ -2820,7 +2822,7 @@ impl<'a> Parser<'a> {
                 LhsExpr::AttributesParsed(attrs) => Some(attrs),
                 _ => None,
             };
-            if [token::DotDot, token::DotDotDot, token::DotDotEq].contains(&self.token) {
+            if [token::DotDot, token::DotDotDot, token::DotDotEq].contains(&self.token.kind) {
                 return self.parse_prefix_range_expr(attrs);
             } else {
                 self.parse_prefix_expr(attrs)?
@@ -3097,7 +3099,7 @@ impl<'a> Parser<'a> {
             self.err_dotdotdot_syntax(self.token.span);
         }
 
-        debug_assert!([token::DotDot, token::DotDotDot, token::DotDotEq].contains(&self.token),
+        debug_assert!([token::DotDot, token::DotDotDot, token::DotDotEq].contains(&self.token.kind),
                       "parse_prefix_range_expr: token {:?} is not DotDot/DotDotEq",
                       self.token);
         let tok = self.token.clone();
@@ -7865,7 +7867,7 @@ pub fn emit_unclosed_delims(unclosed_delims: &mut Vec<UnmatchedBrace>, handler:
     for unmatched in unclosed_delims.iter() {
         let mut err = handler.struct_span_err(unmatched.found_span, &format!(
             "incorrect close delimiter: `{}`",
-            pprust::token_to_string(&token::CloseDelim(unmatched.found_delim)),
+            pprust::token_kind_to_string(&token::CloseDelim(unmatched.found_delim)),
         ));
         err.span_label(unmatched.found_span, "incorrect close delimiter");
         if let Some(sp) = unmatched.candidate_span {
diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs
index 28a733728bf..cc34883e2e8 100644
--- a/src/libsyntax/parse/token.rs
+++ b/src/libsyntax/parse/token.rs
@@ -17,7 +17,6 @@ use log::info;
 
 use std::fmt;
 use std::mem;
-use std::ops::Deref;
 #[cfg(target_arch = "x86_64")]
 use rustc_data_structures::static_assert_size;
 use rustc_data_structures::sync::Lrc;
@@ -242,20 +241,57 @@ pub struct Token {
 }
 
 impl TokenKind {
-    /// Recovers a `TokenKind` from an `ast::Ident`. This creates a raw identifier if necessary.
-    pub fn from_ast_ident(ident: ast::Ident) -> TokenKind {
-        Ident(ident.name, ident.is_raw_guess())
+    pub fn lit(kind: LitKind, symbol: Symbol, suffix: Option<Symbol>) -> TokenKind {
+        Literal(Lit::new(kind, symbol, suffix))
     }
 
-    crate fn is_like_plus(&self) -> bool {
+    /// Returns tokens that are likely to be typed accidentally instead of the current token.
+    /// Enables better error recovery when the wrong token is found.
+    crate fn similar_tokens(&self) -> Option<Vec<TokenKind>> {
         match *self {
-            BinOp(Plus) | BinOpEq(Plus) => true,
-            _ => false,
+            Comma => Some(vec![Dot, Lt, Semi]),
+            Semi => Some(vec![Colon, Comma]),
+            _ => None
         }
     }
 }
 
 impl Token {
+    crate fn new(kind: TokenKind, span: Span) -> Self {
+        Token { kind, span }
+    }
+
+    /// Some token that will be thrown away later.
+    crate fn dummy() -> Self {
+        Token::new(TokenKind::Whitespace, DUMMY_SP)
+    }
+
+    /// Recovers a `Token` from an `ast::Ident`. This creates a raw identifier if necessary.
+    crate fn from_ast_ident(ident: ast::Ident) -> Self {
+        Token::new(Ident(ident.name, ident.is_raw_guess()), ident.span)
+    }
+
+    /// Return this token by value and leave a dummy token in its place.
+    crate fn take(&mut self) -> Self {
+        mem::replace(self, Token::dummy())
+    }
+
+    crate fn is_op(&self) -> bool {
+        match self.kind {
+            OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) |
+            Ident(..) | Lifetime(..) | Interpolated(..) |
+            Whitespace | Comment | Shebang(..) | Eof => false,
+            _ => true,
+        }
+    }
+
+    crate fn is_like_plus(&self) -> bool {
+        match self.kind {
+            BinOp(Plus) | BinOpEq(Plus) => true,
+            _ => false,
+        }
+    }
+
     /// Returns `true` if the token can appear at the start of an expression.
     crate fn can_begin_expr(&self) -> bool {
         match self.kind {
@@ -310,12 +346,10 @@ impl Token {
             _ => false,
         }
     }
-}
 
-impl TokenKind {
     /// Returns `true` if the token can appear at the start of a const param.
-    pub fn can_begin_const_arg(&self) -> bool {
-        match self {
+    crate fn can_begin_const_arg(&self) -> bool {
+        match self.kind {
             OpenDelim(Brace) => true,
             Interpolated(ref nt) => match **nt {
                 NtExpr(..) => true,
@@ -326,31 +360,23 @@ impl TokenKind {
             _ => self.can_begin_literal_or_bool(),
         }
     }
-}
 
-impl Token {
     /// Returns `true` if the token can appear at the start of a generic bound.
     crate fn can_begin_bound(&self) -> bool {
         self.is_path_start() || self.is_lifetime() || self.is_keyword(kw::For) ||
         self == &Question || self == &OpenDelim(Paren)
     }
-}
-
-impl TokenKind {
-    pub fn lit(kind: LitKind, symbol: Symbol, suffix: Option<Symbol>) -> TokenKind {
-        Literal(Lit::new(kind, symbol, suffix))
-    }
 
     /// Returns `true` if the token is any literal
     crate fn is_lit(&self) -> bool {
-        match *self {
+        match self.kind {
             Literal(..) => true,
             _           => false,
         }
     }
 
     crate fn expect_lit(&self) -> Lit {
-        match *self {
+        match self.kind {
             Literal(lit) => lit,
             _=> panic!("`expect_lit` called on non-literal"),
         }
@@ -359,7 +385,7 @@ impl TokenKind {
     /// Returns `true` if the token is any literal, a minus (which can prefix a literal,
     /// for example a '-42', or one of the boolean idents).
     crate fn can_begin_literal_or_bool(&self) -> bool {
-        match *self {
+        match self.kind {
             Literal(..)  => true,
             BinOp(Minus) => true,
             Ident(name, false) if name == kw::True => true,
@@ -371,9 +397,7 @@ impl TokenKind {
             _            => false,
         }
     }
-}
 
-impl Token {
     /// Returns an identifier if this token is an identifier.
     pub fn ident(&self) -> Option<(ast::Ident, /* is_raw */ bool)> {
         match self.kind {
@@ -397,49 +421,25 @@ impl Token {
             _ => None,
         }
     }
-}
 
-impl TokenKind {
-    /// Returns an identifier name if this token is an identifier.
-    pub fn ident_name(&self) -> Option<(ast::Name, /* is_raw */ bool)> {
-        match *self {
-            Ident(name, is_raw) => Some((name, is_raw)),
-            Interpolated(ref nt) => match **nt {
-                NtIdent(ident, is_raw) => Some((ident.name, is_raw)),
-                _ => None,
-            },
-            _ => None,
-        }
-    }
-    /// Returns a lifetime name if this token is a lifetime.
-    pub fn lifetime_name(&self) -> Option<ast::Name> {
-        match *self {
-            Lifetime(name) => Some(name),
-            Interpolated(ref nt) => match **nt {
-                NtLifetime(ident) => Some(ident.name),
-                _ => None,
-            },
-            _ => None,
-        }
-    }
     /// Returns `true` if the token is an identifier.
     pub fn is_ident(&self) -> bool {
-        self.ident_name().is_some()
+        self.ident().is_some()
     }
     /// Returns `true` if the token is a lifetime.
     crate fn is_lifetime(&self) -> bool {
-        self.lifetime_name().is_some()
+        self.lifetime().is_some()
     }
 
     /// Returns `true` if the token is a identifier whose name is the given
     /// string slice.
     crate fn is_ident_named(&self, name: Symbol) -> bool {
-        self.ident_name().map_or(false, |(ident_name, _)| ident_name == name)
+        self.ident().map_or(false, |(ident, _)| ident.name == name)
     }
 
     /// Returns `true` if the token is an interpolated path.
     fn is_path(&self) -> bool {
-        if let Interpolated(ref nt) = *self {
+        if let Interpolated(ref nt) = self.kind {
             if let NtPath(..) = **nt {
                 return true;
             }
@@ -456,33 +456,27 @@ impl TokenKind {
     crate fn is_qpath_start(&self) -> bool {
         self == &Lt || self == &BinOp(Shl)
     }
-}
 
-impl Token {
     crate fn is_path_start(&self) -> bool {
         self == &ModSep || self.is_qpath_start() || self.is_path() ||
         self.is_path_segment_keyword() || self.is_ident() && !self.is_reserved_ident()
     }
-}
 
-impl TokenKind {
     /// Returns `true` if the token is a given keyword, `kw`.
     pub fn is_keyword(&self, kw: Symbol) -> bool {
-        self.ident_name().map(|(name, is_raw)| name == kw && !is_raw).unwrap_or(false)
+        self.ident().map(|(id, is_raw)| id.name == kw && !is_raw).unwrap_or(false)
     }
 
-    pub fn is_path_segment_keyword(&self) -> bool {
-        match self.ident_name() {
-            Some((name, false)) => name.is_path_segment_keyword(),
+    crate fn is_path_segment_keyword(&self) -> bool {
+        match self.ident() {
+            Some((id, false)) => id.is_path_segment_keyword(),
             _ => false,
         }
     }
-}
 
-impl Token {
     // Returns true for reserved identifiers used internally for elided lifetimes,
     // unnamed method parameters, crate root module, error recovery etc.
-    pub fn is_special_ident(&self) -> bool {
+    crate fn is_special_ident(&self) -> bool {
         match self.ident() {
             Some((id, false)) => id.is_special(),
             _ => false,
@@ -512,55 +506,53 @@ impl Token {
             _ => false,
         }
     }
-}
 
-impl TokenKind {
-    crate fn glue(self, joint: TokenKind) -> Option<TokenKind> {
-        Some(match self {
-            Eq => match joint {
+    crate fn glue(self, joint: Token) -> Option<Token> {
+        let kind = match self.kind {
+            Eq => match joint.kind {
                 Eq => EqEq,
                 Gt => FatArrow,
                 _ => return None,
             },
-            Lt => match joint {
+            Lt => match joint.kind {
                 Eq => Le,
                 Lt => BinOp(Shl),
                 Le => BinOpEq(Shl),
                 BinOp(Minus) => LArrow,
                 _ => return None,
             },
-            Gt => match joint {
+            Gt => match joint.kind {
                 Eq => Ge,
                 Gt => BinOp(Shr),
                 Ge => BinOpEq(Shr),
                 _ => return None,
             },
-            Not => match joint {
+            Not => match joint.kind {
                 Eq => Ne,
                 _ => return None,
             },
-            BinOp(op) => match joint {
+            BinOp(op) => match joint.kind {
                 Eq => BinOpEq(op),
                 BinOp(And) if op == And => AndAnd,
                 BinOp(Or) if op == Or => OrOr,
                 Gt if op == Minus => RArrow,
                 _ => return None,
             },
-            Dot => match joint {
+            Dot => match joint.kind {
                 Dot => DotDot,
                 DotDot => DotDotDot,
                 _ => return None,
             },
-            DotDot => match joint {
+            DotDot => match joint.kind {
                 Dot => DotDotDot,
                 Eq => DotDotEq,
                 _ => return None,
             },
-            Colon => match joint {
+            Colon => match joint.kind {
                 Colon => ModSep,
                 _ => return None,
             },
-            SingleQuote => match joint {
+            SingleQuote => match joint.kind {
                 Ident(name, false) => Lifetime(Symbol::intern(&format!("'{}", name))),
                 _ => return None,
             },
@@ -570,26 +562,18 @@ impl TokenKind {
             Question | OpenDelim(..) | CloseDelim(..) |
             Literal(..) | Ident(..) | Lifetime(..) | Interpolated(..) | DocComment(..) |
             Whitespace | Comment | Shebang(..) | Eof => return None,
-        })
-    }
+        };
 
-    /// Returns tokens that are likely to be typed accidentally instead of the current token.
-    /// Enables better error recovery when the wrong token is found.
-    crate fn similar_tokens(&self) -> Option<Vec<TokenKind>> {
-        match *self {
-            Comma => Some(vec![Dot, Lt, Semi]),
-            Semi => Some(vec![Colon, Comma]),
-            _ => None
-        }
+        Some(Token::new(kind, self.span.to(joint.span)))
     }
 
     // See comments in `Nonterminal::to_tokenstream` for why we care about
     // *probably* equal here rather than actual equality
-    crate fn probably_equal_for_proc_macro(&self, other: &TokenKind) -> bool {
-        if mem::discriminant(self) != mem::discriminant(other) {
+    crate fn probably_equal_for_proc_macro(&self, other: &Token) -> bool {
+        if mem::discriminant(&self.kind) != mem::discriminant(&other.kind) {
             return false
         }
-        match (self, other) {
+        match (&self.kind, &other.kind) {
             (&Eq, &Eq) |
             (&Lt, &Lt) |
             (&Le, &Le) |
@@ -643,36 +627,12 @@ impl TokenKind {
     }
 }
 
-impl Token {
-    crate fn new(kind: TokenKind, span: Span) -> Self {
-        Token { kind, span }
-    }
-
-    /// Some token that will be thrown away later.
-    crate fn dummy() -> Self {
-        Token::new(TokenKind::Whitespace, DUMMY_SP)
-    }
-
-    /// Return this token by value and leave a dummy token in its place.
-    crate fn take(&mut self) -> Self {
-        mem::replace(self, Token::dummy())
-    }
-}
-
 impl PartialEq<TokenKind> for Token {
     fn eq(&self, rhs: &TokenKind) -> bool {
         self.kind == *rhs
     }
 }
 
-// FIXME: Remove this after all necessary methods are moved from `TokenKind` to `Token`.
-impl Deref for Token {
-    type Target = TokenKind;
-    fn deref(&self) -> &Self::Target {
-        &self.kind
-    }
-}
-
 #[derive(Clone, RustcEncodable, RustcDecodable)]
 /// For interpolation during macro expansion.
 pub enum Nonterminal {
@@ -812,15 +772,6 @@ impl Nonterminal {
     }
 }
 
-crate fn is_op(tok: &TokenKind) -> bool {
-    match *tok {
-        OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) |
-        Ident(..) | Lifetime(..) | Interpolated(..) |
-        Whitespace | Comment | Shebang(..) | Eof => false,
-        _ => true,
-    }
-}
-
 fn prepend_attrs(sess: &ParseSess,
                  attrs: &[ast::Attribute],
                  tokens: Option<&tokenstream::TokenStream>,
diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs
index d922e1896cc..4cbe590d44b 100644
--- a/src/libsyntax/print/pprust.rs
+++ b/src/libsyntax/print/pprust.rs
@@ -6,7 +6,7 @@ use crate::ast::{Attribute, MacDelimiter, GenericArg};
 use crate::util::parser::{self, AssocOp, Fixity};
 use crate::attr;
 use crate::source_map::{self, SourceMap, Spanned};
-use crate::parse::token::{self, BinOpToken, Nonterminal, TokenKind};
+use crate::parse::token::{self, BinOpToken, Nonterminal, Token, TokenKind};
 use crate::parse::lexer::comments;
 use crate::parse::{self, ParseSess};
 use crate::print::pp::{self, Breaks};
@@ -189,7 +189,7 @@ pub fn literal_to_string(lit: token::Lit) -> String {
     out
 }
 
-pub fn token_to_string(tok: &TokenKind) -> String {
+pub fn token_kind_to_string(tok: &TokenKind) -> String {
     match *tok {
         token::Eq                   => "=".to_string(),
         token::Lt                   => "<".to_string(),
@@ -250,6 +250,10 @@ pub fn token_to_string(tok: &TokenKind) -> String {
     }
 }
 
+pub fn token_to_string(token: &Token) -> String {
+    token_kind_to_string(&token.kind)
+}
+
 pub fn nonterminal_to_string(nt: &Nonterminal) -> String {
     match *nt {
         token::NtExpr(ref e)        => expr_to_string(e),
@@ -734,11 +738,11 @@ pub trait PrintState<'a> {
                 }
             }
             TokenTree::Delimited(_, delim, tts) => {
-                self.writer().word(token_to_string(&token::OpenDelim(delim)))?;
+                self.writer().word(token_kind_to_string(&token::OpenDelim(delim)))?;
                 self.writer().space()?;
                 self.print_tts(tts)?;
                 self.writer().space()?;
-                self.writer().word(token_to_string(&token::CloseDelim(delim)))
+                self.writer().word(token_kind_to_string(&token::CloseDelim(delim)))
             },
         }
     }
diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs
index 9dea3a4dcc1..2daec970279 100644
--- a/src/libsyntax/tokenstream.rs
+++ b/src/libsyntax/tokenstream.rs
@@ -126,14 +126,6 @@ impl TokenTree {
         }
     }
 
-    /// Indicates if the stream is a token that is equal to the provided token.
-    pub fn eq_token(&self, t: TokenKind) -> bool {
-        match self {
-            TokenTree::Token(token) => *token == t,
-            _ => false,
-        }
-    }
-
     pub fn joint(self) -> TokenStream {
         TokenStream::new(vec![(self, Joint)])
     }
@@ -430,11 +422,10 @@ impl TokenStreamBuilder {
         let last_tree_if_joint = self.0.last().and_then(TokenStream::last_tree_if_joint);
         if let Some(TokenTree::Token(last_token)) = last_tree_if_joint {
             if let Some((TokenTree::Token(token), is_joint)) = stream.first_tree_and_joint() {
-                if let Some(glued_tok) = last_token.kind.glue(token.kind) {
+                if let Some(glued_tok) = last_token.glue(token) {
                     let last_stream = self.0.pop().unwrap();
                     self.push_all_but_last_tree(&last_stream);
-                    let glued_span = last_token.span.to(token.span);
-                    let glued_tt = TokenTree::token(glued_tok, glued_span);
+                    let glued_tt = TokenTree::Token(glued_tok);
                     let glued_tokenstream = TokenStream::new(vec![(glued_tt, is_joint)]);
                     self.0.push(glued_tokenstream);
                     self.push_all_but_first_tree(&stream);
diff --git a/src/libsyntax/util/parser.rs b/src/libsyntax/util/parser.rs
index 9e26f1bf7d3..69dd96625cc 100644
--- a/src/libsyntax/util/parser.rs
+++ b/src/libsyntax/util/parser.rs
@@ -1,4 +1,4 @@
-use crate::parse::token::{self, TokenKind, BinOpToken};
+use crate::parse::token::{self, Token, BinOpToken};
 use crate::symbol::kw;
 use crate::ast::{self, BinOpKind};
 
@@ -69,9 +69,9 @@ pub enum Fixity {
 
 impl AssocOp {
     /// Creates a new AssocOP from a token
-    pub fn from_token(t: &TokenKind) -> Option<AssocOp> {
+    pub fn from_token(t: &Token) -> Option<AssocOp> {
         use AssocOp::*;
-        match *t {
+        match t.kind {
             token::BinOpEq(k) => Some(AssignOp(k)),
             token::Eq => Some(Assign),
             token::BinOp(BinOpToken::Star) => Some(Multiply),
generated by cgit-pink 1.4.1-3-g733a5 (git 2.37.1) at 2026-08-22 21:11:35 +0000