Auto merge of #49154 - petrochenkov:spident, r=eddyb

AST: Give spans to all identifiers

Change representation of `ast::Ident` from `{ name: Symbol, ctxt: SyntaxContext }` to `{ name: Symbol, span: Span }`.
Syntax contexts still can be extracted from spans (`span.ctxt()`).

Why this should not require more memory:
- `Span` is `u32` just like `SyntaxContext`.
- Despite keeping more spans in AST we don't actually *create* more spans, so the number of "outlined" spans kept in span interner shouldn't become larger.

Why this may be slightly slower:
- When we need to extract ctxt from an identifier instead of just field read we need to do bit field extraction possibly followed by and access by index into span interner's vector. Both operations should be fast (unless the span interner is under some synchronization) and we already do ctxt extraction from spans all the time during macro expansion, so the difference should be lost in noise.

cc https://github.com/rust-lang/rust/pull/48842#issuecomment-373365661

5 files changed, 91 insertions, 116 deletions

diff --git a/src/libsyntax/parse/attr.rs b/src/libsyntax/parse/attr.rs
index 4c3f42d9c6b..90f08ab1468 100644
--- a/src/libsyntax/parse/attr.rs
+++ b/src/libsyntax/parse/attr.rs
@@ -149,8 +149,7 @@ impl<'a> Parser<'a> {
         };
         Ok(if let Some(meta) = meta {
             self.bump();
-            (ast::Path::from_ident(meta.span, ast::Ident::with_empty_ctxt(meta.name)),
-             meta.node.tokens(meta.span))
+            (ast::Path::from_ident(meta.ident), meta.node.tokens(meta.span))
         } else {
             (self.parse_path(PathStyle::Mod)?, self.parse_tokens())
         })
@@ -228,7 +227,7 @@ impl<'a> Parser<'a> {
         let lo = self.span;
         let ident = self.parse_ident()?;
         let node = self.parse_meta_item_kind()?;
-        Ok(ast::MetaItem { name: ident.name, node: node, span: lo.to(self.prev_span) })
+        Ok(ast::MetaItem { ident, node: node, span: lo.to(self.prev_span) })
     }
 
     pub fn parse_meta_item_kind(&mut self) -> PResult<'a, ast::MetaItemKind> {
diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs
index 068929c8948..152c4c31eb3 100644
--- a/src/libsyntax/parse/lexer/mod.rs
+++ b/src/libsyntax/parse/lexer/mod.rs
@@ -76,7 +76,7 @@ impl<'a> StringReader<'a> {
     fn mk_ident(&self, string: &str) -> Ident {
         let mut ident = Ident::from_str(string);
         if let Some(span) = self.override_span {
-            ident.ctxt = span.ctxt();
+            ident.span = span;
         }
         ident
     }
diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs
index 1483691a1ea..428b3b136df 100644
--- a/src/libsyntax/parse/mod.rs
+++ b/src/libsyntax/parse/mod.rs
@@ -689,7 +689,7 @@ mod tests {
     }
 
     fn str2seg(s: &str, lo: u32, hi: u32) -> ast::PathSegment {
-        ast::PathSegment::from_ident(Ident::from_str(s), sp(lo, hi))
+        ast::PathSegment::from_ident(Ident::new(Symbol::intern(s), sp(lo, hi)))
     }
 
     #[test] fn path_exprs_1() {
@@ -873,10 +873,8 @@ mod tests {
                     == P(ast::Pat{
                     id: ast::DUMMY_NODE_ID,
                     node: PatKind::Ident(ast::BindingMode::ByValue(ast::Mutability::Immutable),
-                                        Spanned{ span:sp(0, 1),
-                                                node: Ident::from_str("b")
-                        },
-                                        None),
+                                         Ident::new(Symbol::intern("b"), sp(0, 1)),
+                                         None),
                     span: sp(0,1)}));
             parser_done(parser);
         })
@@ -912,9 +910,7 @@ mod tests {
                                             node: PatKind::Ident(
                                                 ast::BindingMode::ByValue(
                                                     ast::Mutability::Immutable),
-                                                Spanned{
-                                                    span: sp(6,7),
-                                                    node: Ident::from_str("b")},
+                                                Ident::new(Symbol::intern("b"), sp(6, 7)),
                                                 None
                                             ),
                                             span: sp(6,7)
diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs
index 9936eea9356..e6da5bcaa3a 100644
--- a/src/libsyntax/parse/parser.rs
+++ b/src/libsyntax/parse/parser.rs
@@ -512,13 +512,10 @@ impl From<P<Expr>> for LhsExpr {
 
 /// Create a placeholder argument.
 fn dummy_arg(span: Span) -> Arg {
-    let spanned = Spanned {
-        span,
-        node: keywords::Invalid.ident()
-    };
+    let ident = Ident::new(keywords::Invalid.name(), span);
     let pat = P(Pat {
         id: ast::DUMMY_NODE_ID,
-        node: PatKind::Ident(BindingMode::ByValue(Mutability::Immutable), spanned, None),
+        node: PatKind::Ident(BindingMode::ByValue(Mutability::Immutable), ident, None),
         span,
     });
     let ty = Ty {
@@ -778,7 +775,7 @@ impl<'a> Parser<'a> {
 
     fn parse_ident_common(&mut self, recover: bool) -> PResult<'a, ast::Ident> {
         match self.token {
-            token::Ident(i, _) => {
+            token::Ident(ident, _) => {
                 if self.token.is_reserved_ident() {
                     let mut err = self.expected_ident_found();
                     if recover {
@@ -787,8 +784,9 @@ impl<'a> Parser<'a> {
                         return Err(err);
                     }
                 }
+                let span = self.span;
                 self.bump();
-                Ok(i)
+                Ok(Ident::new(ident.name, span))
             }
             _ => {
                 Err(if self.prev_token_kind == PrevTokenKind::DocComment {
@@ -1319,19 +1317,6 @@ impl<'a> Parser<'a> {
             self.check_keyword(keywords::Extern) && self.is_extern_non_path()
     }
 
-    fn eat_label(&mut self) -> Option<Label> {
-        let ident = match self.token {
-            token::Lifetime(ref ident) => *ident,
-            token::Interpolated(ref nt) => match nt.0 {
-                token::NtLifetime(lifetime) => lifetime.ident,
-                _ => return None,
-            },
-            _ => return None,
-        };
-        self.bump();
-        Some(Label { ident, span: self.prev_span })
-    }
-
     /// parse a TyKind::BareFn type:
     pub fn parse_ty_bare_fn(&mut self, generic_params: Vec<GenericParam>)
                             -> PResult<'a, TyKind> {
@@ -1784,13 +1769,11 @@ impl<'a> Parser<'a> {
             pat
         } else {
             debug!("parse_arg_general ident_to_pat");
-            let sp = self.prev_span;
-            let spanned = Spanned { span: sp, node: keywords::Invalid.ident() };
+            let ident = Ident::new(keywords::Invalid.name(), self.prev_span);
             P(Pat {
                 id: ast::DUMMY_NODE_ID,
-                node: PatKind::Ident(BindingMode::ByValue(Mutability::Immutable),
-                                     spanned, None),
-                span: sp
+                node: PatKind::Ident(BindingMode::ByValue(Mutability::Immutable), ident, None),
+                span: ident.span,
             })
         };
 
@@ -1899,13 +1882,14 @@ impl<'a> Parser<'a> {
 
     pub fn parse_path_segment_ident(&mut self) -> PResult<'a, ast::Ident> {
         match self.token {
-            token::Ident(sid, _) if self.token.is_path_segment_keyword() => {
+            token::Ident(ident, _) if self.token.is_path_segment_keyword() => {
+                let span = self.span;
                 self.bump();
-                Ok(sid)
+                Ok(Ident::new(ident.name, span))
             }
             _ => self.parse_ident(),
-         }
-     }
+        }
+    }
 
     /// Parses qualified path.
     /// Assumes that the leading `<` has been parsed already.
@@ -1973,7 +1957,7 @@ impl<'a> Parser<'a> {
         let meta_ident = match self.token {
             token::Interpolated(ref nt) => match nt.0 {
                 token::NtMeta(ref meta) => match meta.node {
-                    ast::MetaItemKind::Word => Some(ast::Ident::with_empty_ctxt(meta.name)),
+                    ast::MetaItemKind::Word => Some(meta.ident),
                     _ => None,
                 },
                 _ => None,
@@ -1982,7 +1966,7 @@ impl<'a> Parser<'a> {
         };
         if let Some(ident) = meta_ident {
             self.bump();
-            return Ok(ast::Path::from_ident(self.prev_span, ident));
+            return Ok(ast::Path::from_ident(ident));
         }
         self.parse_path(style)
     }
@@ -2003,7 +1987,6 @@ impl<'a> Parser<'a> {
 
     fn parse_path_segment(&mut self, style: PathStyle, enable_warning: bool)
                           -> PResult<'a, PathSegment> {
-        let ident_span = self.span;
         let ident = self.parse_path_segment_ident()?;
 
         let is_args_start = |token: &token::Token| match *token {
@@ -2051,10 +2034,10 @@ impl<'a> Parser<'a> {
                 ParenthesizedParameterData { inputs, output, span }.into()
             };
 
-            PathSegment { identifier: ident, span: ident_span, parameters }
+            PathSegment { ident, parameters }
         } else {
             // Generic arguments are not found.
-            PathSegment::from_ident(ident, ident_span)
+            PathSegment::from_ident(ident)
         })
     }
 
@@ -2065,14 +2048,25 @@ impl<'a> Parser<'a> {
 
     /// Parse single lifetime 'a or panic.
     pub fn expect_lifetime(&mut self) -> Lifetime {
-        if let Some(lifetime) = self.token.lifetime(self.span) {
+        if let Some(ident) = self.token.lifetime() {
+            let span = self.span;
             self.bump();
-            lifetime
+            Lifetime { ident: Ident::new(ident.name, span), id: ast::DUMMY_NODE_ID }
         } else {
             self.span_bug(self.span, "not a lifetime")
         }
     }
 
+    fn eat_label(&mut self) -> Option<Label> {
+        if let Some(ident) = self.token.lifetime() {
+            let span = self.span;
+            self.bump();
+            Some(Label { ident: Ident::new(ident.name, span) })
+        } else {
+            None
+        }
+    }
+
     /// Parse mutability (`mut` or nothing).
     fn parse_mutability(&mut self) -> Mutability {
         if self.eat_keyword(keywords::Mut) {
@@ -2085,7 +2079,7 @@ impl<'a> Parser<'a> {
     pub fn parse_field_name(&mut self) -> PResult<'a, Ident> {
         if let token::Literal(token::Integer(name), None) = self.token {
             self.bump();
-            Ok(Ident::with_empty_ctxt(name))
+            Ok(Ident::new(name, self.prev_span))
         } else {
             self.parse_ident_common(false)
         }
@@ -2095,24 +2089,22 @@ impl<'a> Parser<'a> {
     pub fn parse_field(&mut self) -> PResult<'a, Field> {
         let attrs = self.parse_outer_attributes()?;
         let lo = self.span;
-        let hi;
 
         // Check if a colon exists one ahead. This means we're parsing a fieldname.
         let (fieldname, expr, is_shorthand) = if self.look_ahead(1, |t| t == &token::Colon) {
             let fieldname = self.parse_field_name()?;
-            hi = self.prev_span;
-            self.bump();
+            self.bump(); // `:`
             (fieldname, self.parse_expr()?, false)
         } else {
             let fieldname = self.parse_ident_common(false)?;
-            hi = self.prev_span;
 
             // Mimic `x: x` for the `x` field shorthand.
-            let path = ast::Path::from_ident(lo.to(hi), fieldname);
-            (fieldname, self.mk_expr(lo.to(hi), ExprKind::Path(None, path), ThinVec::new()), true)
+            let path = ast::Path::from_ident(fieldname);
+            let expr = self.mk_expr(fieldname.span, ExprKind::Path(None, path), ThinVec::new());
+            (fieldname, expr, true)
         };
         Ok(ast::Field {
-            ident: respan(lo.to(hi), fieldname),
+            ident: fieldname,
             span: lo.to(expr.span),
             expr,
             is_shorthand,
@@ -2318,7 +2310,7 @@ impl<'a> Parser<'a> {
                     return self.parse_while_expr(None, lo, attrs);
                 }
                 if let Some(label) = self.eat_label() {
-                    let lo = label.span;
+                    let lo = label.ident.span;
                     self.expect(&token::Colon)?;
                     if self.eat_keyword(keywords::While) {
                         return self.parse_while_expr(Some(label), lo, attrs)
@@ -2592,8 +2584,7 @@ impl<'a> Parser<'a> {
                 }
 
                 let span = lo.to(self.prev_span);
-                let ident = respan(segment.span, segment.identifier);
-                self.mk_expr(span, ExprKind::Field(self_arg, ident), ThinVec::new())
+                self.mk_expr(span, ExprKind::Field(self_arg, segment.ident), ThinVec::new())
             }
         })
     }
@@ -2714,7 +2705,7 @@ impl<'a> Parser<'a> {
     }
 
     pub fn process_potential_macro_variable(&mut self) {
-        let (ident, is_raw) = match self.token {
+        let (token, span) = match self.token {
             token::Dollar if self.span.ctxt() != syntax_pos::hygiene::SyntaxContext::empty() &&
                              self.look_ahead(1, |t| t.is_ident()) => {
                 self.bump();
@@ -2729,15 +2720,18 @@ impl<'a> Parser<'a> {
             }
             token::Interpolated(ref nt) => {
                 self.meta_var_span = Some(self.span);
+                // Interpolated identifier and lifetime tokens are replaced with usual identifier
+                // and lifetime tokens, so the former are never encountered during normal parsing.
                 match nt.0 {
-                    token::NtIdent(ident, is_raw) => (ident, is_raw),
+                    token::NtIdent(ident, is_raw) => (token::Ident(ident, is_raw), ident.span),
+                    token::NtLifetime(ident) => (token::Lifetime(ident), ident.span),
                     _ => return,
                 }
             }
             _ => return,
         };
-        self.token = token::Ident(ident.node, is_raw);
-        self.span = ident.span;
+        self.token = token;
+        self.span = span;
     }
 
     /// parse a single token tree from the input.
@@ -3672,10 +3666,9 @@ impl<'a> Parser<'a> {
                     (false, true) => BindingMode::ByValue(Mutability::Mutable),
                     (false, false) => BindingMode::ByValue(Mutability::Immutable),
                 };
-                let fieldpath = codemap::Spanned{span:self.prev_span, node:fieldname};
                 let fieldpat = P(Pat {
                     id: ast::DUMMY_NODE_ID,
-                    node: PatKind::Ident(bind_type, fieldpath, None),
+                    node: PatKind::Ident(bind_type, fieldname, None),
                     span: boxed_span.to(hi),
                 });
 
@@ -3966,9 +3959,7 @@ impl<'a> Parser<'a> {
     fn parse_pat_ident(&mut self,
                        binding_mode: ast::BindingMode)
                        -> PResult<'a, PatKind> {
-        let ident_span = self.span;
         let ident = self.parse_ident()?;
-        let name = codemap::Spanned{span: ident_span, node: ident};
         let sub = if self.eat(&token::At) {
             Some(self.parse_pat()?)
         } else {
@@ -3987,7 +3978,7 @@ impl<'a> Parser<'a> {
                 "expected identifier, found enum pattern"))
         }
 
-        Ok(PatKind::Ident(binding_mode, name, sub))
+        Ok(PatKind::Ident(binding_mode, ident, sub))
     }
 
     /// Parse a local variable declaration
@@ -4699,7 +4690,6 @@ impl<'a> Parser<'a> {
 
     /// Matches typaram = IDENT (`?` unbound)? optbounds ( EQ ty )?
     fn parse_ty_param(&mut self, preceding_attrs: Vec<Attribute>) -> PResult<'a, TyParam> {
-        let span = self.span;
         let ident = self.parse_ident()?;
 
         // Parse optional colon and param bounds.
@@ -4721,7 +4711,6 @@ impl<'a> Parser<'a> {
             id: ast::DUMMY_NODE_ID,
             bounds,
             default,
-            span,
         })
     }
 
@@ -4729,7 +4718,6 @@ impl<'a> Parser<'a> {
     ///     TraitItemAssocTy = Ident ["<"...">"] [":" [TyParamBounds]] ["where" ...] ["=" Ty]
     fn parse_trait_item_assoc_ty(&mut self, preceding_attrs: Vec<Attribute>)
         -> PResult<'a, (ast::Generics, TyParam)> {
-        let span = self.span;
         let ident = self.parse_ident()?;
         let mut generics = self.parse_generics()?;
 
@@ -4754,7 +4742,6 @@ impl<'a> Parser<'a> {
             id: ast::DUMMY_NODE_ID,
             bounds,
             default,
-            span,
         }))
     }
 
@@ -5051,9 +5038,8 @@ impl<'a> Parser<'a> {
     fn parse_self_arg(&mut self) -> PResult<'a, Option<Arg>> {
         let expect_ident = |this: &mut Self| match this.token {
             // Preserve hygienic context.
-            token::Ident(ident, _) => {
-                let sp = this.span; this.bump(); codemap::respan(sp, ident)
-            }
+            token::Ident(ident, _) =>
+                { let span = this.span; this.bump(); Ident::new(ident.name, span) }
             _ => unreachable!()
         };
         let isolated_self = |this: &mut Self, n| {
@@ -5566,7 +5552,7 @@ impl<'a> Parser<'a> {
                     TyKind::Path(None, path) => path,
                     _ => {
                         self.span_err(ty_first.span, "expected a trait, found type");
-                        ast::Path::from_ident(ty_first.span, keywords::Invalid.ident())
+                        ast::Path::from_ident(Ident::new(keywords::Invalid.name(), ty_first.span))
                     }
                 };
                 let trait_ref = TraitRef { path, ref_id: ty_first.id };
@@ -5951,8 +5937,7 @@ impl<'a> Parser<'a> {
                     let attr = Attribute {
                         id: attr::mk_attr_id(),
                         style: ast::AttrStyle::Outer,
-                        path: ast::Path::from_ident(syntax_pos::DUMMY_SP,
-                                                    Ident::from_str("warn_directory_ownership")),
+                        path: ast::Path::from_ident(Ident::from_str("warn_directory_ownership")),
                         tokens: TokenStream::empty(),
                         is_sugared_doc: false,
                         span: syntax_pos::DUMMY_SP,
@@ -6334,7 +6319,7 @@ impl<'a> Parser<'a> {
             }
 
             let vr = ast::Variant_ {
-                name: ident,
+                ident,
                 attrs: variant_attrs,
                 data: struct_def,
                 disr_expr,
diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs
index 5fe2b081566..6544619af9c 100644
--- a/src/libsyntax/parse/token.rs
+++ b/src/libsyntax/parse/token.rs
@@ -317,21 +317,36 @@ impl Token {
         }
     }
 
-    pub fn ident(&self) -> Option<(ast::Ident, bool)> {
+    /// Returns an identifier if this token is an identifier.
+    pub fn ident(&self) -> Option<(ast::Ident, /* is_raw */ bool)> {
         match *self {
             Ident(ident, is_raw) => Some((ident, is_raw)),
             Interpolated(ref nt) => match nt.0 {
-                NtIdent(ident, is_raw) => Some((ident.node, is_raw)),
+                NtIdent(ident, is_raw) => Some((ident, is_raw)),
+                _ => None,
+            },
+            _ => None,
+        }
+    }
+    /// Returns a lifetime identifier if this token is a lifetime.
+    pub fn lifetime(&self) -> Option<ast::Ident> {
+        match *self {
+            Lifetime(ident) => Some(ident),
+            Interpolated(ref nt) => match nt.0 {
+                NtLifetime(ident) => Some(ident),
                 _ => None,
             },
             _ => None,
         }
     }
-
     /// Returns `true` if the token is an identifier.
     pub fn is_ident(&self) -> bool {
         self.ident().is_some()
     }
+    /// Returns `true` if the token is a lifetime.
+    pub fn is_lifetime(&self) -> bool {
+        self.lifetime().is_some()
+    }
 
     /// Returns `true` if the token is a documentation comment.
     pub fn is_doc_comment(&self) -> bool {
@@ -359,26 +374,6 @@ impl Token {
         false
     }
 
-    /// Returns a lifetime with the span and a dummy id if it is a lifetime,
-    /// or the original lifetime if it is an interpolated lifetime, ignoring
-    /// the span.
-    pub fn lifetime(&self, span: Span) -> Option<ast::Lifetime> {
-        match *self {
-            Lifetime(ident) =>
-                Some(ast::Lifetime { ident: ident, span: span, id: ast::DUMMY_NODE_ID }),
-            Interpolated(ref nt) => match nt.0 {
-                NtLifetime(lifetime) => Some(lifetime),
-                _ => None,
-            },
-            _ => None,
-        }
-    }
-
-    /// Returns `true` if the token is a lifetime.
-    pub fn is_lifetime(&self) -> bool {
-        self.lifetime(syntax_pos::DUMMY_SP).is_some()
-    }
-
     /// Returns `true` if the token is either the `mut` or `const` keyword.
     pub fn is_mutability(&self) -> bool {
         self.is_keyword(keywords::Mut) ||
@@ -431,6 +426,14 @@ impl Token {
         }
     }
 
+    /// Returns `true` if the token is either a special identifier or a keyword.
+    pub fn is_reserved_ident(&self) -> bool {
+        match self.ident() {
+            Some((id, false)) => is_reserved_ident(id),
+            _ => false,
+        }
+    }
+
     pub fn glue(self, joint: Token) -> Option<Token> {
         Some(match self {
             Eq => match joint {
@@ -497,14 +500,6 @@ impl Token {
         }
     }
 
-    /// Returns `true` if the token is either a special identifier or a keyword.
-    pub fn is_reserved_ident(&self) -> bool {
-        match self.ident() {
-            Some((id, false)) => is_reserved_ident(id),
-            _ => false,
-        }
-    }
-
     pub fn interpolated_to_tokenstream(&self, sess: &ParseSess, span: Span)
         -> TokenStream
     {
@@ -539,12 +534,12 @@ impl Token {
                 tokens = prepend_attrs(sess, &item.attrs, item.tokens.as_ref(), span);
             }
             Nonterminal::NtIdent(ident, is_raw) => {
-                let token = Token::Ident(ident.node, is_raw);
+                let token = Token::Ident(ident, is_raw);
                 tokens = Some(TokenTree::Token(ident.span, token).into());
             }
-            Nonterminal::NtLifetime(lifetime) => {
-                let token = Token::Lifetime(lifetime.ident);
-                tokens = Some(TokenTree::Token(lifetime.span, token).into());
+            Nonterminal::NtLifetime(ident) => {
+                let token = Token::Lifetime(ident);
+                tokens = Some(TokenTree::Token(ident.span, token).into());
             }
             Nonterminal::NtTT(ref tt) => {
                 tokens = Some(tt.clone().into());
@@ -571,7 +566,8 @@ pub enum Nonterminal {
     NtPat(P<ast::Pat>),
     NtExpr(P<ast::Expr>),
     NtTy(P<ast::Ty>),
-    NtIdent(ast::SpannedIdent, /* is_raw */ bool),
+    NtIdent(ast::Ident, /* is_raw */ bool),
+    NtLifetime(ast::Ident),
     /// Stuff inside brackets for attributes
     NtMeta(ast::MetaItem),
     NtPath(ast::Path),
@@ -585,7 +581,6 @@ pub enum Nonterminal {
     NtGenerics(ast::Generics),
     NtWhereClause(ast::WhereClause),
     NtArg(ast::Arg),
-    NtLifetime(ast::Lifetime),
 }
 
 impl fmt::Debug for Nonterminal {