Auto merge of #50307 - petrochenkov:keyhyg2, r=nikomatsakis

Implement edition hygiene for keywords

Determine "keywordness" of an identifier in its hygienic context.
cc https://github.com/rust-lang/rust/pull/49611

I've resurrected `proc` as an Edition-2015-only keyword for testing purposes, but it should probably be buried again. EDIT: `proc` is removed again.

12 files changed, 63 insertions, 153 deletions

diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs
index 2b6635ec783..1817726d6a1 100644
--- a/src/libsyntax/ast.rs
+++ b/src/libsyntax/ast.rs
@@ -107,8 +107,7 @@ impl Path {
     // or starts with something like `self`/`super`/`$crate`/etc.
     pub fn make_root(&self) -> Option<PathSegment> {
         if let Some(ident) = self.segments.get(0).map(|seg| seg.ident) {
-            if ::parse::token::is_path_segment_keyword(ident) &&
-               ident.name != keywords::Crate.name() {
+            if ident.is_path_segment_keyword() && ident.name != keywords::Crate.name() {
                 return None;
             }
         }
diff --git a/src/libsyntax/edition.rs b/src/libsyntax/edition.rs
deleted file mode 100644
index c98b54581f3..00000000000
--- a/src/libsyntax/edition.rs
+++ /dev/null
@@ -1,82 +0,0 @@
-// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
-// file at the top-level directory of this distribution and at
-// http://rust-lang.org/COPYRIGHT.
-//
-// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
-// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
-// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
-// option. This file may not be copied, modified, or distributed
-// except according to those terms.
-
-use std::fmt;
-use std::str::FromStr;
-
-/// The edition of the compiler (RFC 2052)
-#[derive(Clone, Copy, Hash, PartialOrd, Ord, Eq, PartialEq, Debug)]
-#[non_exhaustive]
-pub enum Edition {
-    // editions must be kept in order, newest to oldest
-
-    /// The 2015 edition
-    Edition2015,
-    /// The 2018 edition
-    Edition2018,
-
-    // when adding new editions, be sure to update:
-    //
-    // - Update the `ALL_EDITIONS` const
-    // - Update the EDITION_NAME_LIST const
-    // - add a `rust_####()` function to the session
-    // - update the enum in Cargo's sources as well
-}
-
-// must be in order from oldest to newest
-pub const ALL_EDITIONS: &[Edition] = &[Edition::Edition2015, Edition::Edition2018];
-
-pub const EDITION_NAME_LIST: &'static str = "2015|2018";
-
-pub const DEFAULT_EDITION: Edition = Edition::Edition2015;
-
-impl fmt::Display for Edition {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        let s = match *self {
-            Edition::Edition2015 => "2015",
-            Edition::Edition2018 => "2018",
-        };
-        write!(f, "{}", s)
-    }
-}
-
-impl Edition {
-    pub fn lint_name(&self) -> &'static str {
-        match *self {
-            Edition::Edition2015 => "rust_2015_compatibility",
-            Edition::Edition2018 => "rust_2018_compatibility",
-        }
-    }
-
-    pub fn feature_name(&self) -> &'static str {
-        match *self {
-            Edition::Edition2015 => "rust_2015_preview",
-            Edition::Edition2018 => "rust_2018_preview",
-        }
-    }
-
-    pub fn is_stable(&self) -> bool {
-        match *self {
-            Edition::Edition2015 => true,
-            Edition::Edition2018 => false,
-        }
-    }
-}
-
-impl FromStr for Edition {
-    type Err = ();
-    fn from_str(s: &str) -> Result<Self, ()> {
-        match s {
-            "2015" => Ok(Edition::Edition2015),
-            "2018" => Ok(Edition::Edition2018),
-            _ => Err(())
-        }
-    }
-}
diff --git a/src/libsyntax/ext/base.rs b/src/libsyntax/ext/base.rs
index 3b76084f2fb..f7d4227977c 100644
--- a/src/libsyntax/ext/base.rs
+++ b/src/libsyntax/ext/base.rs
@@ -14,9 +14,10 @@ use ast::{self, Attribute, Name, PatKind, MetaItem};
 use attr::HasAttrs;
 use codemap::{self, CodeMap, Spanned, respan};
 use syntax_pos::{Span, MultiSpan, DUMMY_SP};
+use edition::Edition;
 use errors::{DiagnosticBuilder, DiagnosticId};
 use ext::expand::{self, Expansion, Invocation};
-use ext::hygiene::{Mark, SyntaxContext};
+use ext::hygiene::{self, Mark, SyntaxContext};
 use fold::{self, Folder};
 use parse::{self, parser, DirectoryOwnership};
 use parse::token;
@@ -586,13 +587,13 @@ pub enum SyntaxExtension {
     MultiModifier(Box<MultiItemModifier + sync::Sync + sync::Send>),
 
     /// A function-like procedural macro. TokenStream -> TokenStream.
-    ProcMacro(Box<ProcMacro + sync::Sync + sync::Send>),
+    ProcMacro(Box<ProcMacro + sync::Sync + sync::Send>, Edition),
 
     /// An attribute-like procedural macro. TokenStream, TokenStream -> TokenStream.
     /// The first TokenSteam is the attribute, the second is the annotated item.
     /// Allows modification of the input items and adding new items, similar to
     /// MultiModifier, but uses TokenStreams, rather than AST nodes.
-    AttrProcMacro(Box<AttrProcMacro + sync::Sync + sync::Send>),
+    AttrProcMacro(Box<AttrProcMacro + sync::Sync + sync::Send>, Edition),
 
     /// A normal, function-like syntax extension.
     ///
@@ -608,6 +609,8 @@ pub enum SyntaxExtension {
         allow_internal_unsafe: bool,
         /// The macro's feature name if it is unstable, and the stability feature
         unstable_feature: Option<(Symbol, u32)>,
+        /// Edition of the crate in which the macro is defined
+        edition: Edition,
     },
 
     /// A function-like syntax extension that has an extra ident before
@@ -619,9 +622,8 @@ pub enum SyntaxExtension {
     /// The input is the annotated item.
     /// Allows generating code to implement a Trait for a given struct
     /// or enum item.
-    ProcMacroDerive(Box<MultiItemModifier +
-                        sync::Sync +
-                        sync::Send>, Vec<Symbol> /* inert attribute names */),
+    ProcMacroDerive(Box<MultiItemModifier + sync::Sync + sync::Send>,
+                    Vec<Symbol> /* inert attribute names */, Edition),
 
     /// An attribute-like procedural macro that derives a builtin trait.
     BuiltinDerive(BuiltinDeriveFn),
@@ -629,7 +631,7 @@ pub enum SyntaxExtension {
     /// A declarative macro, e.g. `macro m() {}`.
     ///
     /// The second element is the definition site span.
-    DeclMacro(Box<TTMacroExpander + sync::Sync + sync::Send>, Option<(ast::NodeId, Span)>),
+    DeclMacro(Box<TTMacroExpander + sync::Sync + sync::Send>, Option<(ast::NodeId, Span)>, Edition),
 }
 
 impl SyntaxExtension {
@@ -660,6 +662,21 @@ impl SyntaxExtension {
             _ => false,
         }
     }
+
+    pub fn edition(&self) -> Edition {
+        match *self {
+            SyntaxExtension::NormalTT { edition, .. } |
+            SyntaxExtension::DeclMacro(.., edition) |
+            SyntaxExtension::ProcMacro(.., edition) |
+            SyntaxExtension::AttrProcMacro(.., edition) |
+            SyntaxExtension::ProcMacroDerive(.., edition) => edition,
+            // Unstable legacy stuff
+            SyntaxExtension::IdentTT(..) |
+            SyntaxExtension::MultiDecorator(..) |
+            SyntaxExtension::MultiModifier(..) |
+            SyntaxExtension::BuiltinDerive(..) => hygiene::default_edition(),
+        }
+    }
 }
 
 pub type NamedSyntaxExtension = (Name, SyntaxExtension);
diff --git a/src/libsyntax/ext/derive.rs b/src/libsyntax/ext/derive.rs
index 6bf166dfe95..0b6a7e1c4f4 100644
--- a/src/libsyntax/ext/derive.rs
+++ b/src/libsyntax/ext/derive.rs
@@ -10,7 +10,7 @@
 
 use attr::HasAttrs;
 use ast;
-use codemap::{ExpnInfo, NameAndSpan, ExpnFormat};
+use codemap::{hygiene, ExpnInfo, NameAndSpan, ExpnFormat};
 use ext::base::ExtCtxt;
 use ext::build::AstBuilder;
 use parse::parser::PathStyle;
@@ -65,6 +65,7 @@ pub fn add_derived_markers<T>(cx: &mut ExtCtxt, span: Span, traits: &[ast::Path]
             span: None,
             allow_internal_unstable: true,
             allow_internal_unsafe: false,
+            edition: hygiene::default_edition(),
         },
     });
 
diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index 584b9455a93..ee96963362b 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -16,7 +16,7 @@ use config::{is_test_or_bench, StripUnconfigured};
 use errors::FatalError;
 use ext::base::*;
 use ext::derive::{add_derived_markers, collect_derives};
-use ext::hygiene::{Mark, SyntaxContext};
+use ext::hygiene::{self, Mark, SyntaxContext};
 use ext::placeholders::{placeholder, PlaceholderExpander};
 use feature_gate::{self, Features, GateIssue, is_builtin_attr, emit_feature_err};
 use fold;
@@ -502,6 +502,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                 span: None,
                 allow_internal_unstable: false,
                 allow_internal_unsafe: false,
+                edition: ext.edition(),
             }
         });
 
@@ -520,7 +521,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                 items.push(item);
                 Some(kind.expect_from_annotatables(items))
             }
-            AttrProcMacro(ref mac) => {
+            AttrProcMacro(ref mac, ..) => {
                 self.gate_proc_macro_attr_item(attr.span, &item);
                 let item_tok = TokenTree::Token(DUMMY_SP, Token::interpolated(match item {
                     Annotatable::Item(item) => token::NtItem(item),
@@ -609,7 +610,8 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                                           allow_internal_unstable,
                                           allow_internal_unsafe,
                                           // can't infer this type
-                                          unstable_feature: Option<(Symbol, u32)>| {
+                                          unstable_feature: Option<(Symbol, u32)>,
+                                          edition| {
 
             // feature-gate the macro invocation
             if let Some((feature, issue)) = unstable_feature {
@@ -642,15 +644,17 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                     span: def_site_span,
                     allow_internal_unstable,
                     allow_internal_unsafe,
+                    edition,
                 },
             });
             Ok(())
         };
 
         let opt_expanded = match *ext {
-            DeclMacro(ref expand, def_span) => {
+            DeclMacro(ref expand, def_span, edition) => {
                 if let Err(dummy_span) = validate_and_set_expn_info(self, def_span.map(|(_, s)| s),
-                                                                    false, false, None) {
+                                                                    false, false, None,
+                                                                    edition) {
                     dummy_span
                 } else {
                     kind.make_from(expand.expand(self.cx, span, mac.node.stream()))
@@ -663,11 +667,13 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                 allow_internal_unstable,
                 allow_internal_unsafe,
                 unstable_feature,
+                edition,
             } => {
                 if let Err(dummy_span) = validate_and_set_expn_info(self, def_info.map(|(_, s)| s),
                                                                     allow_internal_unstable,
                                                                     allow_internal_unsafe,
-                                                                    unstable_feature) {
+                                                                    unstable_feature,
+                                                                    edition) {
                     dummy_span
                 } else {
                     kind.make_from(expander.expand(self.cx, span, mac.node.stream()))
@@ -688,6 +694,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                             span: tt_span,
                             allow_internal_unstable,
                             allow_internal_unsafe: false,
+                            edition: hygiene::default_edition(),
                         }
                     });
 
@@ -709,7 +716,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                 kind.dummy(span)
             }
 
-            ProcMacro(ref expandfun) => {
+            ProcMacro(ref expandfun, edition) => {
                 if ident.name != keywords::Invalid.name() {
                     let msg =
                         format!("macro {}! expects no ident argument, given '{}'", path, ident);
@@ -728,6 +735,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                             // FIXME probably want to follow macro_rules macros here.
                             allow_internal_unstable: false,
                             allow_internal_unsafe: false,
+                            edition,
                         },
                     });
 
@@ -802,11 +810,12 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                 span: None,
                 allow_internal_unstable: false,
                 allow_internal_unsafe: false,
+                edition: ext.edition(),
             }
         };
 
         match *ext {
-            ProcMacroDerive(ref ext, _) => {
+            ProcMacroDerive(ref ext, ..) => {
                 invoc.expansion_data.mark.set_expn_info(expn_info);
                 let span = span.with_ctxt(self.cx.backtrace());
                 let dummy = ast::MetaItem { // FIXME(jseyfried) avoid this
diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs
index e96a0e838cf..d1a7e7aac26 100644
--- a/src/libsyntax/ext/tt/macro_rules.rs
+++ b/src/libsyntax/ext/tt/macro_rules.rs
@@ -10,6 +10,7 @@
 
 use {ast, attr};
 use syntax_pos::{Span, DUMMY_SP};
+use edition::Edition;
 use ext::base::{DummyResult, ExtCtxt, MacResult, SyntaxExtension};
 use ext::base::{NormalTT, TTMacroExpander};
 use ext::expand::{Expansion, ExpansionKind};
@@ -183,7 +184,8 @@ fn generic_extension<'cx>(cx: &'cx mut ExtCtxt,
 // Holy self-referential!
 
 /// Converts a `macro_rules!` invocation into a syntax extension.
-pub fn compile(sess: &ParseSess, features: &Features, def: &ast::Item) -> SyntaxExtension {
+pub fn compile(sess: &ParseSess, features: &Features, def: &ast::Item, edition: Edition)
+               -> SyntaxExtension {
     let lhs_nm = ast::Ident::with_empty_ctxt(Symbol::gensym("lhs"));
     let rhs_nm = ast::Ident::with_empty_ctxt(Symbol::gensym("rhs"));
 
@@ -298,10 +300,11 @@ pub fn compile(sess: &ParseSess, features: &Features, def: &ast::Item) -> Syntax
             def_info: Some((def.id, def.span)),
             allow_internal_unstable,
             allow_internal_unsafe,
-            unstable_feature
+            unstable_feature,
+            edition,
         }
     } else {
-        SyntaxExtension::DeclMacro(expander, Some((def.id, def.span)))
+        SyntaxExtension::DeclMacro(expander, Some((def.id, def.span)), edition)
     }
 }
 
diff --git a/src/libsyntax/lib.rs b/src/libsyntax/lib.rs
index 90af3ba51ec..e9817034569 100644
--- a/src/libsyntax/lib.rs
+++ b/src/libsyntax/lib.rs
@@ -22,7 +22,6 @@
 #![feature(unicode_internals)]
 #![feature(rustc_diagnostic_macros)]
 #![feature(slice_sort_by_cached_key)]
-#![feature(non_exhaustive)]
 #![feature(const_atomic_usize_new)]
 #![feature(rustc_attrs)]
 #![feature(str_escape)]
@@ -142,7 +141,6 @@ pub mod codemap;
 #[macro_use]
 pub mod config;
 pub mod entry;
-pub mod edition;
 pub mod feature_gate;
 pub mod fold;
 pub mod parse;
@@ -150,6 +148,7 @@ pub mod ptr;
 pub mod show_span;
 pub mod std_inject;
 pub mod str;
+pub use syntax_pos::edition;
 pub use syntax_pos::symbol;
 pub mod test;
 pub mod tokenstream;
diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs
index a0434fe6616..bbece1ee5e3 100644
--- a/src/libsyntax/parse/lexer/mod.rs
+++ b/src/libsyntax/parse/lexer/mod.rs
@@ -1128,7 +1128,7 @@ impl<'a> StringReader<'a> {
                 return Ok(self.with_str_from(start, |string| {
                     // FIXME: perform NFKC normalization here. (Issue #2253)
                     let ident = self.mk_ident(string);
-                    if is_raw_ident && (token::is_path_segment_keyword(ident) ||
+                    if is_raw_ident && (ident.is_path_segment_keyword() ||
                                         ident.name == keywords::Underscore.name()) {
                         self.fatal_span_(raw_start, self.pos,
                             &format!("`r#{}` is not currently supported.", ident.name)
diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs
index a1c056cbb2c..5575614a4d4 100644
--- a/src/libsyntax/parse/token.rs
+++ b/src/libsyntax/parse/token.rs
@@ -138,44 +138,6 @@ fn ident_can_begin_type(ident: ast::Ident, is_raw: bool) -> bool {
     ].contains(&ident.name)
 }
 
-pub fn is_path_segment_keyword(id: ast::Ident) -> bool {
-    id.name == keywords::Super.name() ||
-    id.name == keywords::SelfValue.name() ||
-    id.name == keywords::SelfType.name() ||
-    id.name == keywords::Extern.name() ||
-    id.name == keywords::Crate.name() ||
-    id.name == keywords::CrateRoot.name() ||
-    id.name == keywords::DollarCrate.name()
-}
-
-// We see this identifier in a normal identifier position, like variable name or a type.
-// How was it written originally? Did it use the raw form? Let's try to guess.
-pub fn is_raw_guess(ident: ast::Ident) -> bool {
-    ident.name != keywords::Invalid.name() &&
-    is_reserved_ident(ident) && !is_path_segment_keyword(ident)
-}
-
-// Returns true for reserved identifiers used internally for elided lifetimes,
-// unnamed method parameters, crate root module, error recovery etc.
-pub fn is_special_ident(id: ast::Ident) -> bool {
-    id.name <= keywords::Underscore.name()
-}
-
-/// Returns `true` if the token is a keyword used in the language.
-pub fn is_used_keyword(id: ast::Ident) -> bool {
-    id.name >= keywords::As.name() && id.name <= keywords::While.name()
-}
-
-/// Returns `true` if the token is a keyword reserved for possible future use.
-pub fn is_unused_keyword(id: ast::Ident) -> bool {
-    id.name >= keywords::Abstract.name() && id.name <= keywords::Yield.name()
-}
-
-/// Returns `true` if the token is either a special identifier or a keyword.
-pub fn is_reserved_ident(id: ast::Ident) -> bool {
-    is_special_ident(id) || is_used_keyword(id) || is_unused_keyword(id)
-}
-
 #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug)]
 pub enum Token {
     /* Expression-operator symbols. */
@@ -251,7 +213,7 @@ impl Token {
 
     /// Recovers a `Token` from an `ast::Ident`. This creates a raw identifier if necessary.
     pub fn from_ast_ident(ident: ast::Ident) -> Token {
-        Ident(ident, is_raw_guess(ident))
+        Ident(ident, ident.is_raw_guess())
     }
 
     /// Returns `true` if the token starts with '>'.
@@ -431,7 +393,7 @@ impl Token {
 
     pub fn is_path_segment_keyword(&self) -> bool {
         match self.ident() {
-            Some((id, false)) => is_path_segment_keyword(id),
+            Some((id, false)) => id.is_path_segment_keyword(),
             _ => false,
         }
     }
@@ -440,7 +402,7 @@ impl Token {
     // unnamed method parameters, crate root module, error recovery etc.
     pub fn is_special_ident(&self) -> bool {
         match self.ident() {
-            Some((id, false)) => is_special_ident(id),
+            Some((id, false)) => id.is_special(),
             _ => false,
         }
     }
@@ -448,7 +410,7 @@ impl Token {
     /// Returns `true` if the token is a keyword used in the language.
     pub fn is_used_keyword(&self) -> bool {
         match self.ident() {
-            Some((id, false)) => is_used_keyword(id),
+            Some((id, false)) => id.is_used_keyword(),
             _ => false,
         }
     }
@@ -456,7 +418,7 @@ impl Token {
     /// Returns `true` if the token is a keyword reserved for possible future use.
     pub fn is_unused_keyword(&self) -> bool {
         match self.ident() {
-            Some((id, false)) => is_unused_keyword(id),
+            Some((id, false)) => id.is_unused_keyword(),
             _ => false,
         }
     }
@@ -464,7 +426,7 @@ impl Token {
     /// Returns `true` if the token is either a special identifier or a keyword.
     pub fn is_reserved_ident(&self) -> bool {
         match self.ident() {
-            Some((id, false)) => is_reserved_ident(id),
+            Some((id, false)) => id.is_reserved(),
             _ => false,
         }
     }
diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs
index a700799cde5..17f83a09c77 100644
--- a/src/libsyntax/print/pprust.rs
+++ b/src/libsyntax/print/pprust.rs
@@ -2374,7 +2374,7 @@ impl<'a> State<'a> {
     }
 
     pub fn print_ident(&mut self, ident: ast::Ident) -> io::Result<()> {
-        if token::is_raw_guess(ident) {
+        if ident.is_raw_guess() {
             self.s.word(&format!("r#{}", ident))?;
         } else {
             self.s.word(&ident.name.as_str())?;
diff --git a/src/libsyntax/std_inject.rs b/src/libsyntax/std_inject.rs
index 53dc19ba37d..e9cd7adb9c1 100644
--- a/src/libsyntax/std_inject.rs
+++ b/src/libsyntax/std_inject.rs
@@ -14,7 +14,7 @@ use std::cell::Cell;
 use ext::hygiene::{Mark, SyntaxContext};
 use symbol::{Symbol, keywords};
 use syntax_pos::{DUMMY_SP, Span};
-use codemap::{ExpnInfo, NameAndSpan, MacroAttribute, dummy_spanned, respan};
+use codemap::{ExpnInfo, NameAndSpan, MacroAttribute, dummy_spanned, hygiene, respan};
 use ptr::P;
 use tokenstream::TokenStream;
 
@@ -30,6 +30,7 @@ fn ignored_span(sp: Span) -> Span {
             span: None,
             allow_internal_unstable: true,
             allow_internal_unsafe: false,
+            edition: hygiene::default_edition(),
         }
     });
     sp.with_ctxt(SyntaxContext::empty().apply_mark(mark))
diff --git a/src/libsyntax/test.rs b/src/libsyntax/test.rs
index 1734692f9e7..1dfd48a24c3 100644
--- a/src/libsyntax/test.rs
+++ b/src/libsyntax/test.rs
@@ -29,7 +29,7 @@ use entry::{self, EntryPointType};
 use ext::base::{ExtCtxt, Resolver};
 use ext::build::AstBuilder;
 use ext::expand::ExpansionConfig;
-use ext::hygiene::{Mark, SyntaxContext};
+use ext::hygiene::{self, Mark, SyntaxContext};
 use fold::Folder;
 use feature_gate::Features;
 use util::move_map::MoveMap;
@@ -300,6 +300,7 @@ fn generate_test_harness(sess: &ParseSess,
             span: None,
             allow_internal_unstable: true,
             allow_internal_unsafe: false,
+            edition: hygiene::default_edition(),
         }
     });