Auto merge of #35957 - alexcrichton:macros-1.1, r=nrc

rustc: Implement custom derive (macros 1.1)

This commit is an implementation of [RFC 1681] which adds support to the
compiler for first-class user-define custom `#[derive]` modes with a far more
stable API than plugins have today.

[RFC 1681]: https://github.com/rust-lang/rfcs/blob/master/text/1681-macros-1.1.md

The main features added by this commit are:

* A new `rustc-macro` crate-type. This crate type represents one which will
  provide custom `derive` implementations and perhaps eventually flower into the
  implementation of macros 2.0 as well.

* A new `rustc_macro` crate in the standard distribution. This crate will
  provide the runtime interface between macro crates and the compiler. The API
  here is particularly conservative right now but has quite a bit of room to
  expand into any manner of APIs required by macro authors.

* The ability to load new derive modes through the `#[macro_use]` annotations on
  other crates.

All support added here is gated behind the `rustc_macro` feature gate, both for
the library support (the `rustc_macro` crate) as well as the language features.

There are a few minor differences from the implementation outlined in the RFC,
such as the `rustc_macro` crate being available as a dylib and all symbols are
`dlsym`'d directly instead of having a shim compiled. These should only affect
the implementation, however, not the public interface.

This commit also ended up touching a lot of code related to `#[derive]`, making
a few notable changes:

* Recognized derive attributes are no longer desugared to `derive_Foo`. Wasn't
  sure how to keep this behavior and *not* expose it to custom derive.

* Derive attributes no longer have access to unstable features by default, they
  have to opt in on a granular level.

* The `derive(Copy,Clone)` optimization is now done through another "obscure
  attribute" which is just intended to ferry along in the compiler that such an
  optimization is possible. The `derive(PartialEq,Eq)` optimization was also
  updated to do something similar.

---

One part of this PR which needs to be improved before stabilizing are the errors
and exact interfaces here. The error messages are relatively poor quality and
there are surprising spects of this such as `#[derive(PartialEq, Eq, MyTrait)]`
not working by default. The custom attributes added by the compiler end up
becoming unstable again when going through a custom impl.

Hopefully though this is enough to start allowing experimentation on crates.io!

3 files changed, 56 insertions, 12 deletions

diff --git a/src/libsyntax/ext/base.rs b/src/libsyntax/ext/base.rs
index 43e190f5deb..3b1c01319c4 100644
--- a/src/libsyntax/ext/base.rs
+++ b/src/libsyntax/ext/base.rs
@@ -27,10 +27,10 @@ use ptr::P;
 use util::small_vector::SmallVector;
 use util::lev_distance::find_best_match_for_name;
 use fold::Folder;
+use feature_gate;
 
 use std::collections::{HashMap, HashSet};
 use std::rc::Rc;
-use std::default::Default;
 use tokenstream;
 
 
@@ -568,12 +568,18 @@ fn initial_syntax_expander_table<'feat>(ecfg: &expand::ExpansionConfig<'feat>)
 }
 
 pub trait MacroLoader {
-    fn load_crate(&mut self, extern_crate: &ast::Item, allows_macros: bool) -> Vec<ast::MacroDef>;
+    fn load_crate(&mut self, extern_crate: &ast::Item, allows_macros: bool)
+                  -> Vec<LoadedMacro>;
+}
+
+pub enum LoadedMacro {
+    Def(ast::MacroDef),
+    CustomDerive(String, Box<MultiItemModifier>),
 }
 
 pub struct DummyMacroLoader;
 impl MacroLoader for DummyMacroLoader {
-    fn load_crate(&mut self, _: &ast::Item, _: bool) -> Vec<ast::MacroDef> {
+    fn load_crate(&mut self, _: &ast::Item, _: bool) -> Vec<LoadedMacro> {
         Vec::new()
     }
 }
@@ -593,6 +599,7 @@ pub struct ExtCtxt<'a> {
     pub exported_macros: Vec<ast::MacroDef>,
 
     pub syntax_env: SyntaxEnv,
+    pub derive_modes: HashMap<InternedString, Box<MultiItemModifier>>,
     pub recursion_count: usize,
 
     pub filename: Option<String>,
@@ -616,6 +623,7 @@ impl<'a> ExtCtxt<'a> {
             exported_macros: Vec::new(),
             loader: loader,
             syntax_env: env,
+            derive_modes: HashMap::new(),
             recursion_count: 0,
 
             filename: None,
@@ -714,6 +722,25 @@ impl<'a> ExtCtxt<'a> {
         }
     }
 
+    pub fn insert_custom_derive(&mut self,
+                                name: &str,
+                                ext: Box<MultiItemModifier>,
+                                sp: Span) {
+        if !self.ecfg.enable_rustc_macro() {
+            feature_gate::emit_feature_err(&self.parse_sess.span_diagnostic,
+                                           "rustc_macro",
+                                           sp,
+                                           feature_gate::GateIssue::Language,
+                                           "loading custom derive macro crates \
+                                            is experimentally supported");
+        }
+        let name = token::intern_and_get_ident(name);
+        if self.derive_modes.insert(name.clone(), ext).is_some() {
+            self.span_err(sp, &format!("cannot shadow existing derive mode `{}`",
+                                       name));
+        }
+    }
+
     pub fn struct_span_warn(&self,
                             sp: Span,
                             msg: &str)
diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index 15ebf95d623..d06b77a5b05 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -440,8 +440,7 @@ fn expand_annotatable(mut item: Annotatable, fld: &mut MacroExpander) -> SmallVe
                 callee: NameAndSpan {
                     format: MacroAttribute(intern(&attr.name())),
                     span: Some(attr.span),
-                    // attributes can do whatever they like, for now
-                    allow_internal_unstable: true,
+                    allow_internal_unstable: false,
                 }
             });
 
@@ -538,7 +537,12 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                     // We need to error on `#[macro_use] extern crate` when it isn't at the
                     // crate root, because `$crate` won't work properly.
                     for def in self.cx.loader.load_crate(item, self.at_crate_root) {
-                        self.cx.insert_macro(def);
+                        match def {
+                            LoadedMacro::Def(def) => self.cx.insert_macro(def),
+                            LoadedMacro::CustomDerive(name, ext) => {
+                                self.cx.insert_custom_derive(&name, ext, item.span);
+                            }
+                        }
                     }
                 } else {
                     let at_crate_root = ::std::mem::replace(&mut self.at_crate_root, false);
@@ -688,6 +692,7 @@ impl<'feat> ExpansionConfig<'feat> {
         fn enable_allow_internal_unstable = allow_internal_unstable,
         fn enable_custom_derive = custom_derive,
         fn enable_pushpop_unsafe = pushpop_unsafe,
+        fn enable_rustc_macro = rustc_macro,
     }
 }
 
diff --git a/src/libsyntax/feature_gate.rs b/src/libsyntax/feature_gate.rs
index 02c44c3a56d..683d5277359 100644
--- a/src/libsyntax/feature_gate.rs
+++ b/src/libsyntax/feature_gate.rs
@@ -47,7 +47,7 @@ macro_rules! setter {
 }
 
 macro_rules! declare_features {
-    ($((active, $feature: ident, $ver: expr, $issue: expr)),+) => {
+    ($((active, $feature: ident, $ver: expr, $issue: expr),)+) => {
         /// Represents active features that are currently being implemented or
         /// currently being considered for addition/removal.
         const ACTIVE_FEATURES: &'static [(&'static str, &'static str,
@@ -75,14 +75,14 @@ macro_rules! declare_features {
         }
     };
 
-    ($((removed, $feature: ident, $ver: expr, $issue: expr)),+) => {
+    ($((removed, $feature: ident, $ver: expr, $issue: expr),)+) => {
         /// Represents features which has since been removed (it was once Active)
         const REMOVED_FEATURES: &'static [(&'static str, &'static str, Option<u32>)] = &[
             $((stringify!($feature), $ver, $issue)),+
         ];
     };
 
-    ($((accepted, $feature: ident, $ver: expr, $issue: expr)),+) => {
+    ($((accepted, $feature: ident, $ver: expr, $issue: expr),)+) => {
         /// Those language feature has since been Accepted (it was once Active)
         const ACCEPTED_FEATURES: &'static [(&'static str, &'static str, Option<u32>)] = &[
             $((stringify!($feature), $ver, $issue)),+
@@ -288,7 +288,10 @@ declare_features! (
     (active, abi_sysv64, "1.13.0", Some(36167)),
 
     // Use the import semantics from RFC 1560.
-    (active, item_like_imports, "1.13.0", Some(35120))
+    (active, item_like_imports, "1.13.0", Some(35120)),
+
+    // Macros 1.1
+    (active, rustc_macro, "1.13.0", Some(35900)),
 );
 
 declare_features! (
@@ -302,7 +305,6 @@ declare_features! (
     (removed, struct_inherit, "1.0.0", None),
     (removed, test_removed_feature, "1.0.0", None),
     (removed, visible_private_types, "1.0.0", None),
-    (removed, unsafe_no_drop_flag, "1.0.0", None)
 );
 
 declare_features! (
@@ -330,7 +332,7 @@ declare_features! (
     (accepted, type_macros, "1.13.0", Some(27245)),
     (accepted, while_let, "1.0.0", None),
     // Allows `#[deprecated]` attribute
-    (accepted, deprecated, "1.9.0", Some(29935))
+    (accepted, deprecated, "1.9.0", Some(29935)),
 );
 // (changing above list without updating src/doc/reference.md makes @cmr sad)
 
@@ -543,6 +545,15 @@ pub const KNOWN_ATTRIBUTES: &'static [(&'static str, AttributeType, AttributeGat
                                    is an experimental feature",
                                   cfg_fn!(linked_from))),
 
+    ("rustc_macro_derive", Normal, Gated("rustc_macro",
+                                         "the `#[rustc_macro_derive]` attribute \
+                                          is an experimental feature",
+                                         cfg_fn!(rustc_macro))),
+
+    ("rustc_copy_clone_marker", Whitelisted, Gated("rustc_attrs",
+                                                   "internal implementation detail",
+                                                   cfg_fn!(rustc_attrs))),
+
     // FIXME: #14408 whitelist docs since rustdoc looks at them
     ("doc", Whitelisted, Ungated),
 
@@ -616,6 +627,7 @@ const GATED_CFGS: &'static [(&'static str, &'static str, fn(&Features) -> bool)]
     ("target_vendor", "cfg_target_vendor", cfg_fn!(cfg_target_vendor)),
     ("target_thread_local", "cfg_target_thread_local", cfg_fn!(cfg_target_thread_local)),
     ("target_has_atomic", "cfg_target_has_atomic", cfg_fn!(cfg_target_has_atomic)),
+    ("rustc_macro", "rustc_macro", cfg_fn!(rustc_macro)),
 ];
 
 #[derive(Debug, Eq, PartialEq)]