Auto merge of #63269 - Aaron1011:feature/proc-macro-data, r=eddyb,petrochenkov

Serialize additional data for procedural macros

Split off from #62855

This PR serializes the declaration `Span` and attributes for all
procedural macros. This allows Rustdoc to properly render doc comments
and source links when performing inlinig procedural macros across crates

17 files changed, 240 insertions, 210 deletions

diff --git a/src/libproc_macro/bridge/client.rs b/src/libproc_macro/bridge/client.rs
index 6052b4a4d43..5c543165bc2 100644
--- a/src/libproc_macro/bridge/client.rs
+++ b/src/libproc_macro/bridge/client.rs
@@ -468,6 +468,14 @@ pub enum ProcMacro {
 }
 
 impl ProcMacro {
+    pub fn name(&self) -> &'static str {
+        match self {
+            ProcMacro::CustomDerive { trait_name, .. } => trait_name,
+            ProcMacro::Attr { name, .. } => name,
+            ProcMacro::Bang { name, ..} => name
+        }
+    }
+
     pub const fn custom_derive(
         trait_name: &'static str,
         attributes: &'static [&'static str],
diff --git a/src/librustc/hir/def_id.rs b/src/librustc/hir/def_id.rs
index c0a661908a6..c91ad7858d0 100644
--- a/src/librustc/hir/def_id.rs
+++ b/src/librustc/hir/def_id.rs
@@ -1,5 +1,4 @@
 use crate::ty::{self, TyCtxt};
-use crate::hir::map::definitions::FIRST_FREE_DEF_INDEX;
 use rustc_data_structures::indexed_vec::Idx;
 use std::fmt;
 use std::u32;
@@ -102,31 +101,6 @@ newtype_index! {
     }
 }
 
-impl DefIndex {
-    // Proc macros from a proc-macro crate have a kind of virtual DefIndex. This
-    // function maps the index of the macro within the crate (which is also the
-    // index of the macro in the CrateMetadata::proc_macros array) to the
-    // corresponding DefIndex.
-    pub fn from_proc_macro_index(proc_macro_index: usize) -> DefIndex {
-        // DefIndex for proc macros start from FIRST_FREE_DEF_INDEX,
-        // because the first FIRST_FREE_DEF_INDEX indexes are reserved
-        // for internal use.
-        let def_index = DefIndex::from(
-            proc_macro_index.checked_add(FIRST_FREE_DEF_INDEX)
-                .expect("integer overflow adding `proc_macro_index`"));
-        assert!(def_index != CRATE_DEF_INDEX);
-        def_index
-    }
-
-    // This function is the reverse of from_proc_macro_index() above.
-    pub fn to_proc_macro_index(self: DefIndex) -> usize {
-        self.index().checked_sub(FIRST_FREE_DEF_INDEX)
-            .unwrap_or_else(|| {
-                bug!("using local index {:?} as proc-macro index", self)
-            })
-    }
-}
-
 impl rustc_serialize::UseSpecializedEncodable for DefIndex {}
 impl rustc_serialize::UseSpecializedDecodable for DefIndex {}
 
diff --git a/src/librustc/hir/map/definitions.rs b/src/librustc/hir/map/definitions.rs
index 8ee8c6d0e89..6dc3c7038f5 100644
--- a/src/librustc/hir/map/definitions.rs
+++ b/src/librustc/hir/map/definitions.rs
@@ -411,10 +411,6 @@ impl Definitions {
     }
 
     /// Adds a root definition (no parent) and a few other reserved definitions.
-    ///
-    /// After the initial definitions are created the first `FIRST_FREE_DEF_INDEX` indexes
-    /// are taken, so the "user" indexes will be allocated starting with `FIRST_FREE_DEF_INDEX`
-    /// in ascending order.
     pub fn create_root_def(&mut self,
                            crate_name: &str,
                            crate_disambiguator: CrateDisambiguator)
@@ -589,19 +585,6 @@ impl DefPathData {
     }
 }
 
-/// Evaluates to the number of tokens passed to it.
-///
-/// Logarithmic counting: every one or two recursive expansions, the number of
-/// tokens to count is divided by two, instead of being reduced by one.
-/// Therefore, the recursion depth is the binary logarithm of the number of
-/// tokens to count, and the expanded tree is likewise very small.
-macro_rules! count {
-    ()                     => (0usize);
-    ($one:tt)              => (1usize);
-    ($($pairs:tt $_p:tt)*) => (count!($($pairs)*) << 1usize);
-    ($odd:tt $($rest:tt)*) => (count!($($rest)*) | 1usize);
-}
-
 // We define the GlobalMetaDataKind enum with this macro because we want to
 // make sure that we exhaustively iterate over all variants when registering
 // the corresponding DefIndices in the DefTable.
@@ -614,8 +597,6 @@ macro_rules! define_global_metadata_kind {
             $($variant),*
         }
 
-        pub const FIRST_FREE_DEF_INDEX: usize = 1 + count!($($variant)*);
-
         impl GlobalMetaDataKind {
             fn allocate_def_indices(definitions: &mut Definitions) {
                 $({
diff --git a/src/librustc_metadata/creader.rs b/src/librustc_metadata/creader.rs
index d5f1e715186..af41b6a4c85 100644
--- a/src/librustc_metadata/creader.rs
+++ b/src/librustc_metadata/creader.rs
@@ -2,8 +2,7 @@
 
 use crate::cstore::{self, CStore, CrateSource, MetadataBlob};
 use crate::locator::{self, CratePaths};
-use crate::decoder::proc_macro_def_path_table;
-use crate::schema::CrateRoot;
+use crate::schema::{CrateRoot};
 use rustc_data_structures::sync::{Lrc, RwLock, Lock};
 
 use rustc::hir::def_id::CrateNum;
@@ -26,11 +25,11 @@ use std::{cmp, fs};
 use syntax::ast;
 use syntax::attr;
 use syntax::ext::allocator::{global_allocator_spans, AllocatorKind};
-use syntax::ext::base::{SyntaxExtension, SyntaxExtensionKind};
 use syntax::symbol::{Symbol, sym};
 use syntax::{span_err, span_fatal};
 use syntax_pos::{Span, DUMMY_SP};
 use log::{debug, info, log_enabled};
+use proc_macro::bridge::client::ProcMacro;
 
 pub struct Library {
     pub dylib: Option<(PathBuf, PathKind)>,
@@ -230,24 +229,13 @@ impl<'a> CrateLoader<'a> {
 
         let dependencies: Vec<CrateNum> = cnum_map.iter().cloned().collect();
 
-        let proc_macros = crate_root.proc_macro_decls_static.map(|_| {
+        let raw_proc_macros =  crate_root.proc_macro_data.map(|_| {
             if self.sess.opts.debugging_opts.dual_proc_macros {
-                let host_lib = host_lib.unwrap();
-                self.load_derive_macros(
-                    &host_lib.metadata.get_root(),
-                    host_lib.dylib.map(|p| p.0),
-                    span
-                )
+                let host_lib = host_lib.as_ref().unwrap();
+                self.dlsym_proc_macros(host_lib.dylib.as_ref().map(|p| p.0.clone()),
+                                       &host_lib.metadata.get_root(), span)
             } else {
-                self.load_derive_macros(&crate_root, dylib.clone().map(|p| p.0), span)
-            }
-        });
-
-        let def_path_table = record_time(&self.sess.perf_stats.decode_def_path_tables_time, || {
-            if let Some(proc_macros) = &proc_macros {
-                proc_macro_def_path_table(&crate_root, proc_macros)
-            } else {
-                crate_root.def_path_table.decode((&metadata, self.sess))
+                self.dlsym_proc_macros(dylib.clone().map(|p| p.0), &crate_root, span)
             }
         });
 
@@ -260,13 +248,16 @@ impl<'a> CrateLoader<'a> {
             .map(|trait_impls| (trait_impls.trait_id, trait_impls.impls))
             .collect();
 
+        let def_path_table = record_time(&self.sess.perf_stats.decode_def_path_tables_time, || {
+            crate_root.def_path_table.decode((&metadata, self.sess))
+        });
+
         let cmeta = cstore::CrateMetadata {
             name: crate_root.name,
             imported_name: ident,
             extern_crate: Lock::new(None),
             def_path_table: Lrc::new(def_path_table),
             trait_impls,
-            proc_macros,
             root: crate_root,
             blob: metadata,
             cnum_map,
@@ -280,7 +271,10 @@ impl<'a> CrateLoader<'a> {
                 rlib,
                 rmeta,
             },
-            private_dep
+            private_dep,
+            span,
+            host_lib,
+            raw_proc_macros
         };
 
         let cmeta = Lrc::new(cmeta);
@@ -389,7 +383,7 @@ impl<'a> CrateLoader<'a> {
         match result {
             (LoadResult::Previous(cnum), None) => {
                 let data = self.cstore.get_crate_data(cnum);
-                if data.root.proc_macro_decls_static.is_some() {
+                if data.root.proc_macro_data.is_some() {
                     dep_kind = DepKind::UnexportedMacrosOnly;
                 }
                 data.dep_kind.with_lock(|data_dep_kind| {
@@ -482,7 +476,7 @@ impl<'a> CrateLoader<'a> {
                           dep_kind: DepKind)
                           -> cstore::CrateNumMap {
         debug!("resolving deps of external crate");
-        if crate_root.proc_macro_decls_static.is_some() {
+        if crate_root.proc_macro_data.is_some() {
             return cstore::CrateNumMap::new();
         }
 
@@ -574,19 +568,13 @@ impl<'a> CrateLoader<'a> {
         }
     }
 
-    /// Loads custom derive macros.
-    ///
-    /// Note that this is intentionally similar to how we load plugins today,
-    /// but also intentionally separate. Plugins are likely always going to be
-    /// implemented as dynamic libraries, but we have a possible future where
-    /// custom derive (and other macro-1.1 style features) are implemented via
-    /// executables and custom IPC.
-    fn load_derive_macros(&mut self, root: &CrateRoot<'_>, dylib: Option<PathBuf>, span: Span)
-                          -> Vec<(ast::Name, Lrc<SyntaxExtension>)> {
-        use std::{env, mem};
+    fn dlsym_proc_macros(&self,
+                         dylib: Option<PathBuf>,
+                         root: &CrateRoot<'_>,
+                         span: Span
+    ) -> &'static [ProcMacro] {
+        use std::env;
         use crate::dynamic_lib::DynamicLibrary;
-        use proc_macro::bridge::client::ProcMacro;
-        use syntax::ext::proc_macro::{BangProcMacro, AttrProcMacro, ProcMacroDerive};
 
         let path = match dylib {
             Some(dylib) => dylib,
@@ -608,38 +596,11 @@ impl<'a> CrateLoader<'a> {
             *(sym as *const &[ProcMacro])
         };
 
-        let extensions = decls.iter().map(|&decl| {
-            let (name, kind, helper_attrs) = match decl {
-                ProcMacro::CustomDerive { trait_name, attributes, client } => {
-                    let helper_attrs =
-                        attributes.iter().cloned().map(Symbol::intern).collect::<Vec<_>>();
-                    (
-                        trait_name,
-                        SyntaxExtensionKind::Derive(Box::new(ProcMacroDerive {
-                            client, attrs: helper_attrs.clone()
-                        })),
-                        helper_attrs,
-                    )
-                }
-                ProcMacro::Attr { name, client } => (
-                    name, SyntaxExtensionKind::Attr(Box::new(AttrProcMacro { client })), Vec::new()
-                ),
-                ProcMacro::Bang { name, client } => (
-                    name, SyntaxExtensionKind::Bang(Box::new(BangProcMacro { client })), Vec::new()
-                )
-            };
-
-            (Symbol::intern(name), Lrc::new(SyntaxExtension {
-                helper_attrs,
-                ..SyntaxExtension::default(kind, root.edition)
-            }))
-        }).collect();
-
         // Intentionally leak the dynamic library. We can't ever unload it
         // since the library can make things that will live arbitrarily long.
-        mem::forget(lib);
+        std::mem::forget(lib);
 
-        extensions
+        decls
     }
 
     /// Look for a plugin registrar. Returns library path, crate
diff --git a/src/librustc_metadata/cstore.rs b/src/librustc_metadata/cstore.rs
index 4ac0a5b94c0..792922a1837 100644
--- a/src/librustc_metadata/cstore.rs
+++ b/src/librustc_metadata/cstore.rs
@@ -28,6 +28,9 @@ pub use crate::cstore_impl::{provide, provide_extern};
 pub type CrateNumMap = IndexVec<CrateNum, CrateNum>;
 
 pub use rustc_data_structures::sync::MetadataRef;
+use crate::creader::Library;
+use syntax_pos::Span;
+use proc_macro::bridge::client::ProcMacro;
 
 pub struct MetadataBlob(pub MetadataRef);
 
@@ -82,11 +85,19 @@ pub struct CrateMetadata {
     pub dep_kind: Lock<DepKind>,
     pub source: CrateSource,
 
-    pub proc_macros: Option<Vec<(ast::Name, Lrc<SyntaxExtension>)>>,
-
     /// Whether or not this crate should be consider a private dependency
     /// for purposes of the 'exported_private_dependencies' lint
-    pub private_dep: bool
+    pub private_dep: bool,
+
+    pub host_lib: Option<Library>,
+    pub span: Span,
+
+    pub raw_proc_macros: Option<&'static [ProcMacro]>,
+}
+
+pub struct FullProcMacro {
+    pub name: ast::Name,
+    pub ext: Lrc<SyntaxExtension>
 }
 
 pub struct CStore {
diff --git a/src/librustc_metadata/cstore_impl.rs b/src/librustc_metadata/cstore_impl.rs
index b46758abb5f..a66da32fa4d 100644
--- a/src/librustc_metadata/cstore_impl.rs
+++ b/src/librustc_metadata/cstore_impl.rs
@@ -426,8 +426,8 @@ impl cstore::CStore {
 
     pub fn load_macro_untracked(&self, id: DefId, sess: &Session) -> LoadedMacro {
         let data = self.get_crate_data(id.krate);
-        if let Some(ref proc_macros) = data.proc_macros {
-            return LoadedMacro::ProcMacro(proc_macros[id.index.to_proc_macro_index()].1.clone());
+        if data.is_proc_macro_crate() {
+            return LoadedMacro::ProcMacro(data.get_proc_macro(id.index, sess).ext);
         } else if data.name == sym::proc_macro && data.item_name(id.index) == sym::quote {
             let client = proc_macro::bridge::client::Client::expand1(proc_macro::quote);
             let kind = SyntaxExtensionKind::Bang(Box::new(BangProcMacro { client }));
@@ -439,7 +439,8 @@ impl cstore::CStore {
         }
 
         let def = data.get_macro(id.index);
-        let macro_full_name = data.def_path(id.index).to_string_friendly(|_| data.imported_name);
+        let macro_full_name = data.def_path(id.index)
+            .to_string_friendly(|_| data.imported_name);
         let source_name = FileName::Macros(macro_full_name);
 
         let source_file = sess.parse_sess.source_map().new_source_file(source_name, def.body);
diff --git a/src/librustc_metadata/decoder.rs b/src/librustc_metadata/decoder.rs
index 772b2d3ec4d..d29592a5d68 100644
--- a/src/librustc_metadata/decoder.rs
+++ b/src/librustc_metadata/decoder.rs
@@ -1,16 +1,15 @@
 // Decoding metadata from a single crate's metadata
 
-use crate::cstore::{self, CrateMetadata, MetadataBlob, NativeLibrary, ForeignModule};
+use crate::cstore::{self, CrateMetadata, MetadataBlob, NativeLibrary, ForeignModule, FullProcMacro};
 use crate::schema::*;
 
 use rustc_data_structures::sync::{Lrc, ReadGuard};
-use rustc::hir::map::{DefKey, DefPath, DefPathData, DefPathHash, Definitions};
+use rustc::hir::map::{DefKey, DefPath, DefPathData, DefPathHash};
 use rustc::hir;
 use rustc::middle::cstore::LinkagePreference;
 use rustc::middle::exported_symbols::{ExportedSymbol, SymbolExportLevel};
 use rustc::hir::def::{self, Res, DefKind, CtorOf, CtorKind};
 use rustc::hir::def_id::{CrateNum, DefId, DefIndex, LocalDefId, CRATE_DEF_INDEX, LOCAL_CRATE};
-use rustc::hir::map::definitions::DefPathTable;
 use rustc_data_structures::fingerprint::Fingerprint;
 use rustc::middle::lang_items;
 use rustc::mir::{self, interpret};
@@ -30,10 +29,11 @@ use syntax::attr;
 use syntax::ast::{self, Ident};
 use syntax::source_map;
 use syntax::symbol::{Symbol, sym};
-use syntax::ext::base::{MacroKind, SyntaxExtension};
-use syntax::ext::hygiene::ExpnId;
-use syntax_pos::{self, Span, BytePos, Pos, DUMMY_SP};
+use syntax::ext::base::{MacroKind, SyntaxExtensionKind, SyntaxExtension};
+use syntax_pos::{self, Span, BytePos, Pos, DUMMY_SP, symbol::{InternedString}};
 use log::debug;
+use proc_macro::bridge::client::ProcMacro;
+use syntax::ext::proc_macro::{AttrProcMacro, ProcMacroDerive, BangProcMacro};
 
 pub struct DecodeContext<'a, 'tcx> {
     opaque: opaque::Decoder<'a>,
@@ -138,7 +138,7 @@ impl<'a: 'x, 'tcx: 'x, 'x, T: Decodable> LazySeq<T> {
     pub fn decode<M: Metadata<'a, 'tcx>>(
         self,
         meta: M,
-    ) -> impl Iterator<Item = T> + Captures<'a> + Captures<'tcx> + 'x {
+    ) -> impl ExactSizeIterator<Item = T> + Captures<'a> + Captures<'tcx> + 'x {
         let mut dcx = meta.decoder(self.position);
         dcx.lazy_state = LazyState::NodeStart(self.position);
         (0..self.len).map(move |_| T::decode(&mut dcx).unwrap())
@@ -442,46 +442,16 @@ impl<'tcx> EntryKind<'tcx> {
     }
 }
 
-/// Creates the "fake" DefPathTable for a given proc macro crate.
-///
-/// The DefPathTable is as follows:
-///
-/// CRATE_ROOT (DefIndex 0:0)
-///  |- GlobalMetaDataKind data (DefIndex 1:0 .. DefIndex 1:N)
-///  |- proc macro #0 (DefIndex 1:N)
-///  |- proc macro #1 (DefIndex 1:N+1)
-///  \- ...
-crate fn proc_macro_def_path_table(crate_root: &CrateRoot<'_>,
-                                   proc_macros: &[(ast::Name, Lrc<SyntaxExtension>)])
-                                   -> DefPathTable
-{
-    let mut definitions = Definitions::default();
-
-    let name = crate_root.name.as_str();
-    let disambiguator = crate_root.disambiguator;
-    debug!("creating proc macro def path table for {:?}/{:?}", name, disambiguator);
-    let crate_root = definitions.create_root_def(&name, disambiguator);
-    for (index, (name, _)) in proc_macros.iter().enumerate() {
-        let def_index = definitions.create_def_with_parent(
-            crate_root,
-            ast::DUMMY_NODE_ID,
-            DefPathData::MacroNs(name.as_interned_str()),
-            ExpnId::root(),
-            DUMMY_SP);
-        debug!("definition for {:?} is {:?}", name, def_index);
-        assert_eq!(def_index, DefIndex::from_proc_macro_index(index));
-    }
-
-    definitions.def_path_table().clone()
-}
-
 impl<'a, 'tcx> CrateMetadata {
+    pub fn is_proc_macro_crate(&self) -> bool {
+        self.root.proc_macro_decls_static.is_some()
+    }
     fn is_proc_macro(&self, id: DefIndex) -> bool {
-        self.proc_macros.is_some() && id != CRATE_DEF_INDEX
+        self.is_proc_macro_crate() &&
+            self.root.proc_macro_data.unwrap().decode(self).find(|x| *x == id).is_some()
     }
 
     fn maybe_entry(&self, item_id: DefIndex) -> Option<Lazy<Entry<'tcx>>> {
-        assert!(!self.is_proc_macro(item_id));
         self.root.entries_index.lookup(self.blob.raw_bytes(), item_id)
     }
 
@@ -504,13 +474,24 @@ impl<'a, 'tcx> CrateMetadata {
         }
     }
 
+    fn raw_proc_macro(&self, id: DefIndex) -> &ProcMacro {
+        // DefIndex's in root.proc_macro_data have a one-to-one correspondence
+        // with items in 'raw_proc_macros'
+        let pos = self.root.proc_macro_data.unwrap().decode(self).position(|i| i == id).unwrap();
+        &self.raw_proc_macros.unwrap()[pos]
+    }
+
     pub fn item_name(&self, item_index: DefIndex) -> Symbol {
-        self.def_key(item_index)
-            .disambiguated_data
-            .data
-            .get_opt_name()
-            .expect("no name in item_name")
-            .as_symbol()
+        if !self.is_proc_macro(item_index) {
+            self.def_key(item_index)
+                .disambiguated_data
+                .data
+                .get_opt_name()
+                .expect("no name in item_name")
+                .as_symbol()
+        } else {
+            Symbol::intern(self.raw_proc_macro(item_index).name())
+        }
     }
 
     pub fn def_kind(&self, index: DefIndex) -> Option<DefKind> {
@@ -518,15 +499,64 @@ impl<'a, 'tcx> CrateMetadata {
             self.entry(index).kind.def_kind()
         } else {
             Some(DefKind::Macro(
-                self.proc_macros.as_ref().unwrap()[index.to_proc_macro_index()].1.macro_kind()
+                macro_kind(self.raw_proc_macro(index))
             ))
         }
     }
 
     pub fn get_span(&self, index: DefIndex, sess: &Session) -> Span {
-        match self.is_proc_macro(index) {
-            true => DUMMY_SP,
-            false => self.entry(index).span.decode((self, sess)),
+        self.entry(index).span.decode((self, sess))
+    }
+
+
+    pub fn get_proc_macro(&self, id: DefIndex, sess: &Session) -> FullProcMacro {
+        if sess.opts.debugging_opts.dual_proc_macros {
+            let host_lib = self.host_lib.as_ref().unwrap();
+            self.load_proc_macro(
+                &host_lib.metadata.get_root(),
+                id,
+                sess
+            )
+        } else {
+            self.load_proc_macro(&self.root, id, sess)
+        }
+    }
+
+    fn load_proc_macro(&self, root: &CrateRoot<'_>,
+                        id: DefIndex,
+                        sess: &Session)
+                        -> FullProcMacro {
+
+        let raw_macro = self.raw_proc_macro(id);
+        let (name, kind, helper_attrs) = match *raw_macro {
+            ProcMacro::CustomDerive { trait_name, attributes, client } => {
+                let helper_attrs =
+                    attributes.iter().cloned().map(Symbol::intern).collect::<Vec<_>>();
+                (
+                    trait_name,
+                    SyntaxExtensionKind::Derive(Box::new(ProcMacroDerive {
+                        client, attrs: helper_attrs.clone()
+                    })),
+                    helper_attrs,
+                )
+            }
+            ProcMacro::Attr { name, client } => (
+                name, SyntaxExtensionKind::Attr(Box::new(AttrProcMacro { client })), Vec::new()
+            ),
+            ProcMacro::Bang { name, client } => (
+                name, SyntaxExtensionKind::Bang(Box::new(BangProcMacro { client })), Vec::new()
+            )
+        };
+
+        let span = self.get_span(id, sess);
+
+        FullProcMacro {
+            name: Symbol::intern(name),
+            ext: Lrc::new(SyntaxExtension {
+                span,
+                helper_attrs,
+                ..SyntaxExtension::default(kind, root.edition)
+            })
         }
     }
 
@@ -723,7 +753,7 @@ impl<'a, 'tcx> CrateMetadata {
 
     /// Iterates over the language items in the given crate.
     pub fn get_lang_items(&self, tcx: TyCtxt<'tcx>) -> &'tcx [(DefId, usize)] {
-        if self.proc_macros.is_some() {
+        if self.is_proc_macro_crate() {
             // Proc macro crates do not export any lang-items to the target.
             &[]
         } else {
@@ -738,18 +768,18 @@ impl<'a, 'tcx> CrateMetadata {
     pub fn each_child_of_item<F>(&self, id: DefIndex, mut callback: F, sess: &Session)
         where F: FnMut(def::Export<hir::HirId>)
     {
-        if let Some(ref proc_macros) = self.proc_macros {
+        if let Some(proc_macros_ids) = self.root.proc_macro_data.map(|d| d.decode(self)) {
             /* If we are loading as a proc macro, we want to return the view of this crate
-             * as a proc macro crate, not as a Rust crate. See `proc_macro_def_path_table`
-             * for the DefPathTable we are corresponding to.
+             * as a proc macro crate.
              */
             if id == CRATE_DEF_INDEX {
-                for (id, &(name, ref ext)) in proc_macros.iter().enumerate() {
+                for def_index in proc_macros_ids {
+                    let raw_macro = self.raw_proc_macro(def_index);
                     let res = Res::Def(
-                        DefKind::Macro(ext.macro_kind()),
-                        self.local_def_id(DefIndex::from_proc_macro_index(id)),
+                        DefKind::Macro(macro_kind(raw_macro)),
+                        self.local_def_id(def_index),
                     );
-                    let ident = Ident::with_dummy_span(name);
+                    let ident = Ident::from_str(raw_macro.name());
                     callback(def::Export {
                         ident: ident,
                         res: res,
@@ -960,11 +990,8 @@ impl<'a, 'tcx> CrateMetadata {
         }
     }
 
-    pub fn get_item_attrs(&self, node_id: DefIndex, sess: &Session) -> Lrc<[ast::Attribute]> {
-        if self.is_proc_macro(node_id) {
-            return Lrc::new([]);
-        }
 
+    pub fn get_item_attrs(&self, node_id: DefIndex, sess: &Session) -> Lrc<[ast::Attribute]> {
         // The attributes for a tuple struct/variant are attached to the definition, not the ctor;
         // we assume that someone passing in a tuple struct ctor is actually wanting to
         // look at the definition
@@ -1022,7 +1049,7 @@ impl<'a, 'tcx> CrateMetadata {
         tcx: TyCtxt<'tcx>,
         filter: Option<DefId>,
     ) -> &'tcx [DefId] {
-        if self.proc_macros.is_some() {
+        if self.is_proc_macro_crate() {
             // proc-macro crates export no trait impls.
             return &[]
         }
@@ -1066,7 +1093,7 @@ impl<'a, 'tcx> CrateMetadata {
 
 
     pub fn get_native_libraries(&self, sess: &Session) -> Vec<NativeLibrary> {
-        if self.proc_macros.is_some() {
+        if self.is_proc_macro_crate() {
             // Proc macro crates do not have any *target* native libraries.
             vec![]
         } else {
@@ -1075,7 +1102,7 @@ impl<'a, 'tcx> CrateMetadata {
     }
 
     pub fn get_foreign_modules(&self, tcx: TyCtxt<'tcx>) -> &'tcx [ForeignModule] {
-        if self.proc_macros.is_some() {
+        if self.is_proc_macro_crate() {
             // Proc macro crates do not have any *target* foreign modules.
             &[]
         } else {
@@ -1098,7 +1125,7 @@ impl<'a, 'tcx> CrateMetadata {
     }
 
     pub fn get_missing_lang_items(&self, tcx: TyCtxt<'tcx>) -> &'tcx [lang_items::LangItem] {
-        if self.proc_macros.is_some() {
+        if self.is_proc_macro_crate() {
             // Proc macro crates do not depend on any target weak lang-items.
             &[]
         } else {
@@ -1122,7 +1149,7 @@ impl<'a, 'tcx> CrateMetadata {
         &self,
         tcx: TyCtxt<'tcx>,
     ) -> Vec<(ExportedSymbol<'tcx>, SymbolExportLevel)> {
-        if self.proc_macros.is_some() {
+        if self.is_proc_macro_crate() {
             // If this crate is a custom derive crate, then we're not even going to
             // link those in so we skip those crates.
             vec![]
@@ -1191,13 +1218,18 @@ impl<'a, 'tcx> CrateMetadata {
 
     #[inline]
     pub fn def_key(&self, index: DefIndex) -> DefKey {
-        self.def_path_table.def_key(index)
+        let mut key = self.def_path_table.def_key(index);
+        if self.is_proc_macro(index) {
+            let name = self.raw_proc_macro(index).name();
+            key.disambiguated_data.data = DefPathData::MacroNs(InternedString::intern(name));
+        }
+        key
     }
 
     // Returns the path leading to the thing with this `id`.
     pub fn def_path(&self, id: DefIndex) -> DefPath {
         debug!("def_path(cnum={:?}, id={:?})", self.cnum, id);
-        DefPath::make(self.cnum, id, |parent| self.def_path_table.def_key(parent))
+        DefPath::make(self.cnum, id, |parent| self.def_key(parent))
     }
 
     #[inline]
@@ -1310,3 +1342,13 @@ impl<'a, 'tcx> CrateMetadata {
         self.source_map_import_info.borrow()
     }
 }
+
+// Cannot be implemented on 'ProcMacro', as libproc_macro
+// does not depend on libsyntax
+fn macro_kind(raw: &ProcMacro) -> MacroKind {
+    match raw {
+        ProcMacro::CustomDerive { .. } => MacroKind::Derive,
+        ProcMacro::Attr { .. } => MacroKind::Attr,
+        ProcMacro::Bang { .. } => MacroKind::Bang
+    }
+}
diff --git a/src/librustc_metadata/encoder.rs b/src/librustc_metadata/encoder.rs
index e2de0552cd6..6058ae99cf4 100644
--- a/src/librustc_metadata/encoder.rs
+++ b/src/librustc_metadata/encoder.rs
@@ -30,6 +30,7 @@ use rustc_data_structures::sync::Lrc;
 use std::u32;
 use syntax::ast;
 use syntax::attr;
+use syntax::ext::proc_macro::is_proc_macro_attr;
 use syntax::source_map::Spanned;
 use syntax::symbol::{kw, sym, Ident};
 use syntax_pos::{self, FileName, SourceFile, Span};
@@ -383,6 +384,8 @@ impl<'tcx> EncodeContext<'tcx> {
     }
 
     fn encode_crate_root(&mut self) -> Lazy<CrateRoot<'tcx>> {
+        let is_proc_macro = self.tcx.sess.crate_types.borrow().contains(&CrateType::ProcMacro);
+
         let mut i = self.position();
 
         let crate_deps = self.encode_crate_deps();
@@ -463,16 +466,23 @@ impl<'tcx> EncodeContext<'tcx> {
             self.lazy_seq(interpret_alloc_index)
         };
 
+
         i = self.position();
         let entries_index = self.entries_index.write_index(&mut self.opaque);
         let entries_index_bytes = self.position() - i;
 
+        // Encode the proc macro data
+        i = self.position();
+        let proc_macro_data = self.encode_proc_macros();
+        let proc_macro_data_bytes = self.position() - i;
+
+
         let attrs = tcx.hir().krate_attrs();
-        let is_proc_macro = tcx.sess.crate_types.borrow().contains(&CrateType::ProcMacro);
         let has_default_lib_allocator = attr::contains_name(&attrs, sym::default_lib_allocator);
         let has_global_allocator = *tcx.sess.has_global_allocator.get();
         let has_panic_handler = *tcx.sess.has_panic_handler.try_get().unwrap_or(&false);
 
+
         let root = self.lazy(&CrateRoot {
             name: tcx.crate_name(LOCAL_CRATE),
             extra_filename: tcx.sess.opts.cg.extra_filename.clone(),
@@ -491,6 +501,7 @@ impl<'tcx> EncodeContext<'tcx> {
             } else {
                 None
             },
+            proc_macro_data,
             proc_macro_stability: if is_proc_macro {
                 tcx.lookup_stability(DefId::local(CRATE_DEF_INDEX)).map(|stab| stab.clone())
             } else {
@@ -539,6 +550,7 @@ impl<'tcx> EncodeContext<'tcx> {
             println!("            impl bytes: {}", impl_bytes);
             println!("    exp. symbols bytes: {}", exported_symbols_bytes);
             println!("  def-path table bytes: {}", def_path_table_bytes);
+            println!(" proc-macro-data-bytes: {}", proc_macro_data_bytes);
             println!("            item bytes: {}", item_bytes);
             println!("   entries index bytes: {}", entries_index_bytes);
             println!("            zero bytes: {}", zero_bytes);
@@ -1470,6 +1482,22 @@ impl EncodeContext<'tcx> {
         self.lazy_seq(foreign_modules.iter().cloned())
     }
 
+    fn encode_proc_macros(&mut self) -> Option<LazySeq<DefIndex>> {
+        let is_proc_macro = self.tcx.sess.crate_types.borrow().contains(&CrateType::ProcMacro);
+        if is_proc_macro {
+            let proc_macros: Vec<_> = self.tcx.hir().krate().items.values().filter_map(|item| {
+                if item.attrs.iter().any(|attr| is_proc_macro_attr(attr)) {
+                    Some(item.hir_id.owner)
+                } else {
+                    None
+                }
+            }).collect();
+            Some(self.lazy_seq(proc_macros))
+        } else {
+            None
+        }
+    }
+
     fn encode_crate_deps(&mut self) -> LazySeq<CrateDep> {
         let crates = self.tcx.crates();
 
diff --git a/src/librustc_metadata/locator.rs b/src/librustc_metadata/locator.rs
index 3832c8ee227..ceba7cf0fe0 100644
--- a/src/librustc_metadata/locator.rs
+++ b/src/librustc_metadata/locator.rs
@@ -716,7 +716,9 @@ impl<'a> Context<'a> {
 
         let root = metadata.get_root();
         if let Some(is_proc_macro) = self.is_proc_macro {
-            if root.proc_macro_decls_static.is_some() != is_proc_macro {
+            if root.proc_macro_data.is_some() != is_proc_macro {
+                info!("Rejecting via proc macro: expected {} got {}",
+                      is_proc_macro, root.proc_macro_data.is_some());
                 return None;
             }
         }
diff --git a/src/librustc_metadata/schema.rs b/src/librustc_metadata/schema.rs
index c0ac6915933..13c599cf997 100644
--- a/src/librustc_metadata/schema.rs
+++ b/src/librustc_metadata/schema.rs
@@ -182,6 +182,10 @@ pub struct CrateRoot<'tcx> {
 
     pub entries_index: LazySeq<index::Index<'tcx>>,
 
+    /// The DefIndex's of any proc macros delcared by
+    /// this crate
+    pub proc_macro_data: Option<LazySeq<DefIndex>>,
+
     pub compiler_builtins: bool,
     pub needs_allocator: bool,
     pub needs_panic_runtime: bool,
diff --git a/src/test/ui/macros/same-sequence-span.stderr b/src/test/ui/macros/same-sequence-span.stderr
index aee1b4c9c5d..250773a1853 100644
--- a/src/test/ui/macros/same-sequence-span.stderr
+++ b/src/test/ui/macros/same-sequence-span.stderr
@@ -18,7 +18,10 @@ error: `$x:expr` may be followed by `$y:tt`, which is not allowed for `expr` fra
   --> $DIR/same-sequence-span.rs:20:1
    |
 LL | proc_macro_sequence::make_foo!();
-   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ not allowed after `expr` fragments
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   | |
+   | not allowed after `expr` fragments
+   | in this macro invocation
    |
    = note: allowed there are: `=>`, `,` or `;`
 
@@ -26,7 +29,10 @@ error: `$x:expr` may be followed by `=`, which is not allowed for `expr` fragmen
   --> $DIR/same-sequence-span.rs:20:1
    |
 LL | proc_macro_sequence::make_foo!();
-   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ not allowed after `expr` fragments
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   | |
+   | not allowed after `expr` fragments
+   | in this macro invocation
    |
    = note: allowed there are: `=>`, `,` or `;`
 
diff --git a/src/test/ui/proc-macro/generate-mod.stderr b/src/test/ui/proc-macro/generate-mod.stderr
index 51bbb23da75..829d8bf4c81 100644
--- a/src/test/ui/proc-macro/generate-mod.stderr
+++ b/src/test/ui/proc-macro/generate-mod.stderr
@@ -2,13 +2,19 @@ error[E0412]: cannot find type `FromOutside` in this scope
   --> $DIR/generate-mod.rs:9:1
    |
 LL | generate_mod::check!();
-   | ^^^^^^^^^^^^^^^^^^^^^^^ not found in this scope
+   | ^^^^^^^^^^^^^^^^^^^^^^^
+   | |
+   | not found in this scope
+   | in this macro invocation
 
 error[E0412]: cannot find type `Outer` in this scope
   --> $DIR/generate-mod.rs:9:1
    |
 LL | generate_mod::check!();
-   | ^^^^^^^^^^^^^^^^^^^^^^^ not found in this scope
+   | ^^^^^^^^^^^^^^^^^^^^^^^
+   | |
+   | not found in this scope
+   | in this macro invocation
 
 error[E0412]: cannot find type `FromOutside` in this scope
   --> $DIR/generate-mod.rs:12:1
diff --git a/src/test/ui/proc-macro/invalid-punct-ident-4.stderr b/src/test/ui/proc-macro/invalid-punct-ident-4.stderr
index da2bf07a1a3..65e40172ef5 100644
--- a/src/test/ui/proc-macro/invalid-punct-ident-4.stderr
+++ b/src/test/ui/proc-macro/invalid-punct-ident-4.stderr
@@ -2,7 +2,10 @@ error: unexpected close delimiter: `)`
   --> $DIR/invalid-punct-ident-4.rs:6:1
    |
 LL | lexer_failure!();
-   | ^^^^^^^^^^^^^^^^^ unexpected close delimiter
+   | ^^^^^^^^^^^^^^^^^
+   | |
+   | unexpected close delimiter
+   | in this macro invocation
 
 error: proc macro panicked
   --> $DIR/invalid-punct-ident-4.rs:6:1
diff --git a/src/test/ui/proc-macro/lints_in_proc_macros.stderr b/src/test/ui/proc-macro/lints_in_proc_macros.stderr
index 2d97cd700be..f28b8c9fb73 100644
--- a/src/test/ui/proc-macro/lints_in_proc_macros.stderr
+++ b/src/test/ui/proc-macro/lints_in_proc_macros.stderr
@@ -2,7 +2,10 @@ error[E0425]: cannot find value `foobar2` in this scope
   --> $DIR/lints_in_proc_macros.rs:12:5
    |
 LL |     bang_proc_macro2!();
-   |     ^^^^^^^^^^^^^^^^^^^^ help: a local variable with a similar name exists: `foobar`
+   |     ^^^^^^^^^^^^^^^^^^^^
+   |     |
+   |     help: a local variable with a similar name exists: `foobar`
+   |     in this macro invocation
 
 error: aborting due to previous error
 
diff --git a/src/test/ui/proc-macro/multispan.stderr b/src/test/ui/proc-macro/multispan.stderr
index 44af07a7942..a0c1f9cd5c0 100644
--- a/src/test/ui/proc-macro/multispan.stderr
+++ b/src/test/ui/proc-macro/multispan.stderr
@@ -2,7 +2,7 @@ error: hello to you, too!
   --> $DIR/multispan.rs:14:5
    |
 LL |     hello!(hi);
-   |     ^^^^^^^^^^^
+   |     ^^^^^^^^^^^ in this macro invocation
    |
 note: found these 'hi's
   --> $DIR/multispan.rs:14:12
@@ -14,7 +14,7 @@ error: hello to you, too!
   --> $DIR/multispan.rs:17:5
    |
 LL |     hello!(hi hi);
-   |     ^^^^^^^^^^^^^^
+   |     ^^^^^^^^^^^^^^ in this macro invocation
    |
 note: found these 'hi's
   --> $DIR/multispan.rs:17:12
@@ -26,7 +26,7 @@ error: hello to you, too!
   --> $DIR/multispan.rs:20:5
    |
 LL |     hello!(hi hi hi);
-   |     ^^^^^^^^^^^^^^^^^
+   |     ^^^^^^^^^^^^^^^^^ in this macro invocation
    |
 note: found these 'hi's
   --> $DIR/multispan.rs:20:12
@@ -38,7 +38,7 @@ error: hello to you, too!
   --> $DIR/multispan.rs:23:5
    |
 LL |     hello!(hi hey hi yo hi beep beep hi hi);
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ in this macro invocation
    |
 note: found these 'hi's
   --> $DIR/multispan.rs:23:12
@@ -50,7 +50,7 @@ error: hello to you, too!
   --> $DIR/multispan.rs:24:5
    |
 LL |     hello!(hi there, hi how are you? hi... hi.);
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ in this macro invocation
    |
 note: found these 'hi's
   --> $DIR/multispan.rs:24:12
@@ -62,7 +62,7 @@ error: hello to you, too!
   --> $DIR/multispan.rs:25:5
    |
 LL |     hello!(whoah. hi di hi di ho);
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ in this macro invocation
    |
 note: found these 'hi's
   --> $DIR/multispan.rs:25:19
@@ -74,7 +74,7 @@ error: hello to you, too!
   --> $DIR/multispan.rs:26:5
    |
 LL |     hello!(hi good hi and good bye);
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ in this macro invocation
    |
 note: found these 'hi's
   --> $DIR/multispan.rs:26:12
diff --git a/src/test/ui/proc-macro/subspan.stderr b/src/test/ui/proc-macro/subspan.stderr
index 5117dd6d32d..06715c197bc 100644
--- a/src/test/ui/proc-macro/subspan.stderr
+++ b/src/test/ui/proc-macro/subspan.stderr
@@ -2,7 +2,7 @@ error: found 'hi's
   --> $DIR/subspan.rs:11:1
    |
 LL | subspan!("hi");
-   | ^^^^^^^^^^^^^^^
+   | ^^^^^^^^^^^^^^^ in this macro invocation
    |
 note: here
   --> $DIR/subspan.rs:11:11
@@ -14,7 +14,7 @@ error: found 'hi's
   --> $DIR/subspan.rs:14:1
    |
 LL | subspan!("hihi");
-   | ^^^^^^^^^^^^^^^^^
+   | ^^^^^^^^^^^^^^^^^ in this macro invocation
    |
 note: here
   --> $DIR/subspan.rs:14:11
@@ -26,7 +26,7 @@ error: found 'hi's
   --> $DIR/subspan.rs:17:1
    |
 LL | subspan!("hihihi");
-   | ^^^^^^^^^^^^^^^^^^^
+   | ^^^^^^^^^^^^^^^^^^^ in this macro invocation
    |
 note: here
   --> $DIR/subspan.rs:17:11
@@ -38,7 +38,7 @@ error: found 'hi's
   --> $DIR/subspan.rs:20:1
    |
 LL | subspan!("why I hide? hi!");
-   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ in this macro invocation
    |
 note: here
   --> $DIR/subspan.rs:20:17
@@ -50,7 +50,7 @@ error: found 'hi's
   --> $DIR/subspan.rs:21:1
    |
 LL | subspan!("hey, hi, hidy, hidy, hi hi");
-   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ in this macro invocation
    |
 note: here
   --> $DIR/subspan.rs:21:16
@@ -62,7 +62,7 @@ error: found 'hi's
   --> $DIR/subspan.rs:22:1
    |
 LL | subspan!("this is a hi, and this is another hi");
-   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ in this macro invocation
    |
 note: here
   --> $DIR/subspan.rs:22:12
@@ -74,7 +74,7 @@ error: found 'hi's
   --> $DIR/subspan.rs:23:1
    |
 LL | subspan!("how are you this evening");
-   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ in this macro invocation
    |
 note: here
   --> $DIR/subspan.rs:23:24
@@ -86,7 +86,7 @@ error: found 'hi's
   --> $DIR/subspan.rs:24:1
    |
 LL | subspan!("this is highly eradic");
-   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ in this macro invocation
    |
 note: here
   --> $DIR/subspan.rs:24:12
diff --git a/src/test/ui/proc-macro/three-equals.stderr b/src/test/ui/proc-macro/three-equals.stderr
index f8dfa841d4f..0a6cbe13098 100644
--- a/src/test/ui/proc-macro/three-equals.stderr
+++ b/src/test/ui/proc-macro/three-equals.stderr
@@ -2,7 +2,7 @@ error: found 2 equal signs, need exactly 3
   --> $DIR/three-equals.rs:15:5
    |
 LL |     three_equals!(==);
-   |     ^^^^^^^^^^^^^^^^^^
+   |     ^^^^^^^^^^^^^^^^^^ in this macro invocation
    |
    = help: input must be: `===`