Auto merge of #34860 - jseyfried:encapsulate_hygiene, r=nrc

Clean up and encapsulate `syntax::ext::mtwt`, rename `mtwt` to `hygiene`

r? @nrc

3 files changed, 131 insertions, 183 deletions

diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index 3e9837a6995..18342f2e38c 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -9,20 +9,19 @@
 // except according to those terms.
 
 use ast::{Block, Crate, Ident, Mac_, Name, PatKind};
-use ast::{MacStmtStyle, Mrk, Stmt, StmtKind, ItemKind};
+use ast::{MacStmtStyle, Stmt, StmtKind, ItemKind};
 use ast;
-use attr::HasAttrs;
-use ext::mtwt;
-use attr;
+use ext::hygiene::Mark;
+use attr::{self, HasAttrs};
 use attr::AttrMetaMethods;
-use codemap::{dummy_spanned, Spanned, ExpnInfo, NameAndSpan, MacroBang, MacroAttribute};
+use codemap::{dummy_spanned, ExpnInfo, NameAndSpan, MacroBang, MacroAttribute};
 use syntax_pos::{self, Span, ExpnId};
 use config::StripUnconfigured;
 use ext::base::*;
 use feature_gate::{self, Features};
 use fold;
 use fold::*;
-use parse::token::{fresh_mark, intern, keywords};
+use parse::token::{intern, keywords};
 use ptr::P;
 use tokenstream::TokenTree;
 use util::small_vector::SmallVector;
@@ -130,9 +129,9 @@ fn expand_mac_invoc<T>(mac: ast::Mac, ident: Option<Ident>, attrs: Vec<ast::Attr
     // It would almost certainly be cleaner to pass the whole macro invocation in,
     // rather than pulling it apart and marking the tts and the ctxt separately.
     let Mac_ { path, tts, .. } = mac.node;
-    let mark = fresh_mark();
+    let mark = Mark::fresh();
 
-    fn mac_result<'a>(path: &ast::Path, ident: Option<Ident>, tts: Vec<TokenTree>, mark: Mrk,
+    fn mac_result<'a>(path: &ast::Path, ident: Option<Ident>, tts: Vec<TokenTree>, mark: Mark,
                       attrs: Vec<ast::Attribute>, call_site: Span, fld: &'a mut MacroExpander)
                       -> Option<Box<MacResult + 'a>> {
         // Detect use of feature-gated or invalid attributes on macro invoations
@@ -708,30 +707,17 @@ pub fn expand_crate(mut cx: ExtCtxt,
     return (ret, cx.syntax_env.names);
 }
 
-// HYGIENIC CONTEXT EXTENSION:
-// all of these functions are for walking over
-// ASTs and making some change to the context of every
-// element that has one. a CtxtFn is a trait-ified
-// version of a closure in (SyntaxContext -> SyntaxContext).
-// the ones defined here include:
-// Marker - add a mark to a context
-
 // A Marker adds the given mark to the syntax context and
 // sets spans' `expn_id` to the given expn_id (unless it is `None`).
-struct Marker { mark: Mrk, expn_id: Option<ExpnId> }
+struct Marker { mark: Mark, expn_id: Option<ExpnId> }
 
 impl Folder for Marker {
-    fn fold_ident(&mut self, id: Ident) -> Ident {
-        ast::Ident::new(id.name, mtwt::apply_mark(self.mark, id.ctxt))
-    }
-    fn fold_mac(&mut self, Spanned {node, span}: ast::Mac) -> ast::Mac {
-        Spanned {
-            node: Mac_ {
-                path: self.fold_path(node.path),
-                tts: self.fold_tts(&node.tts),
-            },
-            span: self.new_span(span),
-        }
+    fn fold_ident(&mut self, mut ident: Ident) -> Ident {
+        ident.ctxt = ident.ctxt.apply_mark(self.mark);
+        ident
+    }
+    fn fold_mac(&mut self, mac: ast::Mac) -> ast::Mac {
+        noop_fold_mac(mac, self)
     }
 
     fn new_span(&mut self, mut span: Span) -> Span {
@@ -743,7 +729,7 @@ impl Folder for Marker {
 }
 
 // apply a given mark to the given token trees. Used prior to expansion of a macro.
-fn mark_tts(tts: &[TokenTree], m: Mrk) -> Vec<TokenTree> {
+fn mark_tts(tts: &[TokenTree], m: Mark) -> Vec<TokenTree> {
     noop_fold_tts(tts, &mut Marker{mark:m, expn_id: None})
 }
 
diff --git a/src/libsyntax/ext/hygiene.rs b/src/libsyntax/ext/hygiene.rs
new file mode 100644
index 00000000000..ade165e0ef9
--- /dev/null
+++ b/src/libsyntax/ext/hygiene.rs
@@ -0,0 +1,116 @@
+// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+//! Machinery for hygienic macros, inspired by the MTWT[1] paper.
+//!
+//! [1] Matthew Flatt, Ryan Culpepper, David Darais, and Robert Bruce Findler.
+//! 2012. *Macros that work together: Compile-time bindings, partial expansion,
+//! and definition contexts*. J. Funct. Program. 22, 2 (March 2012), 181-216.
+//! DOI=10.1017/S0956796812000093 http://dx.doi.org/10.1017/S0956796812000093
+
+use std::cell::RefCell;
+use std::collections::HashMap;
+use std::fmt;
+
+/// A SyntaxContext represents a chain of macro expansions (represented by marks).
+#[derive(Clone, Copy, PartialEq, Eq, Hash, RustcEncodable, RustcDecodable, Default)]
+pub struct SyntaxContext(u32);
+
+#[derive(Copy, Clone)]
+pub struct SyntaxContextData {
+    pub outer_mark: Mark,
+    pub prev_ctxt: SyntaxContext,
+}
+
+/// A mark represents a unique id associated with a macro expansion.
+#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, Default)]
+pub struct Mark(u32);
+
+impl Mark {
+    pub fn fresh() -> Self {
+        HygieneData::with(|data| {
+            let next_mark = Mark(data.next_mark.0 + 1);
+            ::std::mem::replace(&mut data.next_mark, next_mark)
+        })
+    }
+}
+
+struct HygieneData {
+    syntax_contexts: Vec<SyntaxContextData>,
+    markings: HashMap<(SyntaxContext, Mark), SyntaxContext>,
+    next_mark: Mark,
+}
+
+impl HygieneData {
+    fn new() -> Self {
+        HygieneData {
+            syntax_contexts: vec![SyntaxContextData {
+                outer_mark: Mark(0), // the null mark
+                prev_ctxt: SyntaxContext(0), // the empty context
+            }],
+            markings: HashMap::new(),
+            next_mark: Mark(1),
+        }
+    }
+
+    fn with<T, F: FnOnce(&mut HygieneData) -> T>(f: F) -> T {
+        thread_local! {
+            static HYGIENE_DATA: RefCell<HygieneData> = RefCell::new(HygieneData::new());
+        }
+        HYGIENE_DATA.with(|data| f(&mut *data.borrow_mut()))
+    }
+}
+
+pub fn reset_hygiene_data() {
+    HygieneData::with(|data| *data = HygieneData::new())
+}
+
+impl SyntaxContext {
+    pub const fn empty() -> Self {
+        SyntaxContext(0)
+    }
+
+    pub fn data(self) -> SyntaxContextData {
+        HygieneData::with(|data| data.syntax_contexts[self.0 as usize])
+    }
+
+    /// Extend a syntax context with a given mark
+    pub fn apply_mark(self, mark: Mark) -> SyntaxContext {
+        // Applying the same mark twice is a no-op
+        let ctxt_data = self.data();
+        if mark == ctxt_data.outer_mark {
+            return ctxt_data.prev_ctxt;
+        }
+
+        HygieneData::with(|data| {
+            let syntax_contexts = &mut data.syntax_contexts;
+            *data.markings.entry((self, mark)).or_insert_with(|| {
+                syntax_contexts.push(SyntaxContextData {
+                    outer_mark: mark,
+                    prev_ctxt: self,
+                });
+                SyntaxContext(syntax_contexts.len() as u32 - 1)
+            })
+        })
+    }
+
+   /// If `ident` is macro expanded, return the source ident from the macro definition
+   /// and the mark of the expansion that created the macro definition.
+   pub fn source(self) -> (Self /* source context */, Mark /* source macro */) {
+        let macro_def_ctxt = self.data().prev_ctxt.data();
+        (macro_def_ctxt.prev_ctxt, macro_def_ctxt.outer_mark)
+   }
+}
+
+impl fmt::Debug for SyntaxContext {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "#{}", self.0)
+    }
+}
diff --git a/src/libsyntax/ext/mtwt.rs b/src/libsyntax/ext/mtwt.rs
deleted file mode 100644
index d2f6df9d5db..00000000000
--- a/src/libsyntax/ext/mtwt.rs
+++ /dev/null
@@ -1,154 +0,0 @@
-// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
-// file at the top-level directory of this distribution and at
-// http://rust-lang.org/COPYRIGHT.
-//
-// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
-// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
-// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
-// option. This file may not be copied, modified, or distributed
-// except according to those terms.
-
-//! Machinery for hygienic macros, as described in the MTWT[1] paper.
-//!
-//! [1] Matthew Flatt, Ryan Culpepper, David Darais, and Robert Bruce Findler.
-//! 2012. *Macros that work together: Compile-time bindings, partial expansion,
-//! and definition contexts*. J. Funct. Program. 22, 2 (March 2012), 181-216.
-//! DOI=10.1017/S0956796812000093 http://dx.doi.org/10.1017/S0956796812000093
-
-pub use self::SyntaxContext_::*;
-
-use ast::{Ident, Mrk, SyntaxContext};
-
-use std::cell::RefCell;
-use std::collections::HashMap;
-
-/// The SCTable contains a table of SyntaxContext_'s. It
-/// represents a flattened tree structure, to avoid having
-/// managed pointers everywhere (that caused an ICE).
-/// The `marks` ensures that adding the same mark to the
-/// same context gives you back the same context as before.
-pub struct SCTable {
-    table: RefCell<Vec<SyntaxContext_>>,
-    marks: RefCell<HashMap<(SyntaxContext,Mrk),SyntaxContext>>,
-}
-
-#[derive(PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy, Clone)]
-pub enum SyntaxContext_ {
-    EmptyCtxt,
-    Mark (Mrk,SyntaxContext),
-}
-
-/// Extend a syntax context with a given mark
-pub fn apply_mark(m: Mrk, ctxt: SyntaxContext) -> SyntaxContext {
-    with_sctable(|table| apply_mark_internal(m, ctxt, table))
-}
-
-/// Extend a syntax context with a given mark and sctable (explicit memoization)
-fn apply_mark_internal(m: Mrk, ctxt: SyntaxContext, table: &SCTable) -> SyntaxContext {
-    let ctxts = &mut *table.table.borrow_mut();
-    match ctxts[ctxt.0 as usize] {
-        // Applying the same mark twice is a no-op.
-        Mark(outer_mark, prev_ctxt) if outer_mark == m => return prev_ctxt,
-        _ => *table.marks.borrow_mut().entry((ctxt, m)).or_insert_with(|| {
-            SyntaxContext(idx_push(ctxts, Mark(m, ctxt)))
-        }),
-    }
-}
-
-/// Fetch the SCTable from TLS, create one if it doesn't yet exist.
-pub fn with_sctable<T, F>(op: F) -> T where
-    F: FnOnce(&SCTable) -> T,
-{
-    thread_local!(static SCTABLE_KEY: SCTable = new_sctable_internal());
-    SCTABLE_KEY.with(move |slot| op(slot))
-}
-
-// Make a fresh syntax context table with EmptyCtxt in slot zero.
-fn new_sctable_internal() -> SCTable {
-    SCTable {
-        table: RefCell::new(vec![EmptyCtxt]),
-        marks: RefCell::new(HashMap::new()),
-    }
-}
-
-/// Clear the tables from TLD to reclaim memory.
-pub fn clear_tables() {
-    with_sctable(|table| {
-        *table.table.borrow_mut() = Vec::new();
-        *table.marks.borrow_mut() = HashMap::new();
-    });
-}
-
-/// Reset the tables to their initial state
-pub fn reset_tables() {
-    with_sctable(|table| {
-        *table.table.borrow_mut() = vec![EmptyCtxt];
-        *table.marks.borrow_mut() = HashMap::new();
-    });
-}
-
-/// Add a value to the end of a vec, return its index
-fn idx_push<T>(vec: &mut Vec<T>, val: T) -> u32 {
-    vec.push(val);
-    (vec.len() - 1) as u32
-}
-
-/// Return the outer mark for a context with a mark at the outside.
-/// FAILS when outside is not a mark.
-pub fn outer_mark(ctxt: SyntaxContext) -> Mrk {
-    with_sctable(|sctable| {
-        match (*sctable.table.borrow())[ctxt.0 as usize] {
-            Mark(mrk, _) => mrk,
-            _ => panic!("can't retrieve outer mark when outside is not a mark")
-        }
-    })
-}
-
-/// If `ident` is macro expanded, return the source ident from the macro definition
-/// and the mark of the expansion that created the macro definition.
-pub fn source(ident: Ident) -> Option<(Ident /* source ident */, Mrk /* source macro */)> {
-    with_sctable(|sctable| {
-        let ctxts = sctable.table.borrow();
-        if let Mark(_expansion_mark, macro_ctxt) = ctxts[ident.ctxt.0 as usize] {
-            if let Mark(definition_mark, orig_ctxt) = ctxts[macro_ctxt.0 as usize] {
-                return Some((Ident::new(ident.name, orig_ctxt), definition_mark));
-            }
-        }
-        None
-    })
-}
-
-#[cfg(test)]
-mod tests {
-    use ast::{EMPTY_CTXT, Mrk, SyntaxContext};
-    use super::{apply_mark_internal, new_sctable_internal, Mark, SCTable};
-
-    // extend a syntax context with a sequence of marks given
-    // in a vector. v[0] will be the outermost mark.
-    fn unfold_marks(mrks: Vec<Mrk> , tail: SyntaxContext, table: &SCTable)
-                    -> SyntaxContext {
-        mrks.iter().rev().fold(tail, |tail:SyntaxContext, mrk:&Mrk|
-                   {apply_mark_internal(*mrk,tail,table)})
-    }
-
-    #[test] fn unfold_marks_test() {
-        let mut t = new_sctable_internal();
-
-        assert_eq!(unfold_marks(vec!(3,7),EMPTY_CTXT,&mut t),SyntaxContext(2));
-        {
-            let table = t.table.borrow();
-            assert!((*table)[1] == Mark(7,EMPTY_CTXT));
-            assert!((*table)[2] == Mark(3,SyntaxContext(1)));
-        }
-    }
-
-    #[test]
-    fn hashing_tests () {
-        let mut t = new_sctable_internal();
-        assert_eq!(apply_mark_internal(12,EMPTY_CTXT,&mut t),SyntaxContext(1));
-        assert_eq!(apply_mark_internal(13,EMPTY_CTXT,&mut t),SyntaxContext(2));
-        // using the same one again should result in the same index:
-        assert_eq!(apply_mark_internal(12,EMPTY_CTXT,&mut t),SyntaxContext(1));
-        // I'm assuming that the rename table will behave the same....
-    }
-}