Merge branch 'master' into issue-30961

14 files changed, 246 insertions, 1081 deletions

diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs
index 6b662c6779a..a8bb255fba4 100644
--- a/src/libsyntax/ast.rs
+++ b/src/libsyntax/ast.rs
@@ -19,6 +19,7 @@ pub use util::ThinVec;
 use syntax_pos::{mk_sp, Span, DUMMY_SP, ExpnId};
 use codemap::{respan, Spanned};
 use abi::Abi;
+use ext::hygiene::SyntaxContext;
 use parse::token::{self, keywords, InternedString};
 use print::pprust;
 use ptr::P;
@@ -33,15 +34,6 @@ use serialize::{Encodable, Decodable, Encoder, Decoder};
 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub struct Name(pub u32);
 
-/// A SyntaxContext represents a chain of macro-expandings
-/// and renamings. Each macro expansion corresponds to
-/// a fresh u32. This u32 is a reference to a table stored
-/// in thread-local storage.
-/// The special value EMPTY_CTXT is used to indicate an empty
-/// syntax context.
-#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, RustcEncodable, RustcDecodable)]
-pub struct SyntaxContext(pub u32);
-
 /// An identifier contains a Name (index into the interner
 /// table) and a SyntaxContext to track renaming and
 /// macro expansion per Flatt et al., "Macros That Work Together"
@@ -81,20 +73,15 @@ impl Decodable for Name {
     }
 }
 
-pub const EMPTY_CTXT : SyntaxContext = SyntaxContext(0);
-
 impl Ident {
-    pub fn new(name: Name, ctxt: SyntaxContext) -> Ident {
-        Ident {name: name, ctxt: ctxt}
-    }
     pub const fn with_empty_ctxt(name: Name) -> Ident {
-        Ident {name: name, ctxt: EMPTY_CTXT}
+        Ident { name: name, ctxt: SyntaxContext::empty() }
     }
 }
 
 impl fmt::Debug for Ident {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "{}#{}", self.name, self.ctxt.0)
+        write!(f, "{}{:?}", self.name, self.ctxt)
     }
 }
 
@@ -116,9 +103,6 @@ impl Decodable for Ident {
     }
 }
 
-/// A mark represents a unique id associated with a macro expansion
-pub type Mrk = u32;
-
 #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Copy)]
 pub struct Lifetime {
     pub id: NodeId,
diff --git a/src/libsyntax/codemap.rs b/src/libsyntax/codemap.rs
index 743f96d737e..a8aca90e623 100644
--- a/src/libsyntax/codemap.rs
+++ b/src/libsyntax/codemap.rs
@@ -827,12 +827,6 @@ impl CodeMapper for CodeMap {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use errors::{Level, CodeSuggestion};
-    use errors::emitter::EmitterWriter;
-    use errors::snippet::{SnippetData, RenderedLine, FormatMode};
-    use std::sync::{Arc, Mutex};
-    use std::io::{self, Write};
-    use std::str::from_utf8;
     use std::rc::Rc;
 
     #[test]
@@ -1122,24 +1116,6 @@ mod tests {
         }
     }
 
-    fn splice(start: Span, end: Span) -> Span {
-        Span {
-            lo: start.lo,
-            hi: end.hi,
-            expn_id: NO_EXPANSION,
-        }
-    }
-
-    fn make_string(lines: &[RenderedLine]) -> String {
-        lines.iter()
-            .flat_map(|rl| {
-                rl.text.iter()
-                        .map(|s| &s.text[..])
-                        .chain(Some("\n"))
-            })
-            .collect()
-    }
-
     fn init_expansion_chain(cm: &CodeMap) -> Span {
         // Creates an expansion chain containing two recursive calls
         // root -> expA -> expA -> expB -> expB -> end
@@ -1219,761 +1195,4 @@ r"blork2.rs:2:1: 2:12
 ";
         assert_eq!(sstr, res_str);
     }
-
-    struct Sink(Arc<Mutex<Vec<u8>>>);
-    impl Write for Sink {
-        fn write(&mut self, data: &[u8]) -> io::Result<usize> {
-            Write::write(&mut *self.0.lock().unwrap(), data)
-        }
-        fn flush(&mut self) -> io::Result<()> { Ok(()) }
-    }
-
-    // Diagnostic doesn't align properly in span where line number increases by one digit
-    #[test]
-    fn test_hilight_suggestion_issue_11715() {
-        let data = Arc::new(Mutex::new(Vec::new()));
-        let cm = Rc::new(CodeMap::new());
-        let mut ew = EmitterWriter::new(Box::new(Sink(data.clone())),
-                                        None,
-                                        cm.clone(),
-                                        FormatMode::NewErrorFormat);
-        let content = "abcdefg
-        koksi
-        line3
-        line4
-        cinq
-        line6
-        line7
-        line8
-        line9
-        line10
-        e-lä-vän
-        tolv
-        dreizehn
-        ";
-        let file = cm.new_filemap_and_lines("dummy.txt", None, content);
-        let start = file.lines.borrow()[10];
-        let end = file.lines.borrow()[11];
-        let sp = mk_sp(start, end);
-        let lvl = Level::Error;
-        println!("highlight_lines");
-        ew.highlight_lines(&sp.into(), lvl).unwrap();
-        println!("done");
-        let vec = data.lock().unwrap().clone();
-        let vec: &[u8] = &vec;
-        let str = from_utf8(vec).unwrap();
-        println!("r#\"\n{}\"#", str);
-        assert_eq!(str, &r#"
-  --> dummy.txt:11:1
-   |>
-11 |>         e-lä-vän
-   |> ^
-"#[1..]);
-    }
-
-    #[test]
-    fn test_single_span_splice() {
-        // Test that a `MultiSpan` containing a single span splices a substition correctly
-        let cm = CodeMap::new();
-        let inputtext = "aaaaa\nbbbbBB\nCCC\nDDDDDddddd\neee\n";
-        let selection = "     \n    ~~\n~~~\n~~~~~     \n   \n";
-        cm.new_filemap_and_lines("blork.rs", None, inputtext);
-        let sp = span_from_selection(inputtext, selection);
-        let msp: MultiSpan = sp.into();
-
-        // check that we are extracting the text we thought we were extracting
-        assert_eq!(&cm.span_to_snippet(sp).unwrap(), "BB\nCCC\nDDDDD");
-
-        let substitute = "ZZZZZZ".to_owned();
-        let expected = "bbbbZZZZZZddddd";
-        let suggest = CodeSuggestion {
-            msp: msp,
-            substitutes: vec![substitute],
-        };
-        assert_eq!(suggest.splice_lines(&cm), expected);
-    }
-
-    #[test]
-    fn test_multi_span_splice() {
-        // Test that a `MultiSpan` containing multiple spans splices a substition correctly
-        let cm = CodeMap::new();
-        let inputtext  = "aaaaa\nbbbbBB\nCCC\nDDDDDddddd\neee\n";
-        let selection1 = "     \n      \n   \n          \n ~ \n"; // intentionally out of order
-        let selection2 = "     \n    ~~\n~~~\n~~~~~     \n   \n";
-        cm.new_filemap_and_lines("blork.rs", None, inputtext);
-        let sp1 = span_from_selection(inputtext, selection1);
-        let sp2 = span_from_selection(inputtext, selection2);
-        let msp: MultiSpan = MultiSpan::from_spans(vec![sp1, sp2]);
-
-        let expected = "bbbbZZZZZZddddd\neXYZe";
-        let suggest = CodeSuggestion {
-            msp: msp,
-            substitutes: vec!["ZZZZZZ".to_owned(),
-                              "XYZ".to_owned()]
-        };
-
-        assert_eq!(suggest.splice_lines(&cm), expected);
-    }
-
-    #[test]
-    fn test_multispan_highlight() {
-        let data = Arc::new(Mutex::new(Vec::new()));
-        let cm = Rc::new(CodeMap::new());
-        let mut diag = EmitterWriter::new(Box::new(Sink(data.clone())),
-                                          None,
-                                          cm.clone(),
-                                          FormatMode::NewErrorFormat);
-
-        let inp =       "_____aaaaaa____bbbbbb__cccccdd_";
-        let sp1 =       "     ~~~~~~                    ";
-        let sp2 =       "               ~~~~~~          ";
-        let sp3 =       "                       ~~~~~   ";
-        let sp4 =       "                          ~~~~ ";
-        let sp34 =      "                       ~~~~~~~ ";
-
-        let expect_start = &r#"
- --> dummy.txt:1:6
-  |>
-1 |> _____aaaaaa____bbbbbb__cccccdd_
-  |>      ^^^^^^    ^^^^^^  ^^^^^^^
-"#[1..];
-
-        let span = |sp, expected| {
-            let sp = span_from_selection(inp, sp);
-            assert_eq!(&cm.span_to_snippet(sp).unwrap(), expected);
-            sp
-        };
-        cm.new_filemap_and_lines("dummy.txt", None, inp);
-        let sp1 = span(sp1, "aaaaaa");
-        let sp2 = span(sp2, "bbbbbb");
-        let sp3 = span(sp3, "ccccc");
-        let sp4 = span(sp4, "ccdd");
-        let sp34 = span(sp34, "cccccdd");
-
-        let spans = vec![sp1, sp2, sp3, sp4];
-
-        let test = |expected, highlight: &mut FnMut()| {
-            data.lock().unwrap().clear();
-            highlight();
-            let vec = data.lock().unwrap().clone();
-            let actual = from_utf8(&vec[..]).unwrap();
-            println!("actual=\n{}", actual);
-            assert_eq!(actual, expected);
-        };
-
-        let msp = MultiSpan::from_spans(vec![sp1, sp2, sp34]);
-        test(expect_start, &mut || {
-            diag.highlight_lines(&msp, Level::Error).unwrap();
-        });
-        test(expect_start, &mut || {
-            let msp = MultiSpan::from_spans(spans.clone());
-            diag.highlight_lines(&msp, Level::Error).unwrap();
-        });
-    }
-
-    #[test]
-    fn test_huge_multispan_highlight() {
-        let data = Arc::new(Mutex::new(Vec::new()));
-        let cm = Rc::new(CodeMap::new());
-        let mut diag = EmitterWriter::new(Box::new(Sink(data.clone())),
-                                          None,
-                                          cm.clone(),
-                                          FormatMode::NewErrorFormat);
-
-        let inp = "aaaaa\n\
-                   aaaaa\n\
-                   aaaaa\n\
-                   bbbbb\n\
-                   ccccc\n\
-                   xxxxx\n\
-                   yyyyy\n\
-                   _____\n\
-                   ddd__eee_\n\
-                   elided\n\
-                   __f_gg";
-        let file = cm.new_filemap_and_lines("dummy.txt", None, inp);
-
-        let span = |lo, hi, (off_lo, off_hi)| {
-            let lines = file.lines.borrow();
-            let (mut lo, mut hi): (BytePos, BytePos) = (lines[lo], lines[hi]);
-            lo.0 += off_lo;
-            hi.0 += off_hi;
-            mk_sp(lo, hi)
-        };
-        let sp0 = span(4, 6, (0, 5));
-        let sp1 = span(0, 6, (0, 5));
-        let sp2 = span(8, 8, (0, 3));
-        let sp3 = span(8, 8, (5, 8));
-        let sp4 = span(10, 10, (2, 3));
-        let sp5 = span(10, 10, (4, 6));
-
-        let expect0 = &r#"
-   --> dummy.txt:5:1
-    |>
-5   |> ccccc
-    |> ^
-...
-9   |> ddd__eee_
-    |> ^^^  ^^^
-10  |> elided
-11  |> __f_gg
-    |>   ^ ^^
-"#[1..];
-
-        let expect = &r#"
-   --> dummy.txt:1:1
-    |>
-1   |> aaaaa
-    |> ^
-...
-9   |> ddd__eee_
-    |> ^^^  ^^^
-10  |> elided
-11  |> __f_gg
-    |>   ^ ^^
-"#[1..];
-
-        macro_rules! test {
-            ($expected: expr, $highlight: expr) => ({
-                data.lock().unwrap().clear();
-                $highlight();
-                let vec = data.lock().unwrap().clone();
-                let actual = from_utf8(&vec[..]).unwrap();
-                println!("actual:");
-                println!("{}", actual);
-                println!("expected:");
-                println!("{}", $expected);
-                assert_eq!(&actual[..], &$expected[..]);
-            });
-        }
-
-        let msp0 = MultiSpan::from_spans(vec![sp0, sp2, sp3, sp4, sp5]);
-        let msp = MultiSpan::from_spans(vec![sp1, sp2, sp3, sp4, sp5]);
-
-        test!(expect0, || {
-            diag.highlight_lines(&msp0, Level::Error).unwrap();
-        });
-        test!(expect, || {
-            diag.highlight_lines(&msp, Level::Error).unwrap();
-        });
-    }
-
-    #[test]
-    fn tab() {
-        let file_text = "
-fn foo() {
-\tbar;
-}
-";
-
-        let cm = Rc::new(CodeMap::new());
-        let foo = cm.new_filemap_and_lines("foo.rs", None, file_text);
-        let span_bar = cm.span_substr(&foo, file_text, "bar", 0);
-
-        let mut snippet = SnippetData::new(cm, Some(span_bar), FormatMode::NewErrorFormat);
-        snippet.push(span_bar, true, None);
-
-        let lines = snippet.render_lines();
-        let text = make_string(&lines);
-        assert_eq!(&text[..], &"
- --> foo.rs:3:2
-  |>
-3 |> \tbar;
-  |> \t^^^
-"[1..]);
-    }
-
-    #[test]
-    fn one_line() {
-        let file_text = r#"
-fn foo() {
-    vec.push(vec.pop().unwrap());
-}
-"#;
-
-        let cm = Rc::new(CodeMap::new());
-        let foo = cm.new_filemap_and_lines("foo.rs", None, file_text);
-        let span_vec0 = cm.span_substr(&foo, file_text, "vec", 0);
-        let span_vec1 = cm.span_substr(&foo, file_text, "vec", 1);
-        let span_semi = cm.span_substr(&foo, file_text, ";", 0);
-
-        let mut snippet = SnippetData::new(cm, None, FormatMode::NewErrorFormat);
-        snippet.push(span_vec0, false, Some(format!("previous borrow of `vec` occurs here")));
-        snippet.push(span_vec1, false, Some(format!("error occurs here")));
-        snippet.push(span_semi, false, Some(format!("previous borrow ends here")));
-
-        let lines = snippet.render_lines();
-        println!("{:#?}", lines);
-
-        let text: String = make_string(&lines);
-
-        println!("text=\n{}", text);
-        assert_eq!(&text[..], &r#"
- ::: foo.rs
-  |>
-3 |>     vec.push(vec.pop().unwrap());
-  |>     ---      ---                - previous borrow ends here
-  |>     |        |
-  |>     |        error occurs here
-  |>     previous borrow of `vec` occurs here
-"#[1..]);
-    }
-
-    #[test]
-    fn two_files() {
-        let file_text_foo = r#"
-fn foo() {
-    vec.push(vec.pop().unwrap());
-}
-"#;
-
-        let file_text_bar = r#"
-fn bar() {
-    // these blank links here
-    // serve to ensure that the line numbers
-    // from bar.rs
-    // require more digits
-
-
-
-
-
-
-
-
-
-
-    vec.push();
-
-    // this line will get elided
-
-    vec.pop().unwrap());
-}
-"#;
-
-        let cm = Rc::new(CodeMap::new());
-        let foo_map = cm.new_filemap_and_lines("foo.rs", None, file_text_foo);
-        let span_foo_vec0 = cm.span_substr(&foo_map, file_text_foo, "vec", 0);
-        let span_foo_vec1 = cm.span_substr(&foo_map, file_text_foo, "vec", 1);
-        let span_foo_semi = cm.span_substr(&foo_map, file_text_foo, ";", 0);
-
-        let bar_map = cm.new_filemap_and_lines("bar.rs", None, file_text_bar);
-        let span_bar_vec0 = cm.span_substr(&bar_map, file_text_bar, "vec", 0);
-        let span_bar_vec1 = cm.span_substr(&bar_map, file_text_bar, "vec", 1);
-        let span_bar_semi = cm.span_substr(&bar_map, file_text_bar, ";", 0);
-
-        let mut snippet = SnippetData::new(cm, Some(span_foo_vec1), FormatMode::NewErrorFormat);
-        snippet.push(span_foo_vec0, false, Some(format!("a")));
-        snippet.push(span_foo_vec1, true, Some(format!("b")));
-        snippet.push(span_foo_semi, false, Some(format!("c")));
-        snippet.push(span_bar_vec0, false, Some(format!("d")));
-        snippet.push(span_bar_vec1, false, Some(format!("e")));
-        snippet.push(span_bar_semi, false, Some(format!("f")));
-
-        let lines = snippet.render_lines();
-        println!("{:#?}", lines);
-
-        let text: String = make_string(&lines);
-
-        println!("text=\n{}", text);
-
-        // Note that the `|>` remain aligned across both files:
-        assert_eq!(&text[..], &r#"
-   --> foo.rs:3:14
-    |>
-3   |>     vec.push(vec.pop().unwrap());
-    |>     ---      ^^^                - c
-    |>     |        |
-    |>     |        b
-    |>     a
-   ::: bar.rs
-    |>
-17  |>     vec.push();
-    |>     ---       - f
-    |>     |
-    |>     d
-...
-21  |>     vec.pop().unwrap());
-    |>     --- e
-"#[1..]);
-    }
-
-    #[test]
-    fn multi_line() {
-        let file_text = r#"
-fn foo() {
-    let name = find_id(&data, 22).unwrap();
-
-    // Add one more item we forgot to the vector. Silly us.
-    data.push(Data { name: format!("Hera"), id: 66 });
-
-    // Print everything out.
-    println!("Name: {:?}", name);
-    println!("Data: {:?}", data);
-}
-"#;
-
-        let cm = Rc::new(CodeMap::new());
-        let foo = cm.new_filemap_and_lines("foo.rs", None, file_text);
-        let span_data0 = cm.span_substr(&foo, file_text, "data", 0);
-        let span_data1 = cm.span_substr(&foo, file_text, "data", 1);
-        let span_rbrace = cm.span_substr(&foo, file_text, "}", 3);
-
-        let mut snippet = SnippetData::new(cm, None, FormatMode::NewErrorFormat);
-        snippet.push(span_data0, false, Some(format!("immutable borrow begins here")));
-        snippet.push(span_data1, false, Some(format!("mutable borrow occurs here")));
-        snippet.push(span_rbrace, false, Some(format!("immutable borrow ends here")));
-
-        let lines = snippet.render_lines();
-        println!("{:#?}", lines);
-
-        let text: String = make_string(&lines);
-
-        println!("text=\n{}", text);
-        assert_eq!(&text[..], &r#"
-   ::: foo.rs
-    |>
-3   |>     let name = find_id(&data, 22).unwrap();
-    |>                         ---- immutable borrow begins here
-...
-6   |>     data.push(Data { name: format!("Hera"), id: 66 });
-    |>     ---- mutable borrow occurs here
-...
-11  |> }
-    |> - immutable borrow ends here
-"#[1..]);
-    }
-
-    #[test]
-    fn overlapping() {
-        let file_text = r#"
-fn foo() {
-    vec.push(vec.pop().unwrap());
-}
-"#;
-
-        let cm = Rc::new(CodeMap::new());
-        let foo = cm.new_filemap_and_lines("foo.rs", None, file_text);
-        let span0 = cm.span_substr(&foo, file_text, "vec.push", 0);
-        let span1 = cm.span_substr(&foo, file_text, "vec", 0);
-        let span2 = cm.span_substr(&foo, file_text, "ec.push", 0);
-        let span3 = cm.span_substr(&foo, file_text, "unwrap", 0);
-
-        let mut snippet = SnippetData::new(cm, None, FormatMode::NewErrorFormat);
-        snippet.push(span0, false, Some(format!("A")));
-        snippet.push(span1, false, Some(format!("B")));
-        snippet.push(span2, false, Some(format!("C")));
-        snippet.push(span3, false, Some(format!("D")));
-
-        let lines = snippet.render_lines();
-        println!("{:#?}", lines);
-        let text: String = make_string(&lines);
-
-        println!("text=r#\"\n{}\".trim_left()", text);
-        assert_eq!(&text[..], &r#"
- ::: foo.rs
-  |>
-3 |>     vec.push(vec.pop().unwrap());
-  |>     --------           ------ D
-  |>     ||
-  |>     |C
-  |>     A
-  |>     B
-"#[1..]);
-    }
-
-    #[test]
-    fn one_line_out_of_order() {
-        let file_text = r#"
-fn foo() {
-    vec.push(vec.pop().unwrap());
-}
-"#;
-
-        let cm = Rc::new(CodeMap::new());
-        let foo = cm.new_filemap_and_lines("foo.rs", None, file_text);
-        let span_vec0 = cm.span_substr(&foo, file_text, "vec", 0);
-        let span_vec1 = cm.span_substr(&foo, file_text, "vec", 1);
-        let span_semi = cm.span_substr(&foo, file_text, ";", 0);
-
-        // intentionally don't push the snippets left to right
-        let mut snippet = SnippetData::new(cm, None, FormatMode::NewErrorFormat);
-        snippet.push(span_vec1, false, Some(format!("error occurs here")));
-        snippet.push(span_vec0, false, Some(format!("previous borrow of `vec` occurs here")));
-        snippet.push(span_semi, false, Some(format!("previous borrow ends here")));
-
-        let lines = snippet.render_lines();
-        println!("{:#?}", lines);
-        let text: String = make_string(&lines);
-
-        println!("text=r#\"\n{}\".trim_left()", text);
-        assert_eq!(&text[..], &r#"
- ::: foo.rs
-  |>
-3 |>     vec.push(vec.pop().unwrap());
-  |>     ---      ---                - previous borrow ends here
-  |>     |        |
-  |>     |        error occurs here
-  |>     previous borrow of `vec` occurs here
-"#[1..]);
-    }
-
-    #[test]
-    fn elide_unnecessary_lines() {
-        let file_text = r#"
-fn foo() {
-    let mut vec = vec![0, 1, 2];
-    let mut vec2 = vec;
-    vec2.push(3);
-    vec2.push(4);
-    vec2.push(5);
-    vec2.push(6);
-    vec.push(7);
-}
-"#;
-
-        let cm = Rc::new(CodeMap::new());
-        let foo = cm.new_filemap_and_lines("foo.rs", None, file_text);
-        let span_vec0 = cm.span_substr(&foo, file_text, "vec", 3);
-        let span_vec1 = cm.span_substr(&foo, file_text, "vec", 8);
-
-        let mut snippet = SnippetData::new(cm, None, FormatMode::NewErrorFormat);
-        snippet.push(span_vec0, false, Some(format!("`vec` moved here because it \
-            has type `collections::vec::Vec<i32>`")));
-        snippet.push(span_vec1, false, Some(format!("use of moved value: `vec`")));
-
-        let lines = snippet.render_lines();
-        println!("{:#?}", lines);
-        let text: String = make_string(&lines);
-        println!("text=r#\"\n{}\".trim_left()", text);
-        assert_eq!(&text[..], &r#"
-   ::: foo.rs
-    |>
-4   |>     let mut vec2 = vec;
-    |>                    --- `vec` moved here because it has type `collections::vec::Vec<i32>`
-...
-9   |>     vec.push(7);
-    |>     --- use of moved value: `vec`
-"#[1..]);
-    }
-
-    #[test]
-    fn spans_without_labels() {
-        let file_text = r#"
-fn foo() {
-    let mut vec = vec![0, 1, 2];
-    let mut vec2 = vec;
-    vec2.push(3);
-    vec2.push(4);
-    vec2.push(5);
-    vec2.push(6);
-    vec.push(7);
-}
-"#;
-
-        let cm = Rc::new(CodeMap::new());
-        let foo = cm.new_filemap_and_lines("foo.rs", None, file_text);
-
-        let mut snippet = SnippetData::new(cm.clone(), None, FormatMode::NewErrorFormat);
-        for i in 0..4 {
-            let span_veci = cm.span_substr(&foo, file_text, "vec", i);
-            snippet.push(span_veci, false, None);
-        }
-
-        let lines = snippet.render_lines();
-        let text: String = make_string(&lines);
-        println!("text=&r#\"\n{}\n\"#[1..]", text);
-        assert_eq!(text, &r#"
- ::: foo.rs
-  |>
-3 |>     let mut vec = vec![0, 1, 2];
-  |>             ---   ---
-4 |>     let mut vec2 = vec;
-  |>             ---    ---
-"#[1..]);
-    }
-
-    #[test]
-    fn span_long_selection() {
-        let file_text = r#"
-impl SomeTrait for () {
-    fn foo(x: u32) {
-        // impl 1
-        // impl 2
-        // impl 3
-    }
-}
-"#;
-
-        let cm = Rc::new(CodeMap::new());
-        let foo = cm.new_filemap_and_lines("foo.rs", None, file_text);
-
-        let mut snippet = SnippetData::new(cm.clone(), None, FormatMode::NewErrorFormat);
-        let fn_span = cm.span_substr(&foo, file_text, "fn", 0);
-        let rbrace_span = cm.span_substr(&foo, file_text, "}", 0);
-        snippet.push(splice(fn_span, rbrace_span), false, None);
-        let lines = snippet.render_lines();
-        let text: String = make_string(&lines);
-        println!("r#\"\n{}\"", text);
-        assert_eq!(text, &r#"
- ::: foo.rs
-  |>
-3 |>     fn foo(x: u32) {
-  |>     -
-"#[1..]);
-    }
-
-    #[test]
-    fn span_overlap_label() {
-        // Test that we don't put `x_span` to the right of its highlight,
-        // since there is another highlight that overlaps it.
-
-        let file_text = r#"
-    fn foo(x: u32) {
-    }
-}
-"#;
-
-        let cm = Rc::new(CodeMap::new());
-        let foo = cm.new_filemap_and_lines("foo.rs", None, file_text);
-
-        let mut snippet = SnippetData::new(cm.clone(), None, FormatMode::NewErrorFormat);
-        let fn_span = cm.span_substr(&foo, file_text, "fn foo(x: u32)", 0);
-        let x_span = cm.span_substr(&foo, file_text, "x", 0);
-        snippet.push(fn_span, false, Some(format!("fn_span")));
-        snippet.push(x_span, false, Some(format!("x_span")));
-        let lines = snippet.render_lines();
-        let text: String = make_string(&lines);
-        println!("r#\"\n{}\"", text);
-        assert_eq!(text, &r#"
- ::: foo.rs
-  |>
-2 |>     fn foo(x: u32) {
-  |>     --------------
-  |>     |      |
-  |>     |      x_span
-  |>     fn_span
-"#[1..]);
-    }
-
-    #[test]
-    fn span_overlap_label2() {
-        // Test that we don't put `x_span` to the right of its highlight,
-        // since there is another highlight that overlaps it. In this
-        // case, the overlap is only at the beginning, but it's still
-        // better to show the beginning more clearly.
-
-        let file_text = r#"
-    fn foo(x: u32) {
-    }
-}
-"#;
-
-        let cm = Rc::new(CodeMap::new());
-        let foo = cm.new_filemap_and_lines("foo.rs", None, file_text);
-
-        let mut snippet = SnippetData::new(cm.clone(), None, FormatMode::NewErrorFormat);
-        let fn_span = cm.span_substr(&foo, file_text, "fn foo(x", 0);
-        let x_span = cm.span_substr(&foo, file_text, "x: u32)", 0);
-        snippet.push(fn_span, false, Some(format!("fn_span")));
-        snippet.push(x_span, false, Some(format!("x_span")));
-        let lines = snippet.render_lines();
-        let text: String = make_string(&lines);
-        println!("r#\"\n{}\"", text);
-        assert_eq!(text, &r#"
- ::: foo.rs
-  |>
-2 |>     fn foo(x: u32) {
-  |>     --------------
-  |>     |      |
-  |>     |      x_span
-  |>     fn_span
-"#[1..]);
-    }
-
-    #[test]
-    fn span_overlap_label3() {
-        // Test that we don't put `x_span` to the right of its highlight,
-        // since there is another highlight that overlaps it. In this
-        // case, the overlap is only at the beginning, but it's still
-        // better to show the beginning more clearly.
-
-        let file_text = r#"
-    fn foo() {
-       let closure = || {
-           inner
-       };
-    }
-}
-"#;
-
-        let cm = Rc::new(CodeMap::new());
-        let foo = cm.new_filemap_and_lines("foo.rs", None, file_text);
-
-        let mut snippet = SnippetData::new(cm.clone(), None, FormatMode::NewErrorFormat);
-
-        let closure_span = {
-            let closure_start_span = cm.span_substr(&foo, file_text, "||", 0);
-            let closure_end_span = cm.span_substr(&foo, file_text, "}", 0);
-            splice(closure_start_span, closure_end_span)
-        };
-
-        let inner_span = cm.span_substr(&foo, file_text, "inner", 0);
-
-        snippet.push(closure_span, false, Some(format!("foo")));
-        snippet.push(inner_span, false, Some(format!("bar")));
-
-        let lines = snippet.render_lines();
-        let text: String = make_string(&lines);
-        println!("r#\"\n{}\"", text);
-        assert_eq!(text, &r#"
- ::: foo.rs
-  |>
-3 |>        let closure = || {
-  |>                      - foo
-4 |>            inner
-  |>            ----- bar
-"#[1..]);
-    }
-
-    #[test]
-    fn span_empty() {
-        // In one of the unit tests, we found that the parser sometimes
-        // gives empty spans, and in particular it supplied an EOF span
-        // like this one, which points at the very end. We want to
-        // fallback gracefully in this case.
-
-        let file_text = r#"
-fn main() {
-    struct Foo;
-
-    impl !Sync for Foo {}
-
-    unsafe impl Send for &'static Foo {
-    // error: cross-crate traits with a default impl, like `core::marker::Send`,
-    //        can only be implemented for a struct/enum type, not
-    //        `&'static Foo`
-}"#;
-
-
-        let cm = Rc::new(CodeMap::new());
-        let foo = cm.new_filemap_and_lines("foo.rs", None, file_text);
-
-        let mut rbrace_span = cm.span_substr(&foo, file_text, "}", 1);
-        rbrace_span.lo = rbrace_span.hi;
-
-        let mut snippet = SnippetData::new(cm.clone(),
-                                           Some(rbrace_span),
-                                           FormatMode::NewErrorFormat);
-        snippet.push(rbrace_span, false, None);
-        let lines = snippet.render_lines();
-        let text: String = make_string(&lines);
-        println!("r#\"\n{}\"", text);
-        assert_eq!(text, &r#"
-  --> foo.rs:11:2
-   |>
-11 |> }
-   |>  -
-"#[1..]);
-    }
 }
diff --git a/src/libsyntax/ext/base.rs b/src/libsyntax/ext/base.rs
index 7ebcd12cdb9..ca2be89def0 100644
--- a/src/libsyntax/ext/base.rs
+++ b/src/libsyntax/ext/base.rs
@@ -816,6 +816,12 @@ impl<'a> ExtCtxt<'a> {
 /// compilation on error, merely emits a non-fatal error and returns None.
 pub fn expr_to_string(cx: &mut ExtCtxt, expr: P<ast::Expr>, err_msg: &str)
                       -> Option<(InternedString, ast::StrStyle)> {
+    // Update `expr.span`'s expn_id now in case expr is an `include!` macro invocation.
+    let expr = expr.map(|mut expr| {
+        expr.span.expn_id = cx.backtrace;
+        expr
+    });
+
     // we want to be able to handle e.g. concat("foo", "bar")
     let expr = cx.expander().fold_expr(expr);
     match expr.node {
diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index 3e9837a6995..18342f2e38c 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -9,20 +9,19 @@
 // except according to those terms.
 
 use ast::{Block, Crate, Ident, Mac_, Name, PatKind};
-use ast::{MacStmtStyle, Mrk, Stmt, StmtKind, ItemKind};
+use ast::{MacStmtStyle, Stmt, StmtKind, ItemKind};
 use ast;
-use attr::HasAttrs;
-use ext::mtwt;
-use attr;
+use ext::hygiene::Mark;
+use attr::{self, HasAttrs};
 use attr::AttrMetaMethods;
-use codemap::{dummy_spanned, Spanned, ExpnInfo, NameAndSpan, MacroBang, MacroAttribute};
+use codemap::{dummy_spanned, ExpnInfo, NameAndSpan, MacroBang, MacroAttribute};
 use syntax_pos::{self, Span, ExpnId};
 use config::StripUnconfigured;
 use ext::base::*;
 use feature_gate::{self, Features};
 use fold;
 use fold::*;
-use parse::token::{fresh_mark, intern, keywords};
+use parse::token::{intern, keywords};
 use ptr::P;
 use tokenstream::TokenTree;
 use util::small_vector::SmallVector;
@@ -130,9 +129,9 @@ fn expand_mac_invoc<T>(mac: ast::Mac, ident: Option<Ident>, attrs: Vec<ast::Attr
     // It would almost certainly be cleaner to pass the whole macro invocation in,
     // rather than pulling it apart and marking the tts and the ctxt separately.
     let Mac_ { path, tts, .. } = mac.node;
-    let mark = fresh_mark();
+    let mark = Mark::fresh();
 
-    fn mac_result<'a>(path: &ast::Path, ident: Option<Ident>, tts: Vec<TokenTree>, mark: Mrk,
+    fn mac_result<'a>(path: &ast::Path, ident: Option<Ident>, tts: Vec<TokenTree>, mark: Mark,
                       attrs: Vec<ast::Attribute>, call_site: Span, fld: &'a mut MacroExpander)
                       -> Option<Box<MacResult + 'a>> {
         // Detect use of feature-gated or invalid attributes on macro invoations
@@ -708,30 +707,17 @@ pub fn expand_crate(mut cx: ExtCtxt,
     return (ret, cx.syntax_env.names);
 }
 
-// HYGIENIC CONTEXT EXTENSION:
-// all of these functions are for walking over
-// ASTs and making some change to the context of every
-// element that has one. a CtxtFn is a trait-ified
-// version of a closure in (SyntaxContext -> SyntaxContext).
-// the ones defined here include:
-// Marker - add a mark to a context
-
 // A Marker adds the given mark to the syntax context and
 // sets spans' `expn_id` to the given expn_id (unless it is `None`).
-struct Marker { mark: Mrk, expn_id: Option<ExpnId> }
+struct Marker { mark: Mark, expn_id: Option<ExpnId> }
 
 impl Folder for Marker {
-    fn fold_ident(&mut self, id: Ident) -> Ident {
-        ast::Ident::new(id.name, mtwt::apply_mark(self.mark, id.ctxt))
-    }
-    fn fold_mac(&mut self, Spanned {node, span}: ast::Mac) -> ast::Mac {
-        Spanned {
-            node: Mac_ {
-                path: self.fold_path(node.path),
-                tts: self.fold_tts(&node.tts),
-            },
-            span: self.new_span(span),
-        }
+    fn fold_ident(&mut self, mut ident: Ident) -> Ident {
+        ident.ctxt = ident.ctxt.apply_mark(self.mark);
+        ident
+    }
+    fn fold_mac(&mut self, mac: ast::Mac) -> ast::Mac {
+        noop_fold_mac(mac, self)
     }
 
     fn new_span(&mut self, mut span: Span) -> Span {
@@ -743,7 +729,7 @@ impl Folder for Marker {
 }
 
 // apply a given mark to the given token trees. Used prior to expansion of a macro.
-fn mark_tts(tts: &[TokenTree], m: Mrk) -> Vec<TokenTree> {
+fn mark_tts(tts: &[TokenTree], m: Mark) -> Vec<TokenTree> {
     noop_fold_tts(tts, &mut Marker{mark:m, expn_id: None})
 }
 
diff --git a/src/libsyntax/ext/hygiene.rs b/src/libsyntax/ext/hygiene.rs
new file mode 100644
index 00000000000..ade165e0ef9
--- /dev/null
+++ b/src/libsyntax/ext/hygiene.rs
@@ -0,0 +1,116 @@
+// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+//! Machinery for hygienic macros, inspired by the MTWT[1] paper.
+//!
+//! [1] Matthew Flatt, Ryan Culpepper, David Darais, and Robert Bruce Findler.
+//! 2012. *Macros that work together: Compile-time bindings, partial expansion,
+//! and definition contexts*. J. Funct. Program. 22, 2 (March 2012), 181-216.
+//! DOI=10.1017/S0956796812000093 http://dx.doi.org/10.1017/S0956796812000093
+
+use std::cell::RefCell;
+use std::collections::HashMap;
+use std::fmt;
+
+/// A SyntaxContext represents a chain of macro expansions (represented by marks).
+#[derive(Clone, Copy, PartialEq, Eq, Hash, RustcEncodable, RustcDecodable, Default)]
+pub struct SyntaxContext(u32);
+
+#[derive(Copy, Clone)]
+pub struct SyntaxContextData {
+    pub outer_mark: Mark,
+    pub prev_ctxt: SyntaxContext,
+}
+
+/// A mark represents a unique id associated with a macro expansion.
+#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, Default)]
+pub struct Mark(u32);
+
+impl Mark {
+    pub fn fresh() -> Self {
+        HygieneData::with(|data| {
+            let next_mark = Mark(data.next_mark.0 + 1);
+            ::std::mem::replace(&mut data.next_mark, next_mark)
+        })
+    }
+}
+
+struct HygieneData {
+    syntax_contexts: Vec<SyntaxContextData>,
+    markings: HashMap<(SyntaxContext, Mark), SyntaxContext>,
+    next_mark: Mark,
+}
+
+impl HygieneData {
+    fn new() -> Self {
+        HygieneData {
+            syntax_contexts: vec![SyntaxContextData {
+                outer_mark: Mark(0), // the null mark
+                prev_ctxt: SyntaxContext(0), // the empty context
+            }],
+            markings: HashMap::new(),
+            next_mark: Mark(1),
+        }
+    }
+
+    fn with<T, F: FnOnce(&mut HygieneData) -> T>(f: F) -> T {
+        thread_local! {
+            static HYGIENE_DATA: RefCell<HygieneData> = RefCell::new(HygieneData::new());
+        }
+        HYGIENE_DATA.with(|data| f(&mut *data.borrow_mut()))
+    }
+}
+
+pub fn reset_hygiene_data() {
+    HygieneData::with(|data| *data = HygieneData::new())
+}
+
+impl SyntaxContext {
+    pub const fn empty() -> Self {
+        SyntaxContext(0)
+    }
+
+    pub fn data(self) -> SyntaxContextData {
+        HygieneData::with(|data| data.syntax_contexts[self.0 as usize])
+    }
+
+    /// Extend a syntax context with a given mark
+    pub fn apply_mark(self, mark: Mark) -> SyntaxContext {
+        // Applying the same mark twice is a no-op
+        let ctxt_data = self.data();
+        if mark == ctxt_data.outer_mark {
+            return ctxt_data.prev_ctxt;
+        }
+
+        HygieneData::with(|data| {
+            let syntax_contexts = &mut data.syntax_contexts;
+            *data.markings.entry((self, mark)).or_insert_with(|| {
+                syntax_contexts.push(SyntaxContextData {
+                    outer_mark: mark,
+                    prev_ctxt: self,
+                });
+                SyntaxContext(syntax_contexts.len() as u32 - 1)
+            })
+        })
+    }
+
+   /// If `ident` is macro expanded, return the source ident from the macro definition
+   /// and the mark of the expansion that created the macro definition.
+   pub fn source(self) -> (Self /* source context */, Mark /* source macro */) {
+        let macro_def_ctxt = self.data().prev_ctxt.data();
+        (macro_def_ctxt.prev_ctxt, macro_def_ctxt.outer_mark)
+   }
+}
+
+impl fmt::Debug for SyntaxContext {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "#{}", self.0)
+    }
+}
diff --git a/src/libsyntax/ext/mtwt.rs b/src/libsyntax/ext/mtwt.rs
deleted file mode 100644
index d2f6df9d5db..00000000000
--- a/src/libsyntax/ext/mtwt.rs
+++ /dev/null
@@ -1,154 +0,0 @@
-// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
-// file at the top-level directory of this distribution and at
-// http://rust-lang.org/COPYRIGHT.
-//
-// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
-// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
-// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
-// option. This file may not be copied, modified, or distributed
-// except according to those terms.
-
-//! Machinery for hygienic macros, as described in the MTWT[1] paper.
-//!
-//! [1] Matthew Flatt, Ryan Culpepper, David Darais, and Robert Bruce Findler.
-//! 2012. *Macros that work together: Compile-time bindings, partial expansion,
-//! and definition contexts*. J. Funct. Program. 22, 2 (March 2012), 181-216.
-//! DOI=10.1017/S0956796812000093 http://dx.doi.org/10.1017/S0956796812000093
-
-pub use self::SyntaxContext_::*;
-
-use ast::{Ident, Mrk, SyntaxContext};
-
-use std::cell::RefCell;
-use std::collections::HashMap;
-
-/// The SCTable contains a table of SyntaxContext_'s. It
-/// represents a flattened tree structure, to avoid having
-/// managed pointers everywhere (that caused an ICE).
-/// The `marks` ensures that adding the same mark to the
-/// same context gives you back the same context as before.
-pub struct SCTable {
-    table: RefCell<Vec<SyntaxContext_>>,
-    marks: RefCell<HashMap<(SyntaxContext,Mrk),SyntaxContext>>,
-}
-
-#[derive(PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy, Clone)]
-pub enum SyntaxContext_ {
-    EmptyCtxt,
-    Mark (Mrk,SyntaxContext),
-}
-
-/// Extend a syntax context with a given mark
-pub fn apply_mark(m: Mrk, ctxt: SyntaxContext) -> SyntaxContext {
-    with_sctable(|table| apply_mark_internal(m, ctxt, table))
-}
-
-/// Extend a syntax context with a given mark and sctable (explicit memoization)
-fn apply_mark_internal(m: Mrk, ctxt: SyntaxContext, table: &SCTable) -> SyntaxContext {
-    let ctxts = &mut *table.table.borrow_mut();
-    match ctxts[ctxt.0 as usize] {
-        // Applying the same mark twice is a no-op.
-        Mark(outer_mark, prev_ctxt) if outer_mark == m => return prev_ctxt,
-        _ => *table.marks.borrow_mut().entry((ctxt, m)).or_insert_with(|| {
-            SyntaxContext(idx_push(ctxts, Mark(m, ctxt)))
-        }),
-    }
-}
-
-/// Fetch the SCTable from TLS, create one if it doesn't yet exist.
-pub fn with_sctable<T, F>(op: F) -> T where
-    F: FnOnce(&SCTable) -> T,
-{
-    thread_local!(static SCTABLE_KEY: SCTable = new_sctable_internal());
-    SCTABLE_KEY.with(move |slot| op(slot))
-}
-
-// Make a fresh syntax context table with EmptyCtxt in slot zero.
-fn new_sctable_internal() -> SCTable {
-    SCTable {
-        table: RefCell::new(vec![EmptyCtxt]),
-        marks: RefCell::new(HashMap::new()),
-    }
-}
-
-/// Clear the tables from TLD to reclaim memory.
-pub fn clear_tables() {
-    with_sctable(|table| {
-        *table.table.borrow_mut() = Vec::new();
-        *table.marks.borrow_mut() = HashMap::new();
-    });
-}
-
-/// Reset the tables to their initial state
-pub fn reset_tables() {
-    with_sctable(|table| {
-        *table.table.borrow_mut() = vec![EmptyCtxt];
-        *table.marks.borrow_mut() = HashMap::new();
-    });
-}
-
-/// Add a value to the end of a vec, return its index
-fn idx_push<T>(vec: &mut Vec<T>, val: T) -> u32 {
-    vec.push(val);
-    (vec.len() - 1) as u32
-}
-
-/// Return the outer mark for a context with a mark at the outside.
-/// FAILS when outside is not a mark.
-pub fn outer_mark(ctxt: SyntaxContext) -> Mrk {
-    with_sctable(|sctable| {
-        match (*sctable.table.borrow())[ctxt.0 as usize] {
-            Mark(mrk, _) => mrk,
-            _ => panic!("can't retrieve outer mark when outside is not a mark")
-        }
-    })
-}
-
-/// If `ident` is macro expanded, return the source ident from the macro definition
-/// and the mark of the expansion that created the macro definition.
-pub fn source(ident: Ident) -> Option<(Ident /* source ident */, Mrk /* source macro */)> {
-    with_sctable(|sctable| {
-        let ctxts = sctable.table.borrow();
-        if let Mark(_expansion_mark, macro_ctxt) = ctxts[ident.ctxt.0 as usize] {
-            if let Mark(definition_mark, orig_ctxt) = ctxts[macro_ctxt.0 as usize] {
-                return Some((Ident::new(ident.name, orig_ctxt), definition_mark));
-            }
-        }
-        None
-    })
-}
-
-#[cfg(test)]
-mod tests {
-    use ast::{EMPTY_CTXT, Mrk, SyntaxContext};
-    use super::{apply_mark_internal, new_sctable_internal, Mark, SCTable};
-
-    // extend a syntax context with a sequence of marks given
-    // in a vector. v[0] will be the outermost mark.
-    fn unfold_marks(mrks: Vec<Mrk> , tail: SyntaxContext, table: &SCTable)
-                    -> SyntaxContext {
-        mrks.iter().rev().fold(tail, |tail:SyntaxContext, mrk:&Mrk|
-                   {apply_mark_internal(*mrk,tail,table)})
-    }
-
-    #[test] fn unfold_marks_test() {
-        let mut t = new_sctable_internal();
-
-        assert_eq!(unfold_marks(vec!(3,7),EMPTY_CTXT,&mut t),SyntaxContext(2));
-        {
-            let table = t.table.borrow();
-            assert!((*table)[1] == Mark(7,EMPTY_CTXT));
-            assert!((*table)[2] == Mark(3,SyntaxContext(1)));
-        }
-    }
-
-    #[test]
-    fn hashing_tests () {
-        let mut t = new_sctable_internal();
-        assert_eq!(apply_mark_internal(12,EMPTY_CTXT,&mut t),SyntaxContext(1));
-        assert_eq!(apply_mark_internal(13,EMPTY_CTXT,&mut t),SyntaxContext(2));
-        // using the same one again should result in the same index:
-        assert_eq!(apply_mark_internal(12,EMPTY_CTXT,&mut t),SyntaxContext(1));
-        // I'm assuming that the rename table will behave the same....
-    }
-}
diff --git a/src/libsyntax/json.rs b/src/libsyntax/json.rs
index dc9a5ee4664..a40c30b3e33 100644
--- a/src/libsyntax/json.rs
+++ b/src/libsyntax/json.rs
@@ -22,7 +22,7 @@
 use codemap::CodeMap;
 use syntax_pos::{self, MacroBacktrace, Span, SpanLabel, MultiSpan};
 use errors::registry::Registry;
-use errors::{Level, DiagnosticBuilder, SubDiagnostic, RenderSpan, CodeSuggestion, CodeMapper};
+use errors::{DiagnosticBuilder, SubDiagnostic, RenderSpan, CodeSuggestion, CodeMapper};
 use errors::emitter::Emitter;
 
 use std::rc::Rc;
@@ -53,14 +53,7 @@ impl JsonEmitter {
 }
 
 impl Emitter for JsonEmitter {
-    fn emit(&mut self, span: &MultiSpan, msg: &str, code: Option<&str>, level: Level) {
-        let data = Diagnostic::new(span, msg, code, level, self);
-        if let Err(e) = writeln!(&mut self.dst, "{}", as_json(&data)) {
-            panic!("failed to print diagnostics: {:?}", e);
-        }
-    }
-
-    fn emit_struct(&mut self, db: &DiagnosticBuilder) {
+    fn emit(&mut self, db: &DiagnosticBuilder) {
         let data = Diagnostic::from_diagnostic_builder(db, self);
         if let Err(e) = writeln!(&mut self.dst, "{}", as_json(&data)) {
             panic!("failed to print diagnostics: {:?}", e);
@@ -146,22 +139,6 @@ struct DiagnosticCode {
 }
 
 impl<'a> Diagnostic<'a> {
-    fn new(msp: &MultiSpan,
-           msg: &'a str,
-           code: Option<&str>,
-           level: Level,
-           je: &JsonEmitter)
-           -> Diagnostic<'a> {
-        Diagnostic {
-            message: msg,
-            code: DiagnosticCode::map_opt_string(code.map(|c| c.to_owned()), je),
-            level: level.to_str(),
-            spans: DiagnosticSpan::from_multispan(msp, je),
-            children: vec![],
-            rendered: None,
-        }
-    }
-
     fn from_diagnostic_builder<'c>(db: &'c DiagnosticBuilder,
                                    je: &JsonEmitter)
                                    -> Diagnostic<'c> {
diff --git a/src/libsyntax/lib.rs b/src/libsyntax/lib.rs
index 8febf1c49ec..5ad17444188 100644
--- a/src/libsyntax/lib.rs
+++ b/src/libsyntax/lib.rs
@@ -127,7 +127,7 @@ pub mod ext {
     pub mod base;
     pub mod build;
     pub mod expand;
-    pub mod mtwt;
+    pub mod hygiene;
     pub mod quote;
     pub mod source_util;
 
diff --git a/src/libsyntax/parse/attr.rs b/src/libsyntax/parse/attr.rs
index 15344cef1db..2ae3236cd5a 100644
--- a/src/libsyntax/parse/attr.rs
+++ b/src/libsyntax/parse/attr.rs
@@ -18,23 +18,43 @@ use parse::token;
 use parse::parser::{Parser, TokenType};
 use ptr::P;
 
+#[derive(PartialEq, Eq, Debug)]
+enum InnerAttributeParsePolicy<'a> {
+    Permitted,
+    NotPermitted { reason: &'a str },
+}
+
+const DEFAULT_UNEXPECTED_INNER_ATTR_ERR_MSG: &'static str = "an inner attribute is not \
+                                                             permitted in this context";
+
 impl<'a> Parser<'a> {
     /// Parse attributes that appear before an item
     pub fn parse_outer_attributes(&mut self) -> PResult<'a, Vec<ast::Attribute>> {
         let mut attrs: Vec<ast::Attribute> = Vec::new();
+        let mut just_parsed_doc_comment = false;
         loop {
             debug!("parse_outer_attributes: self.token={:?}", self.token);
             match self.token {
                 token::Pound => {
-                    attrs.push(self.parse_attribute(false)?);
+                    let inner_error_reason = if just_parsed_doc_comment {
+                        "an inner attribute is not permitted following an outer doc comment"
+                    } else if !attrs.is_empty() {
+                        "an inner attribute is not permitted following an outer attribute"
+                    } else {
+                        DEFAULT_UNEXPECTED_INNER_ATTR_ERR_MSG
+                    };
+                    let inner_parse_policy =
+                        InnerAttributeParsePolicy::NotPermitted { reason: inner_error_reason };
+                    attrs.push(self.parse_attribute_with_inner_parse_policy(inner_parse_policy)?);
+                    just_parsed_doc_comment = false;
                 }
                 token::DocComment(s) => {
                     let attr = ::attr::mk_sugared_doc_attr(
-                    attr::mk_attr_id(),
-                    self.id_to_interned_str(ast::Ident::with_empty_ctxt(s)),
-                    self.span.lo,
-                    self.span.hi
-                );
+                        attr::mk_attr_id(),
+                        self.id_to_interned_str(ast::Ident::with_empty_ctxt(s)),
+                        self.span.lo,
+                        self.span.hi
+                    );
                     if attr.node.style != ast::AttrStyle::Outer {
                         let mut err = self.fatal("expected outer doc comment");
                         err.note("inner doc comments like this (starting with \
@@ -43,6 +63,7 @@ impl<'a> Parser<'a> {
                     }
                     attrs.push(attr);
                     self.bump();
+                    just_parsed_doc_comment = true;
                 }
                 _ => break,
             }
@@ -55,26 +76,46 @@ impl<'a> Parser<'a> {
     /// If permit_inner is true, then a leading `!` indicates an inner
     /// attribute
     pub fn parse_attribute(&mut self, permit_inner: bool) -> PResult<'a, ast::Attribute> {
-        debug!("parse_attributes: permit_inner={:?} self.token={:?}",
+        debug!("parse_attribute: permit_inner={:?} self.token={:?}",
                permit_inner,
                self.token);
+        let inner_parse_policy = if permit_inner {
+            InnerAttributeParsePolicy::Permitted
+        } else {
+            InnerAttributeParsePolicy::NotPermitted
+                { reason: DEFAULT_UNEXPECTED_INNER_ATTR_ERR_MSG }
+        };
+        self.parse_attribute_with_inner_parse_policy(inner_parse_policy)
+    }
+
+    /// The same as `parse_attribute`, except it takes in an `InnerAttributeParsePolicy`
+    /// that prescribes how to handle inner attributes.
+    fn parse_attribute_with_inner_parse_policy(&mut self,
+                                               inner_parse_policy: InnerAttributeParsePolicy)
+                                               -> PResult<'a, ast::Attribute> {
+        debug!("parse_attribute_with_inner_parse_policy: inner_parse_policy={:?} self.token={:?}",
+               inner_parse_policy,
+               self.token);
         let (span, value, mut style) = match self.token {
             token::Pound => {
                 let lo = self.span.lo;
                 self.bump();
 
-                if permit_inner {
+                if inner_parse_policy == InnerAttributeParsePolicy::Permitted {
                     self.expected_tokens.push(TokenType::Token(token::Not));
                 }
                 let style = if self.token == token::Not {
                     self.bump();
-                    if !permit_inner {
+                    if let InnerAttributeParsePolicy::NotPermitted { reason } = inner_parse_policy
+                    {
                         let span = self.span;
                         self.diagnostic()
-                            .struct_span_err(span,
-                                             "an inner attribute is not permitted in this context")
-                            .help("place inner attribute at the top of the module or \
-                                   block")
+                            .struct_span_err(span, reason)
+                            .note("inner attributes and doc comments, like `#![no_std]` or \
+                                   `//! My crate`, annotate the item enclosing them, and are \
+                                   usually found at the beginning of source files. Outer \
+                                   attributes and doc comments, like `#[test]` and
+                                   `/// My function`, annotate the item following them.")
                             .emit()
                     }
                     ast::AttrStyle::Inner
@@ -95,7 +136,8 @@ impl<'a> Parser<'a> {
             }
         };
 
-        if permit_inner && self.token == token::Semi {
+        if inner_parse_policy == InnerAttributeParsePolicy::Permitted &&
+           self.token == token::Semi {
             self.bump();
             self.span_warn(span,
                            "this inner attribute syntax is deprecated. The new syntax is \
diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs
index 77b5c10899a..5ea1d6be9fe 100644
--- a/src/libsyntax/parse/lexer/mod.rs
+++ b/src/libsyntax/parse/lexer/mod.rs
@@ -1686,7 +1686,7 @@ mod tests {
         // FIXME (#22405): Replace `Box::new` with `box` here when/if possible.
         let emitter = errors::emitter::EmitterWriter::new(Box::new(io::sink()),
                                                 None,
-                                                cm,
+                                                Some(cm),
                                                 errors::snippet::FormatMode::EnvironmentSelected);
         errors::Handler::with_emitter(true, false, Box::new(emitter))
     }
diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs
index 9502bc48a3e..2147e8ec2eb 100644
--- a/src/libsyntax/parse/mod.rs
+++ b/src/libsyntax/parse/mod.rs
@@ -50,7 +50,11 @@ pub struct ParseSess {
 impl ParseSess {
     pub fn new() -> ParseSess {
         let cm = Rc::new(CodeMap::new());
-        let handler = Handler::with_tty_emitter(ColorConfig::Auto, None, true, false, cm.clone());
+        let handler = Handler::with_tty_emitter(ColorConfig::Auto,
+                                                None,
+                                                true,
+                                                false,
+                                                Some(cm.clone()));
         ParseSess::with_span_handler(handler, cm)
     }
 
@@ -224,10 +228,18 @@ pub fn filemap_to_parser<'a>(sess: &'a ParseSess,
 // compiler expands into it
 pub fn new_parser_from_tts<'a>(sess: &'a ParseSess,
                                cfg: ast::CrateConfig,
-                               tts: Vec<tokenstream::TokenTree>) -> Parser<'a> {
+                               tts: Vec<tokenstream::TokenTree>)
+                               -> Parser<'a> {
     tts_to_parser(sess, tts, cfg)
 }
 
+pub fn new_parser_from_ts<'a>(sess: &'a ParseSess,
+                              cfg: ast::CrateConfig,
+                              ts: tokenstream::TokenStream)
+                              -> Parser<'a> {
+    tts_to_parser(sess, ts.tts, cfg)
+}
+
 
 // base abstractions
 
diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs
index 4656ba03e21..125f1abb062 100644
--- a/src/libsyntax/parse/parser.rs
+++ b/src/libsyntax/parse/parser.rs
@@ -3789,13 +3789,8 @@ impl<'a> Parser<'a> {
 
     /// Parse a statement. This stops just before trailing semicolons on everything but items.
     /// e.g. a `StmtKind::Semi` parses to a `StmtKind::Expr`, leaving the trailing `;` unconsumed.
-    ///
-    /// Also, if a macro begins an expression statement, this only parses the macro. For example,
-    /// ```rust
-    /// vec![1].into_iter(); //< `parse_stmt` only parses the "vec![1]"
-    /// ```
     pub fn parse_stmt(&mut self) -> PResult<'a, Option<Stmt>> {
-        Ok(self.parse_stmt_())
+        Ok(self.parse_stmt_(true))
     }
 
     // Eat tokens until we can be relatively sure we reached the end of the
@@ -3859,15 +3854,15 @@ impl<'a> Parser<'a> {
         }
     }
 
-    fn parse_stmt_(&mut self) -> Option<Stmt> {
-        self.parse_stmt_without_recovery().unwrap_or_else(|mut e| {
+    fn parse_stmt_(&mut self, macro_expanded: bool) -> Option<Stmt> {
+        self.parse_stmt_without_recovery(macro_expanded).unwrap_or_else(|mut e| {
             e.emit();
             self.recover_stmt_(SemiColonMode::Break);
             None
         })
     }
 
-    fn parse_stmt_without_recovery(&mut self) -> PResult<'a, Option<Stmt>> {
+    fn parse_stmt_without_recovery(&mut self, macro_expanded: bool) -> PResult<'a, Option<Stmt>> {
         maybe_whole!(Some deref self, NtStmt);
 
         let attrs = self.parse_outer_attributes()?;
@@ -3930,10 +3925,34 @@ impl<'a> Parser<'a> {
 
             if id.name == keywords::Invalid.name() {
                 let mac = spanned(lo, hi, Mac_ { path: pth, tts: tts });
+                let node = if delim == token::Brace ||
+                              self.token == token::Semi || self.token == token::Eof {
+                    StmtKind::Mac(P((mac, style, attrs.into())))
+                }
+                // We used to incorrectly stop parsing macro-expanded statements here.
+                // If the next token will be an error anyway but could have parsed with the
+                // earlier behavior, stop parsing here and emit a warning to avoid breakage.
+                else if macro_expanded && self.token.can_begin_expr() && match self.token {
+                    // These can continue an expression, so we can't stop parsing and warn.
+                    token::OpenDelim(token::Paren) | token::OpenDelim(token::Bracket) |
+                    token::BinOp(token::Minus) | token::BinOp(token::Star) |
+                    token::BinOp(token::And) | token::BinOp(token::Or) |
+                    token::AndAnd | token::OrOr |
+                    token::DotDot | token::DotDotDot => false,
+                    _ => true,
+                } {
+                    self.warn_missing_semicolon();
+                    StmtKind::Mac(P((mac, style, attrs.into())))
+                } else {
+                    let e = self.mk_mac_expr(lo, hi, mac.node, ThinVec::new());
+                    let e = self.parse_dot_or_call_expr_with(e, lo, attrs.into())?;
+                    let e = self.parse_assoc_expr_with(0, LhsExpr::AlreadyParsed(e))?;
+                    StmtKind::Expr(e)
+                };
                 Stmt {
                     id: ast::DUMMY_NODE_ID,
-                    node: StmtKind::Mac(P((mac, style, attrs.into()))),
                     span: mk_sp(lo, hi),
+                    node: node,
                 }
             } else {
                 // if it has a special ident, it's definitely an item
@@ -4061,49 +4080,12 @@ impl<'a> Parser<'a> {
     }
 
     /// Parse a statement, including the trailing semicolon.
-    /// This parses expression statements that begin with macros correctly (c.f. `parse_stmt`).
     pub fn parse_full_stmt(&mut self, macro_expanded: bool) -> PResult<'a, Option<Stmt>> {
-        let mut stmt = match self.parse_stmt_() {
+        let mut stmt = match self.parse_stmt_(macro_expanded) {
             Some(stmt) => stmt,
             None => return Ok(None),
         };
 
-        if let StmtKind::Mac(mac) = stmt.node {
-            if mac.1 != MacStmtStyle::NoBraces ||
-               self.token == token::Semi || self.token == token::Eof {
-                stmt.node = StmtKind::Mac(mac);
-            } else {
-                // We used to incorrectly stop parsing macro-expanded statements here.
-                // If the next token will be an error anyway but could have parsed with the
-                // earlier behavior, stop parsing here and emit a warning to avoid breakage.
-                if macro_expanded && self.token.can_begin_expr() && match self.token {
-                    // These tokens can continue an expression, so we can't stop parsing and warn.
-                    token::OpenDelim(token::Paren) | token::OpenDelim(token::Bracket) |
-                    token::BinOp(token::Minus) | token::BinOp(token::Star) |
-                    token::BinOp(token::And) | token::BinOp(token::Or) |
-                    token::AndAnd | token::OrOr |
-                    token::DotDot | token::DotDotDot => false,
-                    _ => true,
-                } {
-                    self.warn_missing_semicolon();
-                    stmt.node = StmtKind::Mac(mac);
-                    return Ok(Some(stmt));
-                }
-
-                let (mac, _style, attrs) = mac.unwrap();
-                let e = self.mk_mac_expr(stmt.span.lo, stmt.span.hi, mac.node, ThinVec::new());
-                let e = self.parse_dot_or_call_expr_with(e, stmt.span.lo, attrs)?;
-                let e = self.parse_assoc_expr_with(0, LhsExpr::AlreadyParsed(e))?;
-                stmt.node = StmtKind::Expr(e);
-            }
-        }
-
-        stmt = self.handle_trailing_semicolon(stmt, macro_expanded)?;
-        Ok(Some(stmt))
-    }
-
-    fn handle_trailing_semicolon(&mut self, mut stmt: Stmt, macro_expanded: bool)
-                                 -> PResult<'a, Stmt> {
         match stmt.node {
             StmtKind::Expr(ref expr) if self.token != token::Eof => {
                 // expression without semicolon
@@ -4133,7 +4115,7 @@ impl<'a> Parser<'a> {
         }
 
         stmt.span.hi = self.last_span.hi;
-        Ok(stmt)
+        Ok(Some(stmt))
     }
 
     fn warn_missing_semicolon(&self) {
diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs
index fe9d3ef7c23..f0a6f8edeec 100644
--- a/src/libsyntax/parse/token.rs
+++ b/src/libsyntax/parse/token.rs
@@ -633,8 +633,3 @@ pub fn fresh_name(src: ast::Ident) -> ast::Name {
     /*let num = rand::thread_rng().gen_uint_range(0,0xffff);
     gensym(format!("{}_{}",ident_to_string(src),num))*/
 }
-
-// create a fresh mark.
-pub fn fresh_mark() -> ast::Mrk {
-    gensym("mark").0
-}
diff --git a/src/libsyntax/test.rs b/src/libsyntax/test.rs
index 327696e87b0..faf6a17a150 100644
--- a/src/libsyntax/test.rs
+++ b/src/libsyntax/test.rs
@@ -26,7 +26,7 @@ use std::rc::Rc;
 
 use codemap::{self, CodeMap, ExpnInfo, NameAndSpan, MacroAttribute};
 use errors;
-use errors::snippet::{RenderedLine, SnippetData};
+use errors::snippet::{SnippetData};
 use config;
 use entry::{self, EntryPointType};
 use ext::base::{ExtCtxt, DummyMacroLoader};