Implement `quote!` and other `proc_macro` API.

10 files changed, 245 insertions, 41 deletions

diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs
index 8bd58ec7a52..325a5cdf8fc 100644
--- a/src/libsyntax/ast.rs
+++ b/src/libsyntax/ast.rs
@@ -100,7 +100,7 @@ impl Path {
         let name = self.segments[0].identifier.name;
         if !self.is_global() && name != "$crate" &&
            name != keywords::SelfValue.name() && name != keywords::Super.name() {
-            self.segments.insert(0, PathSegment::crate_root());
+            self.segments.insert(0, PathSegment::crate_root(self.span));
         }
         self
     }
@@ -134,10 +134,10 @@ impl PathSegment {
     pub fn from_ident(ident: Ident, span: Span) -> Self {
         PathSegment { identifier: ident, span: span, parameters: None }
     }
-    pub fn crate_root() -> Self {
+    pub fn crate_root(span: Span) -> Self {
         PathSegment {
-            identifier: keywords::CrateRoot.ident(),
-            span: DUMMY_SP,
+            identifier: Ident { ctxt: span.ctxt, ..keywords::CrateRoot.ident() },
+            span: span,
             parameters: None,
         }
     }
diff --git a/src/libsyntax/ext/base.rs b/src/libsyntax/ext/base.rs
index af5eabf06f8..7a5c9456c53 100644
--- a/src/libsyntax/ext/base.rs
+++ b/src/libsyntax/ext/base.rs
@@ -578,7 +578,10 @@ impl SyntaxExtension {
 
     pub fn is_modern(&self) -> bool {
         match *self {
-            SyntaxExtension::DeclMacro(..) => true,
+            SyntaxExtension::DeclMacro(..) |
+            SyntaxExtension::ProcMacro(..) |
+            SyntaxExtension::AttrProcMacro(..) |
+            SyntaxExtension::ProcMacroDerive(..) => true,
             _ => false,
         }
     }
diff --git a/src/libsyntax/ext/build.rs b/src/libsyntax/ext/build.rs
index a4580ea3939..5168943d108 100644
--- a/src/libsyntax/ext/build.rs
+++ b/src/libsyntax/ext/build.rs
@@ -320,7 +320,7 @@ impl<'a> AstBuilder for ExtCtxt<'a> {
         let last_identifier = idents.pop().unwrap();
         let mut segments: Vec<ast::PathSegment> = Vec::new();
         if global {
-            segments.push(ast::PathSegment::crate_root());
+            segments.push(ast::PathSegment::crate_root(sp));
         }
 
         segments.extend(idents.into_iter().map(|i| ast::PathSegment::from_ident(i, sp)));
diff --git a/src/libsyntax/feature_gate.rs b/src/libsyntax/feature_gate.rs
index d7d3a70f3c7..3f3c94536a6 100644
--- a/src/libsyntax/feature_gate.rs
+++ b/src/libsyntax/feature_gate.rs
@@ -38,12 +38,19 @@ use symbol::Symbol;
 use std::ascii::AsciiExt;
 use std::env;
 
-macro_rules! setter {
+macro_rules! set {
+    (proc_macro) => {{
+        fn f(features: &mut Features, span: Span) {
+            features.declared_lib_features.push((Symbol::intern("proc_macro"), span));
+            features.proc_macro = true;
+        }
+        f as fn(&mut Features, Span)
+    }};
     ($field: ident) => {{
-        fn f(features: &mut Features) -> &mut bool {
-            &mut features.$field
+        fn f(features: &mut Features, _: Span) {
+            features.$field = true;
         }
-        f as fn(&mut Features) -> &mut bool
+        f as fn(&mut Features, Span)
     }}
 }
 
@@ -51,10 +58,9 @@ macro_rules! declare_features {
     ($((active, $feature: ident, $ver: expr, $issue: expr),)+) => {
         /// Represents active features that are currently being implemented or
         /// currently being considered for addition/removal.
-        const ACTIVE_FEATURES: &'static [(&'static str, &'static str,
-                                          Option<u32>, fn(&mut Features) -> &mut bool)] = &[
-            $((stringify!($feature), $ver, $issue, setter!($feature))),+
-        ];
+        const ACTIVE_FEATURES:
+                &'static [(&'static str, &'static str, Option<u32>, fn(&mut Features, Span))] =
+            &[$((stringify!($feature), $ver, $issue, set!($feature))),+];
 
         /// A set of features to be used by later passes.
         pub struct Features {
@@ -1464,9 +1470,9 @@ pub fn get_features(span_handler: &Handler, krate_attrs: &[ast::Attribute]) -> F
                         continue
                     };
 
-                    if let Some(&(_, _, _, setter)) = ACTIVE_FEATURES.iter()
+                    if let Some(&(_, _, _, set)) = ACTIVE_FEATURES.iter()
                         .find(|& &(n, _, _, _)| name == n) {
-                        *(setter(&mut features)) = true;
+                        set(&mut features, mi.span);
                         feature_checker.collect(&features, mi.span);
                     }
                     else if let Some(&(_, _, _)) = REMOVED_FEATURES.iter()
@@ -1500,7 +1506,7 @@ struct MutexFeatureChecker {
 
 impl MutexFeatureChecker {
     // If this method turns out to be a hotspot due to branching,
-    // the branching can be eliminated by modifying `setter!()` to set these spans
+    // the branching can be eliminated by modifying `set!()` to set these spans
     // only for the features that need to be checked for mutual exclusion.
     fn collect(&mut self, features: &Features, span: Span) {
         if features.proc_macro {
diff --git a/src/libsyntax/fold.rs b/src/libsyntax/fold.rs
index 2032aecacbb..ca4814397d8 100644
--- a/src/libsyntax/fold.rs
+++ b/src/libsyntax/fold.rs
@@ -573,7 +573,7 @@ pub fn noop_fold_tt<T: Folder>(tt: TokenTree, fld: &mut T) -> TokenTree {
 }
 
 pub fn noop_fold_tts<T: Folder>(tts: TokenStream, fld: &mut T) -> TokenStream {
-    tts.trees().map(|tt| fld.fold_tt(tt)).collect()
+    tts.map(|tt| fld.fold_tt(tt))
 }
 
 // apply ident folder if it's an ident, apply other folds to interpolated nodes
diff --git a/src/libsyntax/parse/lexer/tokentrees.rs b/src/libsyntax/parse/lexer/tokentrees.rs
index 554a1fcfc71..63a396c14db 100644
--- a/src/libsyntax/parse/lexer/tokentrees.rs
+++ b/src/libsyntax/parse/lexer/tokentrees.rs
@@ -19,7 +19,9 @@ impl<'a> StringReader<'a> {
     pub fn parse_all_token_trees(&mut self) -> PResult<'a, TokenStream> {
         let mut tts = Vec::new();
         while self.token != token::Eof {
-            tts.push(self.parse_token_tree()?.into());
+            let tree = self.parse_token_tree()?;
+            let is_joint = tree.span().hi == self.span.lo && token::is_op(&self.token);
+            tts.push(if is_joint { tree.joint() } else { tree.into() });
         }
         Ok(TokenStream::concat(tts))
     }
@@ -31,13 +33,15 @@ impl<'a> StringReader<'a> {
             if let token::CloseDelim(..) = self.token {
                 return TokenStream::concat(tts);
             }
-            match self.parse_token_tree() {
-                Ok(tt) => tts.push(tt.into()),
+            let tree = match self.parse_token_tree() {
+                Ok(tree) => tree,
                 Err(mut e) => {
                     e.emit();
                     return TokenStream::concat(tts);
                 }
-            }
+            };
+            let is_joint = tree.span().hi == self.span.lo && token::is_op(&self.token);
+            tts.push(if is_joint { tree.joint() } else { tree.into() });
         }
     }
 
diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs
index 25ab46f6f9e..a30dcef6f44 100644
--- a/src/libsyntax/parse/parser.rs
+++ b/src/libsyntax/parse/parser.rs
@@ -1777,7 +1777,7 @@ impl<'a> Parser<'a> {
         };
 
         if is_global {
-            segments.insert(0, PathSegment::crate_root());
+            segments.insert(0, PathSegment::crate_root(lo));
         }
 
         // Assemble the result.
@@ -6187,7 +6187,7 @@ impl<'a> Parser<'a> {
             // `{foo, bar}`, `::{foo, bar}`, `*`, or `::*`.
             self.eat(&token::ModSep);
             let prefix = ast::Path {
-                segments: vec![PathSegment::crate_root()],
+                segments: vec![PathSegment::crate_root(lo)],
                 span: lo.to(self.span),
             };
             let view_path_kind = if self.eat(&token::BinOp(token::Star)) {
diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs
index f208b0f56f8..e568af66e8a 100644
--- a/src/libsyntax/parse/token.rs
+++ b/src/libsyntax/parse/token.rs
@@ -349,6 +349,60 @@ impl Token {
             _ => false,
         }
     }
+
+    pub fn glue(self, joint: Token) -> Option<Token> {
+        Some(match self {
+            Eq => match joint {
+                Eq => EqEq,
+                Gt => FatArrow,
+                _ => return None,
+            },
+            Lt => match joint {
+                Eq => Le,
+                Lt => BinOp(Shl),
+                Le => BinOpEq(Shl),
+                BinOp(Minus) => LArrow,
+                _ => return None,
+            },
+            Gt => match joint {
+                Eq => Ge,
+                Gt => BinOp(Shr),
+                Ge => BinOpEq(Shr),
+                _ => return None,
+            },
+            Not => match joint {
+                Eq => Ne,
+                _ => return None,
+            },
+            BinOp(op) => match joint {
+                Eq => BinOpEq(op),
+                BinOp(And) if op == And => AndAnd,
+                BinOp(Or) if op == Or => OrOr,
+                Gt if op == Minus => RArrow,
+                _ => return None,
+            },
+            Dot => match joint {
+                Dot => DotDot,
+                DotDot => DotDotDot,
+                _ => return None,
+            },
+            DotDot => match joint {
+                Dot => DotDotDot,
+                _ => return None,
+            },
+            Colon => match joint {
+                Colon => ModSep,
+                _ => return None,
+            },
+
+            Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | BinOpEq(..) | At | DotDotDot | Comma |
+            Semi | ModSep | RArrow | LArrow | FatArrow | Pound | Dollar | Question |
+            OpenDelim(..) | CloseDelim(..) | Underscore => return None,
+
+            Literal(..) | Ident(..) | Lifetime(..) | Interpolated(..) | DocComment(..) |
+            Whitespace | Comment | Shebang(..) | Eof => return None,
+        })
+    }
 }
 
 #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash)]
@@ -398,3 +452,12 @@ impl fmt::Debug for Nonterminal {
         }
     }
 }
+
+pub fn is_op(tok: &Token) -> bool {
+    match *tok {
+        OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) |
+        Ident(..) | Underscore | Lifetime(..) | Interpolated(..) |
+        Whitespace | Comment | Shebang(..) | Eof => false,
+        _ => true,
+    }
+}
diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs
index ab4f6970714..2637972cc63 100644
--- a/src/libsyntax/tokenstream.rs
+++ b/src/libsyntax/tokenstream.rs
@@ -138,6 +138,10 @@ impl TokenTree {
             _ => false,
         }
     }
+
+    pub fn joint(self) -> TokenStream {
+        TokenStream { kind: TokenStreamKind::JointTree(self) }
+    }
 }
 
 /// # Token Streams
@@ -155,6 +159,7 @@ pub struct TokenStream {
 enum TokenStreamKind {
     Empty,
     Tree(TokenTree),
+    JointTree(TokenTree),
     Stream(RcSlice<TokenStream>),
 }
 
@@ -196,6 +201,10 @@ impl TokenStream {
         }
     }
 
+    pub fn builder() -> TokenStreamBuilder {
+        TokenStreamBuilder(Vec::new())
+    }
+
     pub fn concat(mut streams: Vec<TokenStream>) -> TokenStream {
         match streams.len() {
             0 => TokenStream::empty(),
@@ -225,6 +234,99 @@ impl TokenStream {
         }
         true
     }
+
+    pub fn as_tree(self) -> (TokenTree, bool /* joint? */) {
+        match self.kind {
+            TokenStreamKind::Tree(tree) => (tree, false),
+            TokenStreamKind::JointTree(tree) => (tree, true),
+            _ => unreachable!(),
+        }
+    }
+
+    pub fn map<F: FnMut(TokenTree) -> TokenTree>(self, mut f: F) -> TokenStream {
+        let mut trees = self.into_trees();
+        let mut result = Vec::new();
+        while let Some(stream) = trees.next_as_stream() {
+            result.push(match stream.kind {
+                TokenStreamKind::Tree(tree) => f(tree).into(),
+                TokenStreamKind::JointTree(tree) => f(tree).joint(),
+                _ => unreachable!()
+            });
+        }
+        TokenStream::concat(result)
+    }
+
+    fn first_tree(&self) -> Option<TokenTree> {
+        match self.kind {
+            TokenStreamKind::Empty => None,
+            TokenStreamKind::Tree(ref tree) |
+            TokenStreamKind::JointTree(ref tree) => Some(tree.clone()),
+            TokenStreamKind::Stream(ref stream) => stream.first().unwrap().first_tree(),
+        }
+    }
+
+    fn last_tree_if_joint(&self) -> Option<TokenTree> {
+        match self.kind {
+            TokenStreamKind::Empty | TokenStreamKind::Tree(..) => None,
+            TokenStreamKind::JointTree(ref tree) => Some(tree.clone()),
+            TokenStreamKind::Stream(ref stream) => stream.last().unwrap().last_tree_if_joint(),
+        }
+    }
+}
+
+pub struct TokenStreamBuilder(Vec<TokenStream>);
+
+impl TokenStreamBuilder {
+    pub fn push<T: Into<TokenStream>>(&mut self, stream: T) {
+        let stream = stream.into();
+        let last_tree_if_joint = self.0.last().and_then(TokenStream::last_tree_if_joint);
+        if let Some(TokenTree::Token(last_span, last_tok)) = last_tree_if_joint {
+            if let Some(TokenTree::Token(span, tok)) = stream.first_tree() {
+                if let Some(glued_tok) = last_tok.glue(tok) {
+                    let last_stream = self.0.pop().unwrap();
+                    self.push_all_but_last_tree(&last_stream);
+                    let glued_span = last_span.to(span);
+                    self.0.push(TokenTree::Token(glued_span, glued_tok).into());
+                    self.push_all_but_first_tree(&stream);
+                    return
+                }
+            }
+        }
+        self.0.push(stream);
+    }
+
+    pub fn add<T: Into<TokenStream>>(mut self, stream: T) -> Self {
+        self.push(stream);
+        self
+    }
+
+    pub fn build(self) -> TokenStream {
+        TokenStream::concat(self.0)
+    }
+
+    fn push_all_but_last_tree(&mut self, stream: &TokenStream) {
+        if let TokenStreamKind::Stream(ref streams) = stream.kind {
+            let len = streams.len();
+            match len {
+                1 => {}
+                2 => self.0.push(streams[0].clone().into()),
+                _ => self.0.push(TokenStream::concat_rc_slice(streams.sub_slice(0 .. len - 1))),
+            }
+            self.push_all_but_last_tree(&streams[len - 1])
+        }
+    }
+
+    fn push_all_but_first_tree(&mut self, stream: &TokenStream) {
+        if let TokenStreamKind::Stream(ref streams) = stream.kind {
+            let len = streams.len();
+            match len {
+                1 => {}
+                2 => self.0.push(streams[1].clone().into()),
+                _ => self.0.push(TokenStream::concat_rc_slice(streams.sub_slice(1 .. len))),
+            }
+            self.push_all_but_first_tree(&streams[0])
+        }
+    }
 }
 
 #[derive(Clone)]
@@ -234,6 +336,7 @@ pub struct Cursor(CursorKind);
 enum CursorKind {
     Empty,
     Tree(TokenTree, bool /* consumed? */),
+    JointTree(TokenTree, bool /* consumed? */),
     Stream(StreamCursor),
 }
 
@@ -245,12 +348,13 @@ struct StreamCursor {
 }
 
 impl StreamCursor {
-    fn next(&mut self) -> Option<TokenTree> {
+    fn next_as_stream(&mut self) -> Option<TokenStream> {
         loop {
             if self.index < self.stream.len() {
                 self.index += 1;
-                match self.stream[self.index - 1].kind.clone() {
-                    TokenStreamKind::Tree(tree) => return Some(tree),
+                let next = self.stream[self.index - 1].clone();
+                match next.kind {
+                    TokenStreamKind::Tree(..) | TokenStreamKind::JointTree(..) => return Some(next),
                     TokenStreamKind::Stream(stream) => {
                         self.stack.push((mem::replace(&mut self.stream, stream),
                                          mem::replace(&mut self.index, 0)));
@@ -271,14 +375,10 @@ impl Iterator for Cursor {
     type Item = TokenTree;
 
     fn next(&mut self) -> Option<TokenTree> {
-        let (tree, consumed) = match self.0 {
-            CursorKind::Tree(ref tree, ref mut consumed @ false) => (tree, consumed),
-            CursorKind::Stream(ref mut cursor) => return cursor.next(),
-            _ => return None,
-        };
-
-        *consumed = true;
-        Some(tree.clone())
+        self.next_as_stream().map(|stream| match stream.kind {
+            TokenStreamKind::Tree(tree) | TokenStreamKind::JointTree(tree) => tree,
+            _ => unreachable!()
+        })
     }
 }
 
@@ -287,16 +387,32 @@ impl Cursor {
         Cursor(match stream.kind {
             TokenStreamKind::Empty => CursorKind::Empty,
             TokenStreamKind::Tree(tree) => CursorKind::Tree(tree, false),
+            TokenStreamKind::JointTree(tree) => CursorKind::JointTree(tree, false),
             TokenStreamKind::Stream(stream) => {
                 CursorKind::Stream(StreamCursor { stream: stream, index: 0, stack: Vec::new() })
             }
         })
     }
 
+    pub fn next_as_stream(&mut self) -> Option<TokenStream> {
+        let (stream, consumed) = match self.0 {
+            CursorKind::Tree(ref tree, ref mut consumed @ false) =>
+                (tree.clone().into(), consumed),
+            CursorKind::JointTree(ref tree, ref mut consumed @ false) =>
+                (tree.clone().joint(), consumed),
+            CursorKind::Stream(ref mut cursor) => return cursor.next_as_stream(),
+            _ => return None,
+        };
+
+        *consumed = true;
+        Some(stream)
+    }
+
     pub fn original_stream(self) -> TokenStream {
         match self.0 {
             CursorKind::Empty => TokenStream::empty(),
             CursorKind::Tree(tree, _) => tree.into(),
+            CursorKind::JointTree(tree, _) => tree.joint(),
             CursorKind::Stream(cursor) => TokenStream::concat_rc_slice({
                 cursor.stack.get(0).cloned().map(|(stream, _)| stream).unwrap_or(cursor.stream)
             }),
@@ -307,8 +423,9 @@ impl Cursor {
         fn look_ahead(streams: &[TokenStream], mut n: usize) -> Result<TokenTree, usize> {
             for stream in streams {
                 n = match stream.kind {
-                    TokenStreamKind::Tree(ref tree) if n == 0 => return Ok(tree.clone()),
-                    TokenStreamKind::Tree(..) => n - 1,
+                    TokenStreamKind::Tree(ref tree) | TokenStreamKind::JointTree(ref tree)
+                        if n == 0 => return Ok(tree.clone()),
+                    TokenStreamKind::Tree(..) | TokenStreamKind::JointTree(..) => n - 1,
                     TokenStreamKind::Stream(ref stream) => match look_ahead(stream, n) {
                         Ok(tree) => return Ok(tree),
                         Err(n) => n,
@@ -316,13 +433,15 @@ impl Cursor {
                     _ => n,
                 };
             }
-
             Err(n)
         }
 
         match self.0 {
-            CursorKind::Empty | CursorKind::Tree(_, true) => Err(n),
-            CursorKind::Tree(ref tree, false) => look_ahead(&[tree.clone().into()], n),
+            CursorKind::Empty |
+            CursorKind::Tree(_, true) |
+            CursorKind::JointTree(_, true) => Err(n),
+            CursorKind::Tree(ref tree, false) |
+            CursorKind::JointTree(ref tree, false) => look_ahead(&[tree.clone().into()], n),
             CursorKind::Stream(ref cursor) => {
                 look_ahead(&cursor.stream[cursor.index ..], n).or_else(|mut n| {
                     for &(ref stream, index) in cursor.stack.iter().rev() {
@@ -350,6 +469,7 @@ impl From<TokenStream> for ThinTokenStream {
         ThinTokenStream(match stream.kind {
             TokenStreamKind::Empty => None,
             TokenStreamKind::Tree(tree) => Some(RcSlice::new(vec![tree.into()])),
+            TokenStreamKind::JointTree(tree) => Some(RcSlice::new(vec![tree.joint()])),
             TokenStreamKind::Stream(stream) => Some(stream),
         })
     }
diff --git a/src/libsyntax/util/rc_slice.rs b/src/libsyntax/util/rc_slice.rs
index 2d9fd7aa875..d6939d71129 100644
--- a/src/libsyntax/util/rc_slice.rs
+++ b/src/libsyntax/util/rc_slice.rs
@@ -9,7 +9,7 @@
 // except according to those terms.
 
 use std::fmt;
-use std::ops::Deref;
+use std::ops::{Deref, Range};
 use std::rc::Rc;
 
 use rustc_data_structures::stable_hasher::{StableHasher, StableHasherResult,
@@ -30,6 +30,14 @@ impl<T> RcSlice<T> {
             data: Rc::new(vec.into_boxed_slice()),
         }
     }
+
+    pub fn sub_slice(&self, range: Range<usize>) -> Self {
+        RcSlice {
+            data: self.data.clone(),
+            offset: self.offset + range.start as u32,
+            len: (range.end - range.start) as u32,
+        }
+    }
 }
 
 impl<T> Deref for RcSlice<T> {