Merge remote-tracking branch 'origin/master' into gen

14 files changed, 236 insertions, 43 deletions

diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs
index 0128bf73a35..cce428cad6d 100644
--- a/src/libsyntax/ast.rs
+++ b/src/libsyntax/ast.rs
@@ -336,6 +336,7 @@ impl Default for Generics {
             where_clause: WhereClause {
                 id: DUMMY_NODE_ID,
                 predicates: Vec::new(),
+                span: DUMMY_SP,
             },
             span: DUMMY_SP,
         }
@@ -347,6 +348,7 @@ impl Default for Generics {
 pub struct WhereClause {
     pub id: NodeId,
     pub predicates: Vec<WherePredicate>,
+    pub span: Span,
 }
 
 /// A single predicate in a `where` clause
@@ -733,6 +735,13 @@ impl Stmt {
         };
         self
     }
+
+    pub fn is_item(&self) -> bool {
+        match self.node {
+            StmtKind::Local(_) => true,
+            _ => false,
+        }
+    }
 }
 
 impl fmt::Debug for Stmt {
@@ -1152,6 +1161,8 @@ pub struct TraitItem {
     pub attrs: Vec<Attribute>,
     pub node: TraitItemKind,
     pub span: Span,
+    /// See `Item::tokens` for what this is
+    pub tokens: Option<TokenStream>,
 }
 
 #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)]
@@ -1171,6 +1182,8 @@ pub struct ImplItem {
     pub attrs: Vec<Attribute>,
     pub node: ImplItemKind,
     pub span: Span,
+    /// See `Item::tokens` for what this is
+    pub tokens: Option<TokenStream>,
 }
 
 #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)]
@@ -1815,6 +1828,15 @@ pub struct Item {
     pub node: ItemKind,
     pub vis: Visibility,
     pub span: Span,
+
+    /// Original tokens this item was parsed from. This isn't necessarily
+    /// available for all items, although over time more and more items should
+    /// have this be `Some`. Right now this is primarily used for procedural
+    /// macros, notably custom attributes.
+    ///
+    /// Note that the tokens here do not include the outer attributes, but will
+    /// include inner attributes.
+    pub tokens: Option<TokenStream>,
 }
 
 #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)]
diff --git a/src/libsyntax/codemap.rs b/src/libsyntax/codemap.rs
index b3d9cf9da36..bfdcae7641d 100644
--- a/src/libsyntax/codemap.rs
+++ b/src/libsyntax/codemap.rs
@@ -561,8 +561,9 @@ impl CodeMapper for CodeMap {
         sp
     }
     fn ensure_filemap_source_present(&self, file_map: Rc<FileMap>) -> bool {
-        let src = self.file_loader.read_file(Path::new(&file_map.name)).ok();
-        return file_map.add_external_src(src)
+        file_map.add_external_src(
+            || self.file_loader.read_file(Path::new(&file_map.name)).ok()
+        )
     }
 }
 
diff --git a/src/libsyntax/diagnostic_list.rs b/src/libsyntax/diagnostic_list.rs
index 508feca9731..6598ecb9444 100644
--- a/src/libsyntax/diagnostic_list.rs
+++ b/src/libsyntax/diagnostic_list.rs
@@ -42,7 +42,7 @@ The `inline` attribute was malformed.
 
 Erroneous code example:
 
-```compile_fail,E0534
+```ignore (compile_fail not working here; see Issue #43707)
 #[inline()] // error: expected one argument
 pub fn something() {}
 
@@ -80,7 +80,7 @@ An unknown argument was given to the `inline` attribute.
 
 Erroneous code example:
 
-```compile_fail,E0535
+```ignore (compile_fail not working here; see Issue #43707)
 #[inline(unknown)] // error: invalid argument
 pub fn something() {}
 
@@ -190,7 +190,9 @@ A literal was used in an attribute that doesn't support literals.
 
 Erroneous code example:
 
-```compile_fail,E0565
+```ignore (compile_fail not working here; see Issue #43707)
+#![feature(attr_literals)]
+
 #[inline("always")] // error: unsupported literal
 pub fn something() {}
 ```
@@ -209,7 +211,7 @@ A file wasn't found for an out-of-line module.
 
 Erroneous code example:
 
-```compile_fail,E0583
+```ignore (compile_fail not working here; see Issue #43707)
 mod file_that_doesnt_exist; // error: file not found for module
 
 fn main() {}
@@ -251,23 +253,33 @@ An inclusive range was used with no end.
 Erroneous code example:
 
 ```compile_fail,E0586
-let tmp = vec![0, 1, 2, 3, 4, 4, 3, 3, 2, 1];
-let x = &tmp[1...]; // error: inclusive range was used with no end
+#![feature(inclusive_range_syntax)]
+
+fn main() {
+    let tmp = vec![0, 1, 2, 3, 4, 4, 3, 3, 2, 1];
+    let x = &tmp[1...]; // error: inclusive range was used with no end
+}
 ```
 
 An inclusive range needs an end in order to *include* it. If you just need a
 start and no end, use a non-inclusive range (with `..`):
 
 ```
-let tmp = vec![0, 1, 2, 3, 4, 4, 3, 3, 2, 1];
-let x = &tmp[1..]; // ok!
+fn main() {
+    let tmp = vec![0, 1, 2, 3, 4, 4, 3, 3, 2, 1];
+    let x = &tmp[1..]; // ok!
+}
 ```
 
 Or put an end to your inclusive range:
 
 ```
-let tmp = vec![0, 1, 2, 3, 4, 4, 3, 3, 2, 1];
-let x = &tmp[1...3]; // ok!
+#![feature(inclusive_range_syntax)]
+
+fn main() {
+    let tmp = vec![0, 1, 2, 3, 4, 4, 3, 3, 2, 1];
+    let x = &tmp[1...3]; // ok!
+}
 ```
 "##,
 
diff --git a/src/libsyntax/diagnostics/plugin.rs b/src/libsyntax/diagnostics/plugin.rs
index 2a5de3c7382..855f4cd3557 100644
--- a/src/libsyntax/diagnostics/plugin.rs
+++ b/src/libsyntax/diagnostics/plugin.rs
@@ -236,6 +236,7 @@ pub fn expand_build_diagnostic_array<'cx>(ecx: &'cx mut ExtCtxt,
             ),
             vis: ast::Visibility::Public,
             span: span,
+            tokens: None,
         })
     ]))
 }
diff --git a/src/libsyntax/ext/build.rs b/src/libsyntax/ext/build.rs
index e004f7354eb..de0538e38b3 100644
--- a/src/libsyntax/ext/build.rs
+++ b/src/libsyntax/ext/build.rs
@@ -979,7 +979,8 @@ impl<'a> AstBuilder for ExtCtxt<'a> {
             id: ast::DUMMY_NODE_ID,
             node: node,
             vis: ast::Visibility::Inherited,
-            span: span
+            span: span,
+            tokens: None,
         })
     }
 
@@ -1147,7 +1148,8 @@ impl<'a> AstBuilder for ExtCtxt<'a> {
             attrs: vec![],
             node: ast::ItemKind::Use(vp),
             vis: vis,
-            span: sp
+            span: sp,
+            tokens: None,
         })
     }
 
diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index f6d56557166..16c264e0f94 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -214,6 +214,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
             ident: keywords::Invalid.ident(),
             id: ast::DUMMY_NODE_ID,
             vis: ast::Visibility::Public,
+            tokens: None,
         })));
 
         match self.expand(krate_item).make_items().pop().map(P::unwrap) {
diff --git a/src/libsyntax/ext/placeholders.rs b/src/libsyntax/ext/placeholders.rs
index 4fb138d506a..e3377c1d8de 100644
--- a/src/libsyntax/ext/placeholders.rs
+++ b/src/libsyntax/ext/placeholders.rs
@@ -46,15 +46,18 @@ pub fn placeholder(kind: ExpansionKind, id: ast::NodeId) -> Expansion {
         ExpansionKind::Items => Expansion::Items(SmallVector::one(P(ast::Item {
             id: id, span: span, ident: ident, vis: vis, attrs: attrs,
             node: ast::ItemKind::Mac(mac_placeholder()),
+            tokens: None,
         }))),
         ExpansionKind::TraitItems => Expansion::TraitItems(SmallVector::one(ast::TraitItem {
             id: id, span: span, ident: ident, attrs: attrs,
             node: ast::TraitItemKind::Macro(mac_placeholder()),
+            tokens: None,
         })),
         ExpansionKind::ImplItems => Expansion::ImplItems(SmallVector::one(ast::ImplItem {
             id: id, span: span, ident: ident, vis: vis, attrs: attrs,
             node: ast::ImplItemKind::Macro(mac_placeholder()),
             defaultness: ast::Defaultness::Final,
+            tokens: None,
         })),
         ExpansionKind::Pat => Expansion::Pat(P(ast::Pat {
             id: id, span: span, node: ast::PatKind::Mac(mac_placeholder()),
diff --git a/src/libsyntax/fold.rs b/src/libsyntax/fold.rs
index 580c2aa58a5..8f9f179f08b 100644
--- a/src/libsyntax/fold.rs
+++ b/src/libsyntax/fold.rs
@@ -737,14 +737,15 @@ pub fn noop_fold_generics<T: Folder>(Generics {ty_params, lifetimes, where_claus
 }
 
 pub fn noop_fold_where_clause<T: Folder>(
-                              WhereClause {id, predicates}: WhereClause,
+                              WhereClause {id, predicates, span}: WhereClause,
                               fld: &mut T)
                               -> WhereClause {
     WhereClause {
         id: fld.new_id(id),
         predicates: predicates.move_map(|predicate| {
             fld.fold_where_predicate(predicate)
-        })
+        }),
+        span: span,
     }
 }
 
@@ -957,7 +958,8 @@ pub fn noop_fold_trait_item<T: Folder>(i: TraitItem, folder: &mut T)
                 TraitItemKind::Macro(folder.fold_mac(mac))
             }
         },
-        span: folder.new_span(i.span)
+        span: folder.new_span(i.span),
+        tokens: i.tokens,
     })
 }
 
@@ -980,7 +982,8 @@ pub fn noop_fold_impl_item<T: Folder>(i: ImplItem, folder: &mut T)
             ast::ImplItemKind::Type(ty) => ast::ImplItemKind::Type(folder.fold_ty(ty)),
             ast::ImplItemKind::Macro(mac) => ast::ImplItemKind::Macro(folder.fold_mac(mac))
         },
-        span: folder.new_span(i.span)
+        span: folder.new_span(i.span),
+        tokens: i.tokens,
     })
 }
 
@@ -1000,6 +1003,7 @@ pub fn noop_fold_crate<T: Folder>(Crate {module, attrs, span}: Crate,
         vis: ast::Visibility::Public,
         span: span,
         node: ast::ItemKind::Mod(module),
+        tokens: None,
     })).into_iter();
 
     let (module, attrs, span) = match items.next() {
@@ -1032,7 +1036,7 @@ pub fn noop_fold_item<T: Folder>(i: P<Item>, folder: &mut T) -> SmallVector<P<It
 }
 
 // fold one item into exactly one item
-pub fn noop_fold_item_simple<T: Folder>(Item {id, ident, attrs, node, vis, span}: Item,
+pub fn noop_fold_item_simple<T: Folder>(Item {id, ident, attrs, node, vis, span, tokens}: Item,
                                         folder: &mut T) -> Item {
     Item {
         id: folder.new_id(id),
@@ -1040,7 +1044,11 @@ pub fn noop_fold_item_simple<T: Folder>(Item {id, ident, attrs, node, vis, span}
         ident: folder.fold_ident(ident),
         attrs: fold_attrs(attrs, folder),
         node: folder.fold_item_kind(node),
-        span: folder.new_span(span)
+        span: folder.new_span(span),
+
+        // FIXME: if this is replaced with a call to `folder.fold_tts` it causes
+        //        an ICE during resolve... odd!
+        tokens: tokens,
     }
 }
 
diff --git a/src/libsyntax/lib.rs b/src/libsyntax/lib.rs
index a8338fccb6b..43345b02bf6 100644
--- a/src/libsyntax/lib.rs
+++ b/src/libsyntax/lib.rs
@@ -148,4 +148,4 @@ pub mod ext {
 #[cfg(test)]
 mod test_snippet;
 
-// __build_diagnostic_array! { libsyntax, DIAGNOSTICS }
+__build_diagnostic_array! { libsyntax, DIAGNOSTICS }
diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs
index bd9a621c00c..893bada2670 100644
--- a/src/libsyntax/parse/mod.rs
+++ b/src/libsyntax/parse/mod.rs
@@ -843,11 +843,18 @@ mod tests {
     // check the contents of the tt manually:
     #[test] fn parse_fundecl () {
         // this test depends on the intern order of "fn" and "i32"
-        assert_eq!(string_to_item("fn a (b : i32) { b; }".to_string()),
+        let item = string_to_item("fn a (b : i32) { b; }".to_string()).map(|m| {
+            m.map(|mut m| {
+                m.tokens = None;
+                m
+            })
+        });
+        assert_eq!(item,
                   Some(
                       P(ast::Item{ident:Ident::from_str("a"),
                             attrs:Vec::new(),
                             id: ast::DUMMY_NODE_ID,
+                            tokens: None,
                             node: ast::ItemKind::Fn(P(ast::FnDecl {
                                 inputs: vec![ast::Arg{
                                     ty: P(ast::Ty{id: ast::DUMMY_NODE_ID,
@@ -860,13 +867,14 @@ mod tests {
                                     pat: P(ast::Pat {
                                         id: ast::DUMMY_NODE_ID,
                                         node: PatKind::Ident(
-                                            ast::BindingMode::ByValue(ast::Mutability::Immutable),
-                                                Spanned{
-                                                    span: sp(6,7),
-                                                    node: Ident::from_str("b")},
-                                                None
-                                                    ),
-                                            span: sp(6,7)
+                                            ast::BindingMode::ByValue(
+                                                ast::Mutability::Immutable),
+                                            Spanned{
+                                                span: sp(6,7),
+                                                node: Ident::from_str("b")},
+                                            None
+                                        ),
+                                        span: sp(6,7)
                                     }),
                                         id: ast::DUMMY_NODE_ID
                                     }],
@@ -885,6 +893,7 @@ mod tests {
                                         where_clause: ast::WhereClause {
                                             id: ast::DUMMY_NODE_ID,
                                             predicates: Vec::new(),
+                                            span: syntax_pos::DUMMY_SP,
                                         },
                                         span: syntax_pos::DUMMY_SP,
                                     },
diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs
index b7ae025db5f..5fe3cf0ddac 100644
--- a/src/libsyntax/parse/parser.rs
+++ b/src/libsyntax/parse/parser.rs
@@ -216,6 +216,30 @@ struct TokenCursorFrame {
     open_delim: bool,
     tree_cursor: tokenstream::Cursor,
     close_delim: bool,
+    last_token: LastToken,
+}
+
+/// This is used in `TokenCursorFrame` above to track tokens that are consumed
+/// by the parser, and then that's transitively used to record the tokens that
+/// each parse AST item is created with.
+///
+/// Right now this has two states, either collecting tokens or not collecting
+/// tokens. If we're collecting tokens we just save everything off into a local
+/// `Vec`. This should eventually though likely save tokens from the original
+/// token stream and just use slicing of token streams to avoid creation of a
+/// whole new vector.
+///
+/// The second state is where we're passively not recording tokens, but the last
+/// token is still tracked for when we want to start recording tokens. This
+/// "last token" means that when we start recording tokens we'll want to ensure
+/// that this, the first token, is included in the output.
+///
+/// You can find some more example usage of this in the `collect_tokens` method
+/// on the parser.
+#[derive(Clone)]
+enum LastToken {
+    Collecting(Vec<TokenTree>),
+    Was(Option<TokenTree>),
 }
 
 impl TokenCursorFrame {
@@ -226,6 +250,7 @@ impl TokenCursorFrame {
             open_delim: delimited.delim == token::NoDelim,
             tree_cursor: delimited.stream().into_trees(),
             close_delim: delimited.delim == token::NoDelim,
+            last_token: LastToken::Was(None),
         }
     }
 }
@@ -250,6 +275,11 @@ impl TokenCursor {
                 return TokenAndSpan { tok: token::Eof, sp: syntax_pos::DUMMY_SP }
             };
 
+            match self.frame.last_token {
+                LastToken::Collecting(ref mut v) => v.push(tree.clone()),
+                LastToken::Was(ref mut t) => *t = Some(tree.clone()),
+            }
+
             match tree {
                 TokenTree::Token(sp, tok) => return TokenAndSpan { tok: tok, sp: sp },
                 TokenTree::Delimited(sp, ref delimited) => {
@@ -1209,7 +1239,20 @@ impl<'a> Parser<'a> {
     /// Parse the items in a trait declaration
     pub fn parse_trait_item(&mut self, at_end: &mut bool) -> PResult<'a, TraitItem> {
         maybe_whole!(self, NtTraitItem, |x| x);
-        let mut attrs = self.parse_outer_attributes()?;
+        let attrs = self.parse_outer_attributes()?;
+        let (mut item, tokens) = self.collect_tokens(|this| {
+            this.parse_trait_item_(at_end, attrs)
+        })?;
+        // See `parse_item` for why this clause is here.
+        if !item.attrs.iter().any(|attr| attr.style == AttrStyle::Inner) {
+            item.tokens = Some(tokens);
+        }
+        Ok(item)
+    }
+
+    fn parse_trait_item_(&mut self,
+                         at_end: &mut bool,
+                         mut attrs: Vec<Attribute>) -> PResult<'a, TraitItem> {
         let lo = self.span;
 
         let (name, node) = if self.eat_keyword(keywords::Type) {
@@ -1304,6 +1347,7 @@ impl<'a> Parser<'a> {
             attrs: attrs,
             node: node,
             span: lo.to(self.prev_span),
+            tokens: None,
         })
     }
 
@@ -2030,14 +2074,14 @@ impl<'a> Parser<'a> {
                 } else {
                     Ok(self.mk_expr(span, ExprKind::Tup(es), attrs))
                 }
-            },
+            }
             token::OpenDelim(token::Brace) => {
                 return self.parse_block_expr(lo, BlockCheckMode::Default, attrs);
-            },
-            token::BinOp(token::Or) |  token::OrOr => {
+            }
+            token::BinOp(token::Or) | token::OrOr => {
                 let lo = self.span;
                 return self.parse_lambda_expr(lo, CaptureBy::Ref, attrs);
-            },
+            }
             token::OpenDelim(token::Bracket) => {
                 self.bump();
 
@@ -2293,7 +2337,6 @@ impl<'a> Parser<'a> {
     pub fn parse_block_expr(&mut self, lo: Span, blk_mode: BlockCheckMode,
                             outer_attrs: ThinVec<Attribute>)
                             -> PResult<'a, P<Expr>> {
-
         self.expect(&token::OpenDelim(token::Brace))?;
 
         let mut attrs = outer_attrs;
@@ -4266,6 +4309,7 @@ impl<'a> Parser<'a> {
                 where_clause: WhereClause {
                     id: ast::DUMMY_NODE_ID,
                     predicates: Vec::new(),
+                    span: syntax_pos::DUMMY_SP,
                 },
                 span: span_lo.to(self.prev_span),
             })
@@ -4333,11 +4377,13 @@ impl<'a> Parser<'a> {
         let mut where_clause = WhereClause {
             id: ast::DUMMY_NODE_ID,
             predicates: Vec::new(),
+            span: syntax_pos::DUMMY_SP,
         };
 
         if !self.eat_keyword(keywords::Where) {
             return Ok(where_clause);
         }
+        let lo = self.prev_span;
 
         // This is a temporary future proofing.
         //
@@ -4415,6 +4461,7 @@ impl<'a> Parser<'a> {
             }
         }
 
+        where_clause.span = lo.to(self.prev_span);
         Ok(where_clause)
     }
 
@@ -4661,6 +4708,7 @@ impl<'a> Parser<'a> {
             node: node,
             vis: vis,
             span: span,
+            tokens: None,
         })
     }
 
@@ -4716,8 +4764,21 @@ impl<'a> Parser<'a> {
     /// Parse an impl item.
     pub fn parse_impl_item(&mut self, at_end: &mut bool) -> PResult<'a, ImplItem> {
         maybe_whole!(self, NtImplItem, |x| x);
+        let attrs = self.parse_outer_attributes()?;
+        let (mut item, tokens) = self.collect_tokens(|this| {
+            this.parse_impl_item_(at_end, attrs)
+        })?;
 
-        let mut attrs = self.parse_outer_attributes()?;
+        // See `parse_item` for why this clause is here.
+        if !item.attrs.iter().any(|attr| attr.style == AttrStyle::Inner) {
+            item.tokens = Some(tokens);
+        }
+        Ok(item)
+    }
+
+    fn parse_impl_item_(&mut self,
+                        at_end: &mut bool,
+                        mut attrs: Vec<Attribute>) -> PResult<'a, ImplItem> {
         let lo = self.span;
         let vis = self.parse_visibility(false)?;
         let defaultness = self.parse_defaultness()?;
@@ -4749,7 +4810,8 @@ impl<'a> Parser<'a> {
             vis: vis,
             defaultness: defaultness,
             attrs: attrs,
-            node: node
+            node: node,
+            tokens: None,
         })
     }
 
@@ -6025,9 +6087,71 @@ impl<'a> Parser<'a> {
         Ok(None)
     }
 
+    fn collect_tokens<F, R>(&mut self, f: F) -> PResult<'a, (R, TokenStream)>
+        where F: FnOnce(&mut Self) -> PResult<'a, R>
+    {
+        // Record all tokens we parse when parsing this item.
+        let mut tokens = Vec::new();
+        match self.token_cursor.frame.last_token {
+            LastToken::Collecting(_) => {
+                panic!("cannot collect tokens recursively yet")
+            }
+            LastToken::Was(ref mut last) => tokens.extend(last.take()),
+        }
+        self.token_cursor.frame.last_token = LastToken::Collecting(tokens);
+        let prev = self.token_cursor.stack.len();
+        let ret = f(self);
+        let last_token = if self.token_cursor.stack.len() == prev {
+            &mut self.token_cursor.frame.last_token
+        } else {
+            &mut self.token_cursor.stack[prev].last_token
+        };
+        let mut tokens = match *last_token {
+            LastToken::Collecting(ref mut v) => mem::replace(v, Vec::new()),
+            LastToken::Was(_) => panic!("our vector went away?"),
+        };
+
+        // If we're not at EOF our current token wasn't actually consumed by
+        // `f`, but it'll still be in our list that we pulled out. In that case
+        // put it back.
+        if self.token == token::Eof {
+            *last_token = LastToken::Was(None);
+        } else {
+            *last_token = LastToken::Was(tokens.pop());
+        }
+
+        Ok((ret?, tokens.into_iter().collect()))
+    }
+
     pub fn parse_item(&mut self) -> PResult<'a, Option<P<Item>>> {
         let attrs = self.parse_outer_attributes()?;
-        self.parse_item_(attrs, true, false)
+
+        let (ret, tokens) = self.collect_tokens(|this| {
+            this.parse_item_(attrs, true, false)
+        })?;
+
+        // Once we've parsed an item and recorded the tokens we got while
+        // parsing we may want to store `tokens` into the item we're about to
+        // return. Note, though, that we specifically didn't capture tokens
+        // related to outer attributes. The `tokens` field here may later be
+        // used with procedural macros to convert this item back into a token
+        // stream, but during expansion we may be removing attributes as we go
+        // along.
+        //
+        // If we've got inner attributes then the `tokens` we've got above holds
+        // these inner attributes. If an inner attribute is expanded we won't
+        // actually remove it from the token stream, so we'll just keep yielding
+        // it (bad!). To work around this case for now we just avoid recording
+        // `tokens` if we detect any inner attributes. This should help keep
+        // expansion correct, but we should fix this bug one day!
+        Ok(ret.map(|item| {
+            item.map(|mut i| {
+                if !i.attrs.iter().any(|attr| attr.style == AttrStyle::Inner) {
+                    i.tokens = Some(tokens);
+                }
+                i
+            })
+        }))
     }
 
     fn parse_path_list_items(&mut self) -> PResult<'a, Vec<ast::PathListItem>> {
diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs
index 9e36fb83696..5832cfcdf36 100644
--- a/src/libsyntax/print/pprust.rs
+++ b/src/libsyntax/print/pprust.rs
@@ -1041,6 +1041,7 @@ impl<'a> State<'a> {
                     where_clause: ast::WhereClause {
                         id: ast::DUMMY_NODE_ID,
                         predicates: Vec::new(),
+                        span: syntax_pos::DUMMY_SP,
                     },
                     span: syntax_pos::DUMMY_SP,
                 };
@@ -2993,6 +2994,7 @@ impl<'a> State<'a> {
             where_clause: ast::WhereClause {
                 id: ast::DUMMY_NODE_ID,
                 predicates: Vec::new(),
+                span: syntax_pos::DUMMY_SP,
             },
             span: syntax_pos::DUMMY_SP,
         };
diff --git a/src/libsyntax/std_inject.rs b/src/libsyntax/std_inject.rs
index a8a9ae556f1..d9ed96f293a 100644
--- a/src/libsyntax/std_inject.rs
+++ b/src/libsyntax/std_inject.rs
@@ -60,6 +60,7 @@ pub fn maybe_inject_crates_ref(mut krate: ast::Crate, alt_std_name: Option<Strin
         ident: ast::Ident::from_str(name),
         id: ast::DUMMY_NODE_ID,
         span: DUMMY_SP,
+        tokens: None,
     }));
 
     let span = ignored_span(DUMMY_SP);
@@ -82,6 +83,7 @@ pub fn maybe_inject_crates_ref(mut krate: ast::Crate, alt_std_name: Option<Strin
         id: ast::DUMMY_NODE_ID,
         ident: keywords::Invalid.ident(),
         span: span,
+        tokens: None,
     }));
 
     krate
diff --git a/src/libsyntax/test.rs b/src/libsyntax/test.rs
index bd2c386cb8a..887479a2472 100644
--- a/src/libsyntax/test.rs
+++ b/src/libsyntax/test.rs
@@ -192,7 +192,7 @@ impl fold::Folder for EntryPointCleaner {
             EntryPointType::MainNamed |
             EntryPointType::MainAttr |
             EntryPointType::Start =>
-                folded.map(|ast::Item {id, ident, attrs, node, vis, span}| {
+                folded.map(|ast::Item {id, ident, attrs, node, vis, span, tokens}| {
                     let allow_str = Symbol::intern("allow");
                     let dead_code_str = Symbol::intern("dead_code");
                     let word_vec = vec![attr::mk_list_word_item(dead_code_str)];
@@ -212,7 +212,8 @@ impl fold::Folder for EntryPointCleaner {
                             .collect(),
                         node: node,
                         vis: vis,
-                        span: span
+                        span: span,
+                        tokens: tokens,
                     }
                 }),
             EntryPointType::None |
@@ -255,6 +256,7 @@ fn mk_reexport_mod(cx: &mut TestCtxt,
         node: ast::ItemKind::Mod(reexport_mod),
         vis: ast::Visibility::Public,
         span: DUMMY_SP,
+        tokens: None,
     })).pop().unwrap();
 
     (it, sym)
@@ -465,7 +467,8 @@ fn mk_std(cx: &TestCtxt) -> P<ast::Item> {
         node: vi,
         attrs: vec![],
         vis: vis,
-        span: sp
+        span: sp,
+        tokens: None,
     })
 }
 
@@ -506,7 +509,8 @@ fn mk_main(cx: &mut TestCtxt) -> P<ast::Item> {
         id: ast::DUMMY_NODE_ID,
         node: main,
         vis: ast::Visibility::Public,
-        span: sp
+        span: sp,
+        tokens: None,
     })
 }
 
@@ -536,6 +540,7 @@ fn mk_test_module(cx: &mut TestCtxt) -> (P<ast::Item>, Option<P<ast::Item>>) {
         node: item_,
         vis: ast::Visibility::Public,
         span: DUMMY_SP,
+        tokens: None,
     })).pop().unwrap();
     let reexport = cx.reexport_test_harness_main.map(|s| {
         // building `use <ident> = __test::main`
@@ -551,7 +556,8 @@ fn mk_test_module(cx: &mut TestCtxt) -> (P<ast::Item>, Option<P<ast::Item>>) {
             attrs: vec![],
             node: ast::ItemKind::Use(P(use_path)),
             vis: ast::Visibility::Inherited,
-            span: DUMMY_SP
+            span: DUMMY_SP,
+            tokens: None,
         })).pop().unwrap()
     });