Auto merge of #43230 - alexcrichton:more-tokenstream, r=nrc,jseyfried

Implement tokenization for some items in proc_macro

This PR is a partial implementation of https://github.com/rust-lang/rust/issues/43081 targeted towards preserving span information in attribute-like procedural macros. Currently all attribute-like macros will lose span information with the input token stream if it's iterated over due to the inability of the compiler to losslessly tokenize an AST node. This PR takes a strategy of saving off a list of tokens in particular AST nodes to return a lossless tokenized version. There's a few limitations with this PR, however, so the old fallback remains in place.

4 files changed, 235 insertions, 0 deletions

diff --git a/src/test/compile-fail-fulldeps/proc-macro/attribute-with-error.rs b/src/test/compile-fail-fulldeps/proc-macro/attribute-with-error.rs
new file mode 100644
index 00000000000..65f4b6350c4
--- /dev/null
+++ b/src/test/compile-fail-fulldeps/proc-macro/attribute-with-error.rs
@@ -0,0 +1,51 @@
+// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// aux-build:attribute-with-error.rs
+
+#![feature(proc_macro)]
+
+extern crate attribute_with_error;
+
+use attribute_with_error::foo;
+
+#[foo]
+fn test1() {
+    let a: i32 = "foo";
+    //~^ ERROR: mismatched types
+}
+
+fn test2() {
+    #![foo]
+
+    // FIXME: should have a type error here and assert it works but it doesn't
+}
+
+trait A {
+    // FIXME: should have a #[foo] attribute here and assert that it works
+    fn foo(&self) {
+        let a: i32 = "foo";
+        //~^ ERROR: mismatched types
+    }
+}
+
+struct B;
+
+impl A for B {
+    #[foo]
+    fn foo(&self) {
+        let a: i32 = "foo";
+        //~^ ERROR: mismatched types
+    }
+}
+
+#[foo]
+fn main() {
+}
diff --git a/src/test/compile-fail-fulldeps/proc-macro/attributes-included.rs b/src/test/compile-fail-fulldeps/proc-macro/attributes-included.rs
new file mode 100644
index 00000000000..508f8dac571
--- /dev/null
+++ b/src/test/compile-fail-fulldeps/proc-macro/attributes-included.rs
@@ -0,0 +1,30 @@
+// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// aux-build:attributes-included.rs
+
+#![feature(proc_macro, rustc_attrs)]
+
+extern crate attributes_included;
+
+#[attributes_included::bar]
+#[inline]
+/// doc
+#[attributes_included::foo]
+#[inline]
+/// doc
+fn foo() {
+    let a: i32 = "foo"; //~ WARN: unused variable
+}
+
+#[rustc_error]
+fn main() { //~ ERROR: compilation successful
+    foo()
+}
diff --git a/src/test/compile-fail-fulldeps/proc-macro/auxiliary/attribute-with-error.rs b/src/test/compile-fail-fulldeps/proc-macro/auxiliary/attribute-with-error.rs
new file mode 100644
index 00000000000..85a7a0bf633
--- /dev/null
+++ b/src/test/compile-fail-fulldeps/proc-macro/auxiliary/attribute-with-error.rs
@@ -0,0 +1,24 @@
+// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// force-host
+// no-prefer-dynamic
+
+#![crate_type = "proc-macro"]
+#![feature(proc_macro)]
+
+extern crate proc_macro;
+
+use proc_macro::TokenStream;
+
+#[proc_macro_attribute]
+pub fn foo(_attr: TokenStream, input: TokenStream) -> TokenStream {
+    input.into_iter().collect()
+}
diff --git a/src/test/compile-fail-fulldeps/proc-macro/auxiliary/attributes-included.rs b/src/test/compile-fail-fulldeps/proc-macro/auxiliary/attributes-included.rs
new file mode 100644
index 00000000000..a1efbb88a4d
--- /dev/null
+++ b/src/test/compile-fail-fulldeps/proc-macro/auxiliary/attributes-included.rs
@@ -0,0 +1,130 @@
+// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// force-host
+// no-prefer-dynamic
+
+#![feature(proc_macro)]
+#![crate_type = "proc-macro"]
+
+extern crate proc_macro;
+
+use proc_macro::{TokenStream, TokenTree, TokenNode, Delimiter, Literal};
+
+#[proc_macro_attribute]
+pub fn foo(attr: TokenStream, input: TokenStream) -> TokenStream {
+    assert!(attr.is_empty());
+    let input = input.into_iter().collect::<Vec<_>>();
+    {
+        let mut cursor = &input[..];
+        assert_inline(&mut cursor);
+        assert_doc(&mut cursor);
+        assert_inline(&mut cursor);
+        assert_doc(&mut cursor);
+        assert_foo(&mut cursor);
+        assert!(cursor.is_empty());
+    }
+    fold_stream(input.into_iter().collect())
+}
+
+#[proc_macro_attribute]
+pub fn bar(attr: TokenStream, input: TokenStream) -> TokenStream {
+    assert!(attr.is_empty());
+    let input = input.into_iter().collect::<Vec<_>>();
+    {
+        let mut cursor = &input[..];
+        assert_inline(&mut cursor);
+        assert_doc(&mut cursor);
+        assert_invoc(&mut cursor);
+        assert_inline(&mut cursor);
+        assert_doc(&mut cursor);
+        assert_foo(&mut cursor);
+        assert!(cursor.is_empty());
+    }
+    input.into_iter().collect()
+}
+
+fn assert_inline(slice: &mut &[TokenTree]) {
+    match slice[0].kind {
+        TokenNode::Op('#', _) => {}
+        _ => panic!("expected '#' char"),
+    }
+    match slice[1].kind {
+        TokenNode::Group(Delimiter::Bracket, _) => {}
+        _ => panic!("expected brackets"),
+    }
+    *slice = &slice[2..];
+}
+
+fn assert_doc(slice: &mut &[TokenTree]) {
+    match slice[0].kind {
+        TokenNode::Literal(_) => {}
+        _ => panic!("expected literal doc comment got other"),
+    }
+    *slice = &slice[1..];
+}
+
+fn assert_invoc(slice: &mut &[TokenTree]) {
+    match slice[0].kind {
+        TokenNode::Op('#', _) => {}
+        _ => panic!("expected '#' char"),
+    }
+    match slice[1].kind {
+        TokenNode::Group(Delimiter::Bracket, _) => {}
+        _ => panic!("expected brackets"),
+    }
+    *slice = &slice[2..];
+}
+
+fn assert_foo(slice: &mut &[TokenTree]) {
+    match slice[0].kind {
+        TokenNode::Term(ref name) => assert_eq!(name.as_str(), "fn"),
+        _ => panic!("expected fn"),
+    }
+    match slice[1].kind {
+        TokenNode::Term(ref name) => assert_eq!(name.as_str(), "foo"),
+        _ => panic!("expected foo"),
+    }
+    match slice[2].kind {
+        TokenNode::Group(Delimiter::Parenthesis, ref s) => assert!(s.is_empty()),
+        _ => panic!("expected parens"),
+    }
+    match slice[3].kind {
+        TokenNode::Group(Delimiter::Brace, _) => {}
+        _ => panic!("expected braces"),
+    }
+    *slice = &slice[4..];
+}
+
+fn fold_stream(input: TokenStream) -> TokenStream {
+    input.into_iter().map(fold_tree).collect()
+}
+
+fn fold_tree(input: TokenTree) -> TokenTree {
+    TokenTree {
+        span: input.span,
+        kind: fold_node(input.kind),
+    }
+}
+
+fn fold_node(input: TokenNode) -> TokenNode {
+    match input {
+        TokenNode::Group(a, b) => TokenNode::Group(a, fold_stream(b)),
+        TokenNode::Op(a, b) => TokenNode::Op(a, b),
+        TokenNode::Term(a) => TokenNode::Term(a),
+        TokenNode::Literal(a) => {
+            if a.to_string() != "\"foo\"" {
+                TokenNode::Literal(a)
+            } else {
+                TokenNode::Literal(Literal::integer(3))
+            }
+        }
+    }
+}