Auto merge of #65324 - Centril:organize-syntax, r=petrochenkov

Split libsyntax apart

In this PR the general idea is to separate the AST, parser, and friends by a more data / logic structure (tho not fully realized!) by separating out the parser and macro expansion code from libsyntax. Specifically have now three crates instead of one (libsyntax):

- libsyntax:

   - concrete syntax tree (`syntax::ast`)

   - definition of tokens and token-streams (`syntax::{token, tokenstream}`) -- used by `syntax::ast`

   - visitors (`syntax::visit`, `syntax::mut_visit`)

   - shared definitions between `libsyntax_expand`

   - feature gating (`syntax::feature_gate`) -- we could possibly move this out to its own crater later.

   - attribute and meta item utilities, including used-marking (`syntax::attr`)

   - pretty printer (`syntax::print`) -- this should possibly be moved out later. For now I've reduced down the dependencies to a single essential one which could be broken via `ParseSess`. This entails that e.g. `Debug` impls for `Path` cannot reference the pretty printer.

   - definition of `ParseSess` (`syntax::sess`) -- this is used by `syntax::{attr, print, feature_gate}` and is a common definition used by the parser and other things like librustc.

   - the `syntax::source_map` -- this includes definitions used by `syntax::ast` and other things but could ostensibly be moved `syntax_pos` since that is more related to this module.

   - a smattering of misc utilities not sufficiently important to itemize -- some of these could be moved to where they are used (often a single place) but I wanted to limit the scope of this PR.

- librustc_parse:

   - parser (`rustc_parse::parser`) -- reading a file and such are defined in the crate root tho.

   - lexer (`rustc_parse::lexer`)

   - validation of meta grammar (post-expansion) in (`rustc_parse::validate_attr`)

- libsyntax_expand -- this defines the infra for macro expansion and conditional compilation but this is not libsyntax_ext; we might want to merge them later but currently libsyntax_expand is depended on by librustc_metadata which libsyntax_ext is not.

   - conditional compilation (`syntax_expand::config`) -- moved from `syntax::config` to here

   - the bulk of this crate is made up of the old `syntax::ext`

r? @estebank

4 files changed, 39 insertions, 27 deletions

diff --git a/src/test/ui-fulldeps/ast_stmt_expr_attr.rs b/src/test/ui-fulldeps/ast_stmt_expr_attr.rs
index ac864e76784..0bf17302b06 100644
--- a/src/test/ui-fulldeps/ast_stmt_expr_attr.rs
+++ b/src/test/ui-fulldeps/ast_stmt_expr_attr.rs
@@ -6,21 +6,23 @@
 #![feature(rustc_private)]
 
 extern crate syntax;
+extern crate syntax_expand;
+extern crate rustc_parse;
 extern crate rustc_errors;
 
 use rustc_errors::PResult;
+use rustc_parse::parser::attr::*;
+use rustc_parse::new_parser_from_source_str;
+use rustc_parse::parser::Parser;
 use syntax::ast::*;
 use syntax::attr::*;
 use syntax::ast;
 use syntax::sess::ParseSess;
 use syntax::source_map::{FilePathMapping, FileName};
-use syntax::parse;
-use syntax::parse::new_parser_from_source_str;
-use syntax::parse::parser::Parser;
-use syntax::token;
 use syntax::ptr::P;
-use syntax::parse::parser::attr::*;
 use syntax::print::pprust;
+use syntax::token;
+use syntax_expand::config::process_configure_mod;
 use std::fmt;
 
 // Copied out of syntax::util::parser_testing
@@ -72,8 +74,12 @@ fn str_compare<T, F: Fn(&T) -> String>(e: &str, expected: &[T], actual: &[T], f:
     }
 }
 
+fn sess() -> ParseSess {
+    ParseSess::new(FilePathMapping::empty(), process_configure_mod)
+}
+
 fn check_expr_attrs(es: &str, expected: &[&str]) {
-    let ps = ParseSess::new(FilePathMapping::empty());
+    let ps = sess();
     let e = expr(es, &ps).expect("parse error");
     let actual = &e.attrs;
     str_compare(es,
@@ -83,7 +89,7 @@ fn check_expr_attrs(es: &str, expected: &[&str]) {
 }
 
 fn check_stmt_attrs(es: &str, expected: &[&str]) {
-    let ps = ParseSess::new(FilePathMapping::empty());
+    let ps = sess();
     let e = stmt(es, &ps).expect("parse error");
     let actual = e.kind.attrs();
     str_compare(es,
@@ -93,7 +99,7 @@ fn check_stmt_attrs(es: &str, expected: &[&str]) {
 }
 
 fn reject_expr_parse(es: &str) {
-    let ps = ParseSess::new(FilePathMapping::empty());
+    let ps = sess();
     match expr(es, &ps) {
         Ok(_) => panic!("parser did not reject `{}`", es),
         Err(mut e) => e.cancel(),
@@ -101,7 +107,7 @@ fn reject_expr_parse(es: &str) {
 }
 
 fn reject_stmt_parse(es: &str) {
-    let ps = ParseSess::new(FilePathMapping::empty());
+    let ps = sess();
     match stmt(es, &ps) {
         Ok(_) => panic!("parser did not reject `{}`", es),
         Err(mut e) => e.cancel(),
diff --git a/src/test/ui-fulldeps/mod_dir_path_canonicalized.rs b/src/test/ui-fulldeps/mod_dir_path_canonicalized.rs
index ac97ec70be2..2411b9634c3 100644
--- a/src/test/ui-fulldeps/mod_dir_path_canonicalized.rs
+++ b/src/test/ui-fulldeps/mod_dir_path_canonicalized.rs
@@ -5,11 +5,14 @@
 #![feature(rustc_private)]
 
 extern crate syntax;
+extern crate syntax_expand;
+extern crate rustc_parse;
 
+use rustc_parse::new_parser_from_file;
 use std::path::Path;
 use syntax::sess::ParseSess;
 use syntax::source_map::FilePathMapping;
-use syntax::parse;
+use syntax_expand::config::process_configure_mod;
 
 #[path = "mod_dir_simple/test.rs"]
 mod gravy;
@@ -21,10 +24,10 @@ pub fn main() {
 }
 
 fn parse() {
-    let parse_session = ParseSess::new(FilePathMapping::empty());
+    let parse_session = ParseSess::new(FilePathMapping::empty(), process_configure_mod);
 
     let path = Path::new(file!());
     let path = path.canonicalize().unwrap();
-    let mut parser = parse::new_parser_from_file(&parse_session, &path);
+    let mut parser = new_parser_from_file(&parse_session, &path);
     let _ = parser.parse_crate_mod();
 }
diff --git a/src/test/ui-fulldeps/pprust-expr-roundtrip.rs b/src/test/ui-fulldeps/pprust-expr-roundtrip.rs
index 932a173bc67..5da0297f645 100644
--- a/src/test/ui-fulldeps/pprust-expr-roundtrip.rs
+++ b/src/test/ui-fulldeps/pprust-expr-roundtrip.rs
@@ -21,21 +21,24 @@
 
 extern crate rustc_data_structures;
 extern crate syntax;
+extern crate syntax_expand;
+extern crate rustc_parse;
 
 use rustc_data_structures::thin_vec::ThinVec;
+use rustc_parse::new_parser_from_source_str;
 use syntax::ast::*;
 use syntax::sess::ParseSess;
 use syntax::source_map::{Spanned, DUMMY_SP, FileName};
 use syntax::source_map::FilePathMapping;
 use syntax::mut_visit::{self, MutVisitor, visit_clobber};
-use syntax::parse;
 use syntax::print::pprust;
 use syntax::ptr::P;
+use syntax_expand::config::process_configure_mod;
 
 fn parse_expr(ps: &ParseSess, src: &str) -> Option<P<Expr>> {
     let src_as_string = src.to_string();
 
-    let mut p = parse::new_parser_from_source_str(
+    let mut p = new_parser_from_source_str(
         ps,
         FileName::Custom(src_as_string.clone()),
         src_as_string,
@@ -202,7 +205,7 @@ fn main() {
 }
 
 fn run() {
-    let ps = ParseSess::new(FilePathMapping::empty());
+    let ps = ParseSess::new(FilePathMapping::empty(), process_configure_mod);
 
     iter_exprs(2, &mut |mut e| {
         // If the pretty printer is correct, then `parse(print(e))` should be identical to `e`,
diff --git a/src/test/ui/proc-macro/attribute.stderr b/src/test/ui/proc-macro/attribute.stderr
index 1503f62cb6c..021e7cad09b 100644
--- a/src/test/ui/proc-macro/attribute.stderr
+++ b/src/test/ui/proc-macro/attribute.stderr
@@ -1,3 +1,15 @@
+error: malformed `proc_macro_derive` attribute input
+  --> $DIR/attribute.rs:9:1
+   |
+LL | #[proc_macro_derive]
+   | ^^^^^^^^^^^^^^^^^^^^ help: must be of the form: `#[proc_macro_derive(TraitName, /*opt*/ attributes(name1, name2, ...))]`
+
+error: malformed `proc_macro_derive` attribute input
+  --> $DIR/attribute.rs:12:1
+   |
+LL | #[proc_macro_derive = ""]
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^ help: must be of the form: `#[proc_macro_derive(TraitName, /*opt*/ attributes(name1, name2, ...))]`
+
 error: attribute must have either one or two arguments
   --> $DIR/attribute.rs:15:1
    |
@@ -88,17 +100,5 @@ error: `self` cannot be a name of derive helper attribute
 LL | #[proc_macro_derive(d17, attributes(self))]
    |                                     ^^^^
 
-error: malformed `proc_macro_derive` attribute input
-  --> $DIR/attribute.rs:9:1
-   |
-LL | #[proc_macro_derive]
-   | ^^^^^^^^^^^^^^^^^^^^ help: must be of the form: `#[proc_macro_derive(TraitName, /*opt*/ attributes(name1, name2, ...))]`
-
-error: malformed `proc_macro_derive` attribute input
-  --> $DIR/attribute.rs:12:1
-   |
-LL | #[proc_macro_derive = ""]
-   | ^^^^^^^^^^^^^^^^^^^^^^^^^ help: must be of the form: `#[proc_macro_derive(TraitName, /*opt*/ attributes(name1, name2, ...))]`
-
 error: aborting due to 17 previous errors