Auto merge of #65324 - Centril:organize-syntax, r=petrochenkov

Split libsyntax apart

In this PR the general idea is to separate the AST, parser, and friends by a more data / logic structure (tho not fully realized!) by separating out the parser and macro expansion code from libsyntax. Specifically have now three crates instead of one (libsyntax):

- libsyntax:

   - concrete syntax tree (`syntax::ast`)

   - definition of tokens and token-streams (`syntax::{token, tokenstream}`) -- used by `syntax::ast`

   - visitors (`syntax::visit`, `syntax::mut_visit`)

   - shared definitions between `libsyntax_expand`

   - feature gating (`syntax::feature_gate`) -- we could possibly move this out to its own crater later.

   - attribute and meta item utilities, including used-marking (`syntax::attr`)

   - pretty printer (`syntax::print`) -- this should possibly be moved out later. For now I've reduced down the dependencies to a single essential one which could be broken via `ParseSess`. This entails that e.g. `Debug` impls for `Path` cannot reference the pretty printer.

   - definition of `ParseSess` (`syntax::sess`) -- this is used by `syntax::{attr, print, feature_gate}` and is a common definition used by the parser and other things like librustc.

   - the `syntax::source_map` -- this includes definitions used by `syntax::ast` and other things but could ostensibly be moved `syntax_pos` since that is more related to this module.

   - a smattering of misc utilities not sufficiently important to itemize -- some of these could be moved to where they are used (often a single place) but I wanted to limit the scope of this PR.

- librustc_parse:

   - parser (`rustc_parse::parser`) -- reading a file and such are defined in the crate root tho.

   - lexer (`rustc_parse::lexer`)

   - validation of meta grammar (post-expansion) in (`rustc_parse::validate_attr`)

- libsyntax_expand -- this defines the infra for macro expansion and conditional compilation but this is not libsyntax_ext; we might want to merge them later but currently libsyntax_expand is depended on by librustc_metadata which libsyntax_ext is not.

   - conditional compilation (`syntax_expand::config`) -- moved from `syntax::config` to here

   - the bulk of this crate is made up of the old `syntax::ext`

r? @estebank

5 files changed, 12 insertions, 11 deletions

diff --git a/src/libsyntax_ext/Cargo.toml b/src/libsyntax_ext/Cargo.toml
index 703d51e1c4c..5ae8b3a3e0c 100644
--- a/src/libsyntax_ext/Cargo.toml
+++ b/src/libsyntax_ext/Cargo.toml
@@ -14,6 +14,7 @@ errors = { path = "../librustc_errors", package = "rustc_errors" }
 fmt_macros = { path = "../libfmt_macros" }
 log = "0.4"
 rustc_data_structures = { path = "../librustc_data_structures" }
+rustc_parse = { path = "../librustc_parse" }
 rustc_target = { path = "../librustc_target" }
 smallvec = { version = "1.0", features = ["union", "may_dangle"] }
 syntax = { path = "../libsyntax" }
diff --git a/src/libsyntax_ext/assert.rs b/src/libsyntax_ext/assert.rs
index a15423b7ad8..c4f3c03813f 100644
--- a/src/libsyntax_ext/assert.rs
+++ b/src/libsyntax_ext/assert.rs
@@ -1,13 +1,13 @@
 use errors::{Applicability, DiagnosticBuilder};
 
+use rustc_parse::parser::Parser;
 use syntax::ast::{self, *};
-use syntax_expand::base::*;
 use syntax::token::{self, TokenKind};
-use syntax::parse::parser::Parser;
 use syntax::print::pprust;
 use syntax::ptr::P;
 use syntax::symbol::{sym, Symbol};
 use syntax::tokenstream::{TokenStream, TokenTree};
+use syntax_expand::base::*;
 use syntax_pos::{Span, DUMMY_SP};
 
 pub fn expand_assert<'cx>(
diff --git a/src/libsyntax_ext/cmdline_attrs.rs b/src/libsyntax_ext/cmdline_attrs.rs
index 171f2405573..5c3009c432c 100644
--- a/src/libsyntax_ext/cmdline_attrs.rs
+++ b/src/libsyntax_ext/cmdline_attrs.rs
@@ -2,7 +2,6 @@
 
 use syntax::ast::{self, AttrItem, AttrStyle};
 use syntax::attr::mk_attr;
-use syntax::parse;
 use syntax::token;
 use syntax::sess::ParseSess;
 use syntax_expand::panictry;
@@ -10,7 +9,7 @@ use syntax_pos::FileName;
 
 pub fn inject(mut krate: ast::Crate, parse_sess: &ParseSess, attrs: &[String]) -> ast::Crate {
     for raw_attr in attrs {
-        let mut parser = parse::new_parser_from_source_str(
+        let mut parser = rustc_parse::new_parser_from_source_str(
             parse_sess,
             FileName::cli_crate_attr_source_code(&raw_attr),
             raw_attr.clone(),
diff --git a/src/libsyntax_ext/source_util.rs b/src/libsyntax_ext/source_util.rs
index 7e47b40714d..120b533a69b 100644
--- a/src/libsyntax_ext/source_util.rs
+++ b/src/libsyntax_ext/source_util.rs
@@ -1,13 +1,13 @@
-use syntax_expand::panictry;
-use syntax_expand::base::{self, *};
+use rustc_parse::{self, DirectoryOwnership, new_sub_parser_from_file, parser::Parser};
 use syntax::ast;
-use syntax::parse::{self, DirectoryOwnership};
 use syntax::print::pprust;
 use syntax::ptr::P;
 use syntax::symbol::Symbol;
 use syntax::token;
 use syntax::tokenstream::TokenStream;
 use syntax::early_buffered_lints::BufferedEarlyLintId;
+use syntax_expand::panictry;
+use syntax_expand::base::{self, *};
 
 use smallvec::SmallVec;
 use syntax_pos::{self, Pos, Span};
@@ -85,10 +85,10 @@ pub fn expand_include<'cx>(cx: &'cx mut ExtCtxt<'_>, sp: Span, tts: TokenStream)
         },
     };
     let directory_ownership = DirectoryOwnership::Owned { relative: None };
-    let p = parse::new_sub_parser_from_file(cx.parse_sess(), &file, directory_ownership, None, sp);
+    let p = new_sub_parser_from_file(cx.parse_sess(), &file, directory_ownership, None, sp);
 
     struct ExpandResult<'a> {
-        p: parse::parser::Parser<'a>,
+        p: Parser<'a>,
     }
     impl<'a> base::MacResult for ExpandResult<'a> {
         fn make_expr(mut self: Box<ExpandResult<'a>>) -> Option<P<ast::Expr>> {
diff --git a/src/libsyntax_ext/util.rs b/src/libsyntax_ext/util.rs
index d84fe19b3ea..e59daab1770 100644
--- a/src/libsyntax_ext/util.rs
+++ b/src/libsyntax_ext/util.rs
@@ -1,11 +1,12 @@
+use rustc_parse::validate_attr;
 use syntax_pos::Symbol;
 use syntax::ast::MetaItem;
-use syntax::attr::{check_builtin_attribute, AttributeTemplate};
+use syntax::attr::AttributeTemplate;
 use syntax_expand::base::ExtCtxt;
 
 pub fn check_builtin_macro_attribute(ecx: &ExtCtxt<'_>, meta_item: &MetaItem, name: Symbol) {
     // All the built-in macro attributes are "words" at the moment.
     let template = AttributeTemplate::only_word();
     let attr = ecx.attribute(meta_item.clone());
-    check_builtin_attribute(ecx.parse_sess, &attr, name, template);
+    validate_attr::check_builtin_attribute(ecx.parse_sess, &attr, name, template);
 }