Auto merge of #65324 - Centril:organize-syntax, r=petrochenkov

Split libsyntax apart

In this PR the general idea is to separate the AST, parser, and friends by a more data / logic structure (tho not fully realized!) by separating out the parser and macro expansion code from libsyntax. Specifically have now three crates instead of one (libsyntax):

- libsyntax:

   - concrete syntax tree (`syntax::ast`)

   - definition of tokens and token-streams (`syntax::{token, tokenstream}`) -- used by `syntax::ast`

   - visitors (`syntax::visit`, `syntax::mut_visit`)

   - shared definitions between `libsyntax_expand`

   - feature gating (`syntax::feature_gate`) -- we could possibly move this out to its own crater later.

   - attribute and meta item utilities, including used-marking (`syntax::attr`)

   - pretty printer (`syntax::print`) -- this should possibly be moved out later. For now I've reduced down the dependencies to a single essential one which could be broken via `ParseSess`. This entails that e.g. `Debug` impls for `Path` cannot reference the pretty printer.

   - definition of `ParseSess` (`syntax::sess`) -- this is used by `syntax::{attr, print, feature_gate}` and is a common definition used by the parser and other things like librustc.

   - the `syntax::source_map` -- this includes definitions used by `syntax::ast` and other things but could ostensibly be moved `syntax_pos` since that is more related to this module.

   - a smattering of misc utilities not sufficiently important to itemize -- some of these could be moved to where they are used (often a single place) but I wanted to limit the scope of this PR.

- librustc_parse:

   - parser (`rustc_parse::parser`) -- reading a file and such are defined in the crate root tho.

   - lexer (`rustc_parse::lexer`)

   - validation of meta grammar (post-expansion) in (`rustc_parse::validate_attr`)

- libsyntax_expand -- this defines the infra for macro expansion and conditional compilation but this is not libsyntax_ext; we might want to merge them later but currently libsyntax_expand is depended on by librustc_metadata which libsyntax_ext is not.

   - conditional compilation (`syntax_expand::config`) -- moved from `syntax::config` to here

   - the bulk of this crate is made up of the old `syntax::ext`

r? @estebank

5 files changed, 40 insertions, 47 deletions

diff --git a/src/librustc_interface/Cargo.toml b/src/librustc_interface/Cargo.toml
index 35b93db1d65..de59882bbdf 100644
--- a/src/librustc_interface/Cargo.toml
+++ b/src/librustc_interface/Cargo.toml
@@ -16,6 +16,7 @@ smallvec = { version = "1.0", features = ["union", "may_dangle"] }
 syntax = { path = "../libsyntax" }
 syntax_ext = { path = "../libsyntax_ext" }
 syntax_expand = { path = "../libsyntax_expand" }
+rustc_parse = { path = "../librustc_parse" }
 syntax_pos = { path = "../libsyntax_pos" }
 rustc_serialize = { path = "../libserialize", package = "serialize" }
 rustc = { path = "../librustc" }
diff --git a/src/librustc_interface/interface.rs b/src/librustc_interface/interface.rs
index 034e861b212..02068b2ce38 100644
--- a/src/librustc_interface/interface.rs
+++ b/src/librustc_interface/interface.rs
@@ -11,14 +11,15 @@ use rustc_codegen_utils::codegen_backend::CodegenBackend;
 use rustc_data_structures::OnDrop;
 use rustc_data_structures::sync::Lrc;
 use rustc_data_structures::fx::{FxHashSet, FxHashMap};
+use rustc_parse::new_parser_from_source_str;
 use std::path::PathBuf;
 use std::result;
 use std::sync::{Arc, Mutex};
-use syntax::{self, parse};
 use syntax::ast::{self, MetaItemKind};
 use syntax::token;
 use syntax::source_map::{FileName, FileLoader, SourceMap};
 use syntax::sess::ParseSess;
+use syntax_expand::config::process_configure_mod;
 use syntax_pos::edition;
 
 pub type Result<T> = result::Result<T, ErrorReported>;
@@ -64,9 +65,9 @@ impl Compiler {
 pub fn parse_cfgspecs(cfgspecs: Vec<String>) -> FxHashSet<(String, Option<String>)> {
     syntax::with_default_globals(move || {
         let cfg = cfgspecs.into_iter().map(|s| {
-            let sess = ParseSess::with_silent_emitter();
+            let sess = ParseSess::with_silent_emitter(process_configure_mod);
             let filename = FileName::cfg_spec_source_code(&s);
-            let mut parser = parse::new_parser_from_source_str(&sess, filename, s.to_string());
+            let mut parser = new_parser_from_source_str(&sess, filename, s.to_string());
 
             macro_rules! error {($reason: expr) => {
                 early_error(ErrorOutputType::default(),
diff --git a/src/librustc_interface/passes.rs b/src/librustc_interface/passes.rs
index 04a0b0e7619..453007c5642 100644
--- a/src/librustc_interface/passes.rs
+++ b/src/librustc_interface/passes.rs
@@ -27,6 +27,7 @@ use rustc_errors::PResult;
 use rustc_incremental;
 use rustc_metadata::cstore;
 use rustc_mir as mir;
+use rustc_parse::{parse_crate_from_file, parse_crate_from_source_str};
 use rustc_passes::{self, ast_validation, hir_stats, layout_test};
 use rustc_plugin as plugin;
 use rustc_plugin::registry::Registry;
@@ -38,7 +39,6 @@ use syntax::{self, ast, visit};
 use syntax::early_buffered_lints::BufferedEarlyLint;
 use syntax_expand::base::{NamedSyntaxExtension, ExtCtxt};
 use syntax::mut_visit::MutVisitor;
-use syntax::parse;
 use syntax::util::node_count::NodeCounter;
 use syntax::symbol::Symbol;
 use syntax_pos::FileName;
@@ -61,12 +61,11 @@ pub fn parse<'a>(sess: &'a Session, input: &Input) -> PResult<'a, ast::Crate> {
     let krate = time(sess, "parsing", || {
         let _prof_timer = sess.prof.generic_activity("parse_crate");
 
-        match *input {
-            Input::File(ref file) => parse::parse_crate_from_file(file, &sess.parse_sess),
-            Input::Str {
-                ref input,
-                ref name,
-            } => parse::parse_crate_from_source_str(name.clone(), input.clone(), &sess.parse_sess),
+        match input {
+            Input::File(file) => parse_crate_from_file(file, &sess.parse_sess),
+            Input::Str { input, name } => {
+                parse_crate_from_source_str(name.clone(), input.clone(), &sess.parse_sess)
+            }
         }
     })?;
 
@@ -182,7 +181,7 @@ pub fn register_plugins<'a>(
         )
     });
 
-    let (krate, features) = syntax::config::features(
+    let (krate, features) = syntax_expand::config::features(
         krate,
         &sess.parse_sess,
         sess.edition(),
@@ -489,7 +488,7 @@ pub fn lower_to_hir(
 ) -> Result<hir::map::Forest> {
     // Lower AST to HIR.
     let hir_forest = time(sess, "lowering AST -> HIR", || {
-        let nt_to_tokenstream = syntax::parse::nt_to_tokenstream;
+        let nt_to_tokenstream = rustc_parse::nt_to_tokenstream;
         let hir_crate = lower_crate(sess, &dep_graph, &krate, resolver, nt_to_tokenstream);
 
         if sess.opts.debugging_opts.hir_stats {
diff --git a/src/librustc_interface/tests.rs b/src/librustc_interface/tests.rs
index 7a57605da58..8c1dac21576 100644
--- a/src/librustc_interface/tests.rs
+++ b/src/librustc_interface/tests.rs
@@ -8,7 +8,7 @@ use rustc::session::config::{build_configuration, build_session_options, to_crat
 use rustc::session::config::{LtoCli, LinkerPluginLto, SwitchWithOptPath, ExternEntry};
 use rustc::session::config::{Externs, OutputType, OutputTypes, SymbolManglingVersion};
 use rustc::session::config::{rustc_optgroups, Options, ErrorOutputType, Passes};
-use rustc::session::build_session;
+use rustc::session::{build_session, Session};
 use rustc::session::search_paths::SearchPath;
 use std::collections::{BTreeMap, BTreeSet};
 use std::iter::FromIterator;
@@ -17,16 +17,23 @@ use rustc_target::spec::{MergeFunctions, PanicStrategy, RelroLevel};
 use syntax::symbol::sym;
 use syntax::edition::{Edition, DEFAULT_EDITION};
 use syntax;
+use syntax_expand::config::process_configure_mod;
 use rustc_data_structures::fx::FxHashSet;
 use rustc_errors::{ColorConfig, emitter::HumanReadableErrorType, registry};
 
-pub fn build_session_options_and_crate_config(
-    matches: &getopts::Matches,
-) -> (Options, FxHashSet<(String, Option<String>)>) {
-    (
-        build_session_options(matches),
-        parse_cfgspecs(matches.opt_strs("cfg")),
-    )
+type CfgSpecs = FxHashSet<(String, Option<String>)>;
+
+fn build_session_options_and_crate_config(matches: getopts::Matches) -> (Options, CfgSpecs) {
+    let sessopts = build_session_options(&matches);
+    let cfg = parse_cfgspecs(matches.opt_strs("cfg"));
+    (sessopts, cfg)
+}
+
+fn mk_session(matches: getopts::Matches) -> (Session, CfgSpecs) {
+    let registry = registry::Registry::new(&[]);
+    let (sessopts, cfg) = build_session_options_and_crate_config(matches);
+    let sess = build_session(sessopts, None, registry, process_configure_mod);
+    (sess, cfg)
 }
 
 fn new_public_extern_entry<S, I>(locations: I) -> ExternEntry
@@ -59,31 +66,19 @@ fn mk_map<K: Ord, V>(entries: Vec<(K, V)>) -> BTreeMap<K, V> {
 #[test]
 fn test_switch_implies_cfg_test() {
     syntax::with_default_globals(|| {
-        let matches = &match optgroups().parse(&["--test".to_string()]) {
-            Ok(m) => m,
-            Err(f) => panic!("test_switch_implies_cfg_test: {}", f),
-        };
-        let registry = registry::Registry::new(&[]);
-        let (sessopts, cfg) = build_session_options_and_crate_config(matches);
-        let sess = build_session(sessopts, None, registry);
+        let matches = optgroups().parse(&["--test".to_string()]).unwrap();
+        let (sess, cfg) = mk_session(matches);
         let cfg = build_configuration(&sess, to_crate_config(cfg));
         assert!(cfg.contains(&(sym::test, None)));
     });
 }
 
-// When the user supplies --test and --cfg test, don't implicitly add
-// another --cfg test
+// When the user supplies --test and --cfg test, don't implicitly add another --cfg test
 #[test]
 fn test_switch_implies_cfg_test_unless_cfg_test() {
     syntax::with_default_globals(|| {
-        let matches = &match optgroups().parse(&["--test".to_string(),
-                                                 "--cfg=test".to_string()]) {
-            Ok(m) => m,
-            Err(f) => panic!("test_switch_implies_cfg_test_unless_cfg_test: {}", f),
-        };
-        let registry = registry::Registry::new(&[]);
-        let (sessopts, cfg) = build_session_options_and_crate_config(matches);
-        let sess = build_session(sessopts, None, registry);
+        let matches = optgroups().parse(&["--test".to_string(), "--cfg=test".to_string()]).unwrap();
+        let (sess, cfg) = mk_session(matches);
         let cfg = build_configuration(&sess, to_crate_config(cfg));
         let mut test_items = cfg.iter().filter(|&&(name, _)| name == sym::test);
         assert!(test_items.next().is_some());
@@ -95,9 +90,7 @@ fn test_switch_implies_cfg_test_unless_cfg_test() {
 fn test_can_print_warnings() {
     syntax::with_default_globals(|| {
         let matches = optgroups().parse(&["-Awarnings".to_string()]).unwrap();
-        let registry = registry::Registry::new(&[]);
-        let (sessopts, _) = build_session_options_and_crate_config(&matches);
-        let sess = build_session(sessopts, None, registry);
+        let (sess, _) = mk_session(matches);
         assert!(!sess.diagnostic().can_emit_warnings());
     });
 
@@ -105,17 +98,13 @@ fn test_can_print_warnings() {
         let matches = optgroups()
             .parse(&["-Awarnings".to_string(), "-Dwarnings".to_string()])
             .unwrap();
-        let registry = registry::Registry::new(&[]);
-        let (sessopts, _) = build_session_options_and_crate_config(&matches);
-        let sess = build_session(sessopts, None, registry);
+        let (sess, _) = mk_session(matches);
         assert!(sess.diagnostic().can_emit_warnings());
     });
 
     syntax::with_default_globals(|| {
         let matches = optgroups().parse(&["-Adead_code".to_string()]).unwrap();
-        let registry = registry::Registry::new(&[]);
-        let (sessopts, _) = build_session_options_and_crate_config(&matches);
-        let sess = build_session(sessopts, None, registry);
+        let (sess, _) = mk_session(matches);
         assert!(sess.diagnostic().can_emit_warnings());
     });
 }
@@ -704,6 +693,6 @@ fn test_edition_parsing() {
     let matches = optgroups()
         .parse(&["--edition=2018".to_string()])
         .unwrap();
-    let (sessopts, _) = build_session_options_and_crate_config(&matches);
+    let (sessopts, _) = build_session_options_and_crate_config(matches);
     assert!(sessopts.edition == Edition::Edition2018)
 }
diff --git a/src/librustc_interface/util.rs b/src/librustc_interface/util.rs
index 0c08f65d11d..115345955ae 100644
--- a/src/librustc_interface/util.rs
+++ b/src/librustc_interface/util.rs
@@ -36,6 +36,7 @@ use syntax::util::lev_distance::find_best_match_for_name;
 use syntax::source_map::{FileLoader, RealFileLoader, SourceMap};
 use syntax::symbol::{Symbol, sym};
 use syntax::{self, ast, attr};
+use syntax_expand::config::process_configure_mod;
 use syntax_pos::edition::Edition;
 #[cfg(not(parallel_compiler))]
 use std::{thread, panic};
@@ -49,6 +50,7 @@ pub fn diagnostics_registry() -> Registry {
     // FIXME: need to figure out a way to get these back in here
     // all_errors.extend_from_slice(get_codegen_backend(sess).diagnostics());
     all_errors.extend_from_slice(&rustc_metadata::error_codes::DIAGNOSTICS);
+    all_errors.extend_from_slice(&rustc_parse::error_codes::DIAGNOSTICS);
     all_errors.extend_from_slice(&rustc_passes::error_codes::DIAGNOSTICS);
     all_errors.extend_from_slice(&rustc_plugin::error_codes::DIAGNOSTICS);
     all_errors.extend_from_slice(&rustc_mir::error_codes::DIAGNOSTICS);
@@ -103,6 +105,7 @@ pub fn create_session(
         source_map.clone(),
         diagnostic_output,
         lint_caps,
+        process_configure_mod,
     );
 
     let codegen_backend = get_codegen_backend(&sess);