Auto merge of #65324 - Centril:organize-syntax, r=petrochenkov

Split libsyntax apart

In this PR the general idea is to separate the AST, parser, and friends by a more data / logic structure (tho not fully realized!) by separating out the parser and macro expansion code from libsyntax. Specifically have now three crates instead of one (libsyntax):

- libsyntax:

   - concrete syntax tree (`syntax::ast`)

   - definition of tokens and token-streams (`syntax::{token, tokenstream}`) -- used by `syntax::ast`

   - visitors (`syntax::visit`, `syntax::mut_visit`)

   - shared definitions between `libsyntax_expand`

   - feature gating (`syntax::feature_gate`) -- we could possibly move this out to its own crater later.

   - attribute and meta item utilities, including used-marking (`syntax::attr`)

   - pretty printer (`syntax::print`) -- this should possibly be moved out later. For now I've reduced down the dependencies to a single essential one which could be broken via `ParseSess`. This entails that e.g. `Debug` impls for `Path` cannot reference the pretty printer.

   - definition of `ParseSess` (`syntax::sess`) -- this is used by `syntax::{attr, print, feature_gate}` and is a common definition used by the parser and other things like librustc.

   - the `syntax::source_map` -- this includes definitions used by `syntax::ast` and other things but could ostensibly be moved `syntax_pos` since that is more related to this module.

   - a smattering of misc utilities not sufficiently important to itemize -- some of these could be moved to where they are used (often a single place) but I wanted to limit the scope of this PR.

- librustc_parse:

   - parser (`rustc_parse::parser`) -- reading a file and such are defined in the crate root tho.

   - lexer (`rustc_parse::lexer`)

   - validation of meta grammar (post-expansion) in (`rustc_parse::validate_attr`)

- libsyntax_expand -- this defines the infra for macro expansion and conditional compilation but this is not libsyntax_ext; we might want to merge them later but currently libsyntax_expand is depended on by librustc_metadata which libsyntax_ext is not.

   - conditional compilation (`syntax_expand::config`) -- moved from `syntax::config` to here

   - the bulk of this crate is made up of the old `syntax::ext`

r? @estebank

1 files changed, 0 insertions, 108 deletions

diff --git a/src/libsyntax/tokenstream/tests.rs b/src/libsyntax/tokenstream/tests.rs
deleted file mode 100644
index 5017e5f5424..00000000000
--- a/src/libsyntax/tokenstream/tests.rs
+++ /dev/null
@@ -1,108 +0,0 @@
-use super::*;
-
-use crate::ast::Name;
-use crate::with_default_globals;
-use crate::tests::string_to_stream;
-use syntax_pos::{Span, BytePos};
-
-fn string_to_ts(string: &str) -> TokenStream {
-    string_to_stream(string.to_owned())
-}
-
-fn sp(a: u32, b: u32) -> Span {
-    Span::with_root_ctxt(BytePos(a), BytePos(b))
-}
-
-#[test]
-fn test_concat() {
-    with_default_globals(|| {
-        let test_res = string_to_ts("foo::bar::baz");
-        let test_fst = string_to_ts("foo::bar");
-        let test_snd = string_to_ts("::baz");
-        let eq_res = TokenStream::from_streams(smallvec![test_fst, test_snd]);
-        assert_eq!(test_res.trees().count(), 5);
-        assert_eq!(eq_res.trees().count(), 5);
-        assert_eq!(test_res.eq_unspanned(&eq_res), true);
-    })
-}
-
-#[test]
-fn test_to_from_bijection() {
-    with_default_globals(|| {
-        let test_start = string_to_ts("foo::bar(baz)");
-        let test_end = test_start.trees().collect();
-        assert_eq!(test_start, test_end)
-    })
-}
-
-#[test]
-fn test_eq_0() {
-    with_default_globals(|| {
-        let test_res = string_to_ts("foo");
-        let test_eqs = string_to_ts("foo");
-        assert_eq!(test_res, test_eqs)
-    })
-}
-
-#[test]
-fn test_eq_1() {
-    with_default_globals(|| {
-        let test_res = string_to_ts("::bar::baz");
-        let test_eqs = string_to_ts("::bar::baz");
-        assert_eq!(test_res, test_eqs)
-    })
-}
-
-#[test]
-fn test_eq_3() {
-    with_default_globals(|| {
-        let test_res = string_to_ts("");
-        let test_eqs = string_to_ts("");
-        assert_eq!(test_res, test_eqs)
-    })
-}
-
-#[test]
-fn test_diseq_0() {
-    with_default_globals(|| {
-        let test_res = string_to_ts("::bar::baz");
-        let test_eqs = string_to_ts("bar::baz");
-        assert_eq!(test_res == test_eqs, false)
-    })
-}
-
-#[test]
-fn test_diseq_1() {
-    with_default_globals(|| {
-        let test_res = string_to_ts("(bar,baz)");
-        let test_eqs = string_to_ts("bar,baz");
-        assert_eq!(test_res == test_eqs, false)
-    })
-}
-
-#[test]
-fn test_is_empty() {
-    with_default_globals(|| {
-        let test0: TokenStream = Vec::<TokenTree>::new().into_iter().collect();
-        let test1: TokenStream =
-            TokenTree::token(token::Ident(Name::intern("a"), false), sp(0, 1)).into();
-        let test2 = string_to_ts("foo(bar::baz)");
-
-        assert_eq!(test0.is_empty(), true);
-        assert_eq!(test1.is_empty(), false);
-        assert_eq!(test2.is_empty(), false);
-    })
-}
-
-#[test]
-fn test_dotdotdot() {
-    with_default_globals(|| {
-        let mut builder = TokenStreamBuilder::new();
-        builder.push(TokenTree::token(token::Dot, sp(0, 1)).joint());
-        builder.push(TokenTree::token(token::Dot, sp(1, 2)).joint());
-        builder.push(TokenTree::token(token::Dot, sp(2, 3)));
-        let stream = builder.build();
-        assert!(stream.eq_unspanned(&string_to_ts("...")));
-        assert_eq!(stream.trees().count(), 1);
-    })
-}