Auto merge of #65324 - Centril:organize-syntax, r=petrochenkov

Split libsyntax apart

In this PR the general idea is to separate the AST, parser, and friends by a more data / logic structure (tho not fully realized!) by separating out the parser and macro expansion code from libsyntax. Specifically have now three crates instead of one (libsyntax):

- libsyntax:

   - concrete syntax tree (`syntax::ast`)

   - definition of tokens and token-streams (`syntax::{token, tokenstream}`) -- used by `syntax::ast`

   - visitors (`syntax::visit`, `syntax::mut_visit`)

   - shared definitions between `libsyntax_expand`

   - feature gating (`syntax::feature_gate`) -- we could possibly move this out to its own crater later.

   - attribute and meta item utilities, including used-marking (`syntax::attr`)

   - pretty printer (`syntax::print`) -- this should possibly be moved out later. For now I've reduced down the dependencies to a single essential one which could be broken via `ParseSess`. This entails that e.g. `Debug` impls for `Path` cannot reference the pretty printer.

   - definition of `ParseSess` (`syntax::sess`) -- this is used by `syntax::{attr, print, feature_gate}` and is a common definition used by the parser and other things like librustc.

   - the `syntax::source_map` -- this includes definitions used by `syntax::ast` and other things but could ostensibly be moved `syntax_pos` since that is more related to this module.

   - a smattering of misc utilities not sufficiently important to itemize -- some of these could be moved to where they are used (often a single place) but I wanted to limit the scope of this PR.

- librustc_parse:

   - parser (`rustc_parse::parser`) -- reading a file and such are defined in the crate root tho.

   - lexer (`rustc_parse::lexer`)

   - validation of meta grammar (post-expansion) in (`rustc_parse::validate_attr`)

- libsyntax_expand -- this defines the infra for macro expansion and conditional compilation but this is not libsyntax_ext; we might want to merge them later but currently libsyntax_expand is depended on by librustc_metadata which libsyntax_ext is not.

   - conditional compilation (`syntax_expand::config`) -- moved from `syntax::config` to here

   - the bulk of this crate is made up of the old `syntax::ext`

r? @estebank

1 files changed, 0 insertions, 309 deletions

diff --git a/src/libsyntax/parse/parser/generics.rs b/src/libsyntax/parse/parser/generics.rs
deleted file mode 100644
index ae9ecd8fe39..00000000000
--- a/src/libsyntax/parse/parser/generics.rs
+++ /dev/null
@@ -1,309 +0,0 @@
-use super::Parser;
-
-use crate::ast::{self, WhereClause, GenericParam, GenericParamKind, GenericBounds, Attribute};
-use crate::token;
-use crate::source_map::DUMMY_SP;
-
-use syntax_pos::symbol::{kw, sym};
-
-use errors::PResult;
-
-impl<'a> Parser<'a> {
-    /// Parses bounds of a lifetime parameter `BOUND + BOUND + BOUND`, possibly with trailing `+`.
-    ///
-    /// ```
-    /// BOUND = LT_BOUND (e.g., `'a`)
-    /// ```
-    fn parse_lt_param_bounds(&mut self) -> GenericBounds {
-        let mut lifetimes = Vec::new();
-        while self.check_lifetime() {
-            lifetimes.push(ast::GenericBound::Outlives(self.expect_lifetime()));
-
-            if !self.eat_plus() {
-                break
-            }
-        }
-        lifetimes
-    }
-
-    /// Matches `typaram = IDENT (`?` unbound)? optbounds ( EQ ty )?`.
-    fn parse_ty_param(&mut self,
-                      preceding_attrs: Vec<Attribute>)
-                      -> PResult<'a, GenericParam> {
-        let ident = self.parse_ident()?;
-
-        // Parse optional colon and param bounds.
-        let bounds = if self.eat(&token::Colon) {
-            self.parse_generic_bounds(Some(self.prev_span))?
-        } else {
-            Vec::new()
-        };
-
-        let default = if self.eat(&token::Eq) {
-            Some(self.parse_ty()?)
-        } else {
-            None
-        };
-
-        Ok(GenericParam {
-            ident,
-            id: ast::DUMMY_NODE_ID,
-            attrs: preceding_attrs.into(),
-            bounds,
-            kind: GenericParamKind::Type {
-                default,
-            },
-            is_placeholder: false
-        })
-    }
-
-    fn parse_const_param(&mut self, preceding_attrs: Vec<Attribute>) -> PResult<'a, GenericParam> {
-        let lo = self.token.span;
-
-        self.expect_keyword(kw::Const)?;
-        let ident = self.parse_ident()?;
-        self.expect(&token::Colon)?;
-        let ty = self.parse_ty()?;
-
-        self.sess.gated_spans.gate(sym::const_generics, lo.to(self.prev_span));
-
-        Ok(GenericParam {
-            ident,
-            id: ast::DUMMY_NODE_ID,
-            attrs: preceding_attrs.into(),
-            bounds: Vec::new(),
-            kind: GenericParamKind::Const {
-                ty,
-            },
-            is_placeholder: false
-        })
-    }
-
-    /// Parses a (possibly empty) list of lifetime and type parameters, possibly including
-    /// a trailing comma and erroneous trailing attributes.
-    pub(super) fn parse_generic_params(&mut self) -> PResult<'a, Vec<ast::GenericParam>> {
-        let mut params = Vec::new();
-        loop {
-            let attrs = self.parse_outer_attributes()?;
-            if self.check_lifetime() {
-                let lifetime = self.expect_lifetime();
-                // Parse lifetime parameter.
-                let bounds = if self.eat(&token::Colon) {
-                    self.parse_lt_param_bounds()
-                } else {
-                    Vec::new()
-                };
-                params.push(ast::GenericParam {
-                    ident: lifetime.ident,
-                    id: lifetime.id,
-                    attrs: attrs.into(),
-                    bounds,
-                    kind: ast::GenericParamKind::Lifetime,
-                    is_placeholder: false
-                });
-            } else if self.check_keyword(kw::Const) {
-                // Parse const parameter.
-                params.push(self.parse_const_param(attrs)?);
-            } else if self.check_ident() {
-                // Parse type parameter.
-                params.push(self.parse_ty_param(attrs)?);
-            } else if self.token.can_begin_type() {
-                // Trying to write an associated type bound? (#26271)
-                let snapshot = self.clone();
-                match self.parse_ty_where_predicate() {
-                    Ok(where_predicate) => {
-                        self.struct_span_err(
-                            where_predicate.span(),
-                            "bounds on associated types do not belong here",
-                        )
-                        .span_label(where_predicate.span(), "belongs in `where` clause")
-                        .emit();
-                    }
-                    Err(mut err) => {
-                        err.cancel();
-                        std::mem::replace(self, snapshot);
-                        break
-                    }
-                }
-            } else {
-                // Check for trailing attributes and stop parsing.
-                if !attrs.is_empty() {
-                    if !params.is_empty() {
-                        self.struct_span_err(
-                            attrs[0].span,
-                            "trailing attribute after generic parameter",
-                        )
-                        .span_label(attrs[0].span, "attributes must go before parameters")
-                        .emit();
-                    } else {
-                        self.struct_span_err(
-                            attrs[0].span,
-                            &format!("attribute without generic parameters"),
-                        )
-                        .span_label(
-                            attrs[0].span,
-                            "attributes are only permitted when preceding parameters",
-                        )
-                        .emit();
-                    }
-                }
-                break
-            }
-
-            if !self.eat(&token::Comma) {
-                break
-            }
-        }
-        Ok(params)
-    }
-
-    /// Parses a set of optional generic type parameter declarations. Where
-    /// clauses are not parsed here, and must be added later via
-    /// `parse_where_clause()`.
-    ///
-    /// matches generics = ( ) | ( < > ) | ( < typaramseq ( , )? > ) | ( < lifetimes ( , )? > )
-    ///                  | ( < lifetimes , typaramseq ( , )? > )
-    /// where   typaramseq = ( typaram ) | ( typaram , typaramseq )
-    pub(super) fn parse_generics(&mut self) -> PResult<'a, ast::Generics> {
-        let span_lo = self.token.span;
-        let (params, span) = if self.eat_lt() {
-            let params = self.parse_generic_params()?;
-            self.expect_gt()?;
-            (params, span_lo.to(self.prev_span))
-        } else {
-            (vec![], self.prev_span.between(self.token.span))
-        };
-        Ok(ast::Generics {
-            params,
-            where_clause: WhereClause {
-                predicates: Vec::new(),
-                span: DUMMY_SP,
-            },
-            span,
-        })
-    }
-
-    /// Parses an optional where-clause and places it in `generics`.
-    ///
-    /// ```ignore (only-for-syntax-highlight)
-    /// where T : Trait<U, V> + 'b, 'a : 'b
-    /// ```
-    pub(super) fn parse_where_clause(&mut self) -> PResult<'a, WhereClause> {
-        let mut where_clause = WhereClause {
-            predicates: Vec::new(),
-            span: self.prev_span.to(self.prev_span),
-        };
-
-        if !self.eat_keyword(kw::Where) {
-            return Ok(where_clause);
-        }
-        let lo = self.prev_span;
-
-        // We are considering adding generics to the `where` keyword as an alternative higher-rank
-        // parameter syntax (as in `where<'a>` or `where<T>`. To avoid that being a breaking
-        // change we parse those generics now, but report an error.
-        if self.choose_generics_over_qpath() {
-            let generics = self.parse_generics()?;
-            self.struct_span_err(
-                generics.span,
-                "generic parameters on `where` clauses are reserved for future use",
-            )
-                .span_label(generics.span, "currently unsupported")
-                .emit();
-        }
-
-        loop {
-            let lo = self.token.span;
-            if self.check_lifetime() && self.look_ahead(1, |t| !t.is_like_plus()) {
-                let lifetime = self.expect_lifetime();
-                // Bounds starting with a colon are mandatory, but possibly empty.
-                self.expect(&token::Colon)?;
-                let bounds = self.parse_lt_param_bounds();
-                where_clause.predicates.push(ast::WherePredicate::RegionPredicate(
-                    ast::WhereRegionPredicate {
-                        span: lo.to(self.prev_span),
-                        lifetime,
-                        bounds,
-                    }
-                ));
-            } else if self.check_type() {
-                where_clause.predicates.push(self.parse_ty_where_predicate()?);
-            } else {
-                break
-            }
-
-            if !self.eat(&token::Comma) {
-                break
-            }
-        }
-
-        where_clause.span = lo.to(self.prev_span);
-        Ok(where_clause)
-    }
-
-    fn parse_ty_where_predicate(&mut self) -> PResult<'a, ast::WherePredicate> {
-        let lo = self.token.span;
-        // Parse optional `for<'a, 'b>`.
-        // This `for` is parsed greedily and applies to the whole predicate,
-        // the bounded type can have its own `for` applying only to it.
-        // Examples:
-        // * `for<'a> Trait1<'a>: Trait2<'a /* ok */>`
-        // * `(for<'a> Trait1<'a>): Trait2<'a /* not ok */>`
-        // * `for<'a> for<'b> Trait1<'a, 'b>: Trait2<'a /* ok */, 'b /* not ok */>`
-        let lifetime_defs = self.parse_late_bound_lifetime_defs()?;
-
-        // Parse type with mandatory colon and (possibly empty) bounds,
-        // or with mandatory equality sign and the second type.
-        let ty = self.parse_ty()?;
-        if self.eat(&token::Colon) {
-            let bounds = self.parse_generic_bounds(Some(self.prev_span))?;
-            Ok(ast::WherePredicate::BoundPredicate(
-                ast::WhereBoundPredicate {
-                    span: lo.to(self.prev_span),
-                    bound_generic_params: lifetime_defs,
-                    bounded_ty: ty,
-                    bounds,
-                }
-            ))
-        // FIXME: Decide what should be used here, `=` or `==`.
-        // FIXME: We are just dropping the binders in lifetime_defs on the floor here.
-        } else if self.eat(&token::Eq) || self.eat(&token::EqEq) {
-            let rhs_ty = self.parse_ty()?;
-            Ok(ast::WherePredicate::EqPredicate(
-                ast::WhereEqPredicate {
-                    span: lo.to(self.prev_span),
-                    lhs_ty: ty,
-                    rhs_ty,
-                    id: ast::DUMMY_NODE_ID,
-                }
-            ))
-        } else {
-            self.unexpected()
-        }
-    }
-
-    pub(super) fn choose_generics_over_qpath(&self) -> bool {
-        // There's an ambiguity between generic parameters and qualified paths in impls.
-        // If we see `<` it may start both, so we have to inspect some following tokens.
-        // The following combinations can only start generics,
-        // but not qualified paths (with one exception):
-        //     `<` `>` - empty generic parameters
-        //     `<` `#` - generic parameters with attributes
-        //     `<` (LIFETIME|IDENT) `>` - single generic parameter
-        //     `<` (LIFETIME|IDENT) `,` - first generic parameter in a list
-        //     `<` (LIFETIME|IDENT) `:` - generic parameter with bounds
-        //     `<` (LIFETIME|IDENT) `=` - generic parameter with a default
-        //     `<` const                - generic const parameter
-        // The only truly ambiguous case is
-        //     `<` IDENT `>` `::` IDENT ...
-        // we disambiguate it in favor of generics (`impl<T> ::absolute::Path<T> { ... }`)
-        // because this is what almost always expected in practice, qualified paths in impls
-        // (`impl <Type>::AssocTy { ... }`) aren't even allowed by type checker at the moment.
-        self.token == token::Lt &&
-            (self.look_ahead(1, |t| t == &token::Pound || t == &token::Gt) ||
-             self.look_ahead(1, |t| t.is_lifetime() || t.is_ident()) &&
-                self.look_ahead(2, |t| t == &token::Gt || t == &token::Comma ||
-                                       t == &token::Colon || t == &token::Eq) ||
-            self.is_keyword_ahead(1, &[kw::Const]))
-    }
-}