Rollup merge of #69194 - Centril:assoc-extern-fuse, r=petrochenkov

parse: fuse associated and extern items up to defaultness

Language changes:

- The grammar of extern `type` aliases is unified with associated ones, and becomes:
  ```rust
  TypeItem = "type" ident generics {":" bounds}? where_clause {"=" type}? ";" ;
  ```

  Semantic restrictions (`ast_validation`) are added to forbid any parameters in `generics`, any bounds in `bounds`, and any predicates in `where_clause`, as well as the presence of a type expression (`= u8`).

  (Work still remains to fuse this with free `type` aliases, but this can be done later.)

- The grammar of constants and static items (free, associated, and extern) now permits the absence of an expression, and becomes:

  ```rust
  GlobalItem = {"const" {ident | "_"} | "static" "mut"? ident} {"=" expr}? ";" ;
  ```

  - A semantic restriction is added to enforce the presence of the expression (the body).
  - A semantic restriction is added to reject `const _` in associated contexts.

Together, these changes allow us to fuse the grammar of associated items and extern items up to `default`ness which is the main goal of the PR.

-----------------------

We are now very close to fully fusing the entirely of item parsing and their ASTs. To progress further, we must make a decision: should we parse e.g. `default use foo::bar;` and whatnot? Accepting that is likely easiest from a parsing perspective, as it does not require using look-ahead, but it is perhaps not too onerous to only accept it for `fn`s (and all their various qualifiers), `const`s, `static`s, and `type`s.

r? @petrochenkov

3 files changed, 90 insertions, 118 deletions

diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs
index a9c6fcddc77..9ae3010a0f6 100644
--- a/src/libsyntax/ast.rs
+++ b/src/libsyntax/ast.rs
@@ -1612,46 +1612,6 @@ pub struct FnSig {
     pub decl: P<FnDecl>,
 }
 
-/// Represents associated items.
-/// These include items in `impl` and `trait` definitions.
-#[derive(Clone, RustcEncodable, RustcDecodable, Debug)]
-pub struct AssocItem {
-    pub attrs: Vec<Attribute>,
-    pub id: NodeId,
-    pub span: Span,
-    pub vis: Visibility,
-    pub ident: Ident,
-
-    pub defaultness: Defaultness,
-    pub generics: Generics,
-    pub kind: AssocItemKind,
-    /// See `Item::tokens` for what this is.
-    pub tokens: Option<TokenStream>,
-}
-
-/// Represents various kinds of content within an `impl`.
-///
-/// The term "provided" in the variants below refers to the item having a default
-/// definition / body. Meanwhile, a "required" item lacks a definition / body.
-/// In an implementation, all items must be provided.
-/// The `Option`s below denote the bodies, where `Some(_)`
-/// means "provided" and conversely `None` means "required".
-#[derive(Clone, RustcEncodable, RustcDecodable, Debug)]
-pub enum AssocItemKind {
-    /// An associated constant, `const $ident: $ty $def?;` where `def ::= "=" $expr? ;`.
-    /// If `def` is parsed, then the associated constant is provided, and otherwise required.
-    Const(P<Ty>, Option<P<Expr>>),
-
-    /// An associated function.
-    Fn(FnSig, Option<P<Block>>),
-
-    /// An associated type.
-    TyAlias(GenericBounds, Option<P<Ty>>),
-
-    /// A macro expanding to an associated item.
-    Macro(Mac),
-}
-
 #[derive(
     Clone,
     Copy,
@@ -2542,11 +2502,11 @@ pub enum ItemKind {
     /// A static item (`static`).
     ///
     /// E.g., `static FOO: i32 = 42;` or `static FOO: &'static str = "bar";`.
-    Static(P<Ty>, Mutability, P<Expr>),
+    Static(P<Ty>, Mutability, Option<P<Expr>>),
     /// A constant item (`const`).
     ///
     /// E.g., `const FOO: i32 = 42;`.
-    Const(P<Ty>, P<Expr>),
+    Const(P<Ty>, Option<P<Expr>>),
     /// A function declaration (`fn`).
     ///
     /// E.g., `fn foo(bar: usize) -> usize { .. }`.
@@ -2646,28 +2606,45 @@ impl ItemKind {
     }
 }
 
-pub type ForeignItem = Item<ForeignItemKind>;
+// FIXME(Centril): These definitions should be unmerged;
+// see https://github.com/rust-lang/rust/pull/69194#discussion_r379899975
+pub type ForeignItem = Item<AssocItemKind>;
+pub type ForeignItemKind = AssocItemKind;
 
-/// An item within an `extern` block.
+/// Represents associated items.
+/// These include items in `impl` and `trait` definitions.
 #[derive(Clone, RustcEncodable, RustcDecodable, Debug)]
-pub enum ForeignItemKind {
-    /// A foreign function.
-    Fn(FnSig, Generics, Option<P<Block>>),
-    /// A foreign static item (`static ext: u8`).
-    Static(P<Ty>, Mutability),
-    /// A foreign type.
-    Ty,
-    /// A macro invocation.
-    Macro(Mac),
+pub struct AssocItem {
+    pub attrs: Vec<Attribute>,
+    pub id: NodeId,
+    pub span: Span,
+    pub vis: Visibility,
+    pub ident: Ident,
+
+    pub defaultness: Defaultness,
+    pub kind: AssocItemKind,
+    /// See `Item::tokens` for what this is.
+    pub tokens: Option<TokenStream>,
 }
 
-impl ForeignItemKind {
-    pub fn descriptive_variant(&self) -> &str {
-        match *self {
-            ForeignItemKind::Fn(..) => "foreign function",
-            ForeignItemKind::Static(..) => "foreign static item",
-            ForeignItemKind::Ty => "foreign type",
-            ForeignItemKind::Macro(..) => "macro in foreign module",
-        }
-    }
+/// Represents non-free item kinds.
+///
+/// The term "provided" in the variants below refers to the item having a default
+/// definition / body. Meanwhile, a "required" item lacks a definition / body.
+/// In an implementation, all items must be provided.
+/// The `Option`s below denote the bodies, where `Some(_)`
+/// means "provided" and conversely `None` means "required".
+#[derive(Clone, RustcEncodable, RustcDecodable, Debug)]
+pub enum AssocItemKind {
+    /// A constant, `const $ident: $ty $def?;` where `def ::= "=" $expr? ;`.
+    /// If `def` is parsed, then the constant is provided, and otherwise required.
+    Const(P<Ty>, Option<P<Expr>>),
+    /// A static item (`static FOO: u8`).
+    Static(P<Ty>, Mutability, Option<P<Expr>>),
+    /// A function.
+    Fn(FnSig, Generics, Option<P<Block>>),
+    /// A type.
+    TyAlias(Generics, GenericBounds, Option<P<Ty>>),
+    /// A macro expanding to items.
+    Macro(Mac),
 }
diff --git a/src/libsyntax/mut_visit.rs b/src/libsyntax/mut_visit.rs
index 2afe4159e93..92f20b719f8 100644
--- a/src/libsyntax/mut_visit.rs
+++ b/src/libsyntax/mut_visit.rs
@@ -890,13 +890,9 @@ pub fn noop_visit_item_kind<T: MutVisitor>(kind: &mut ItemKind, vis: &mut T) {
     match kind {
         ItemKind::ExternCrate(_orig_name) => {}
         ItemKind::Use(use_tree) => vis.visit_use_tree(use_tree),
-        ItemKind::Static(ty, _mut, expr) => {
+        ItemKind::Static(ty, _, expr) | ItemKind::Const(ty, expr) => {
             vis.visit_ty(ty);
-            vis.visit_expr(expr);
-        }
-        ItemKind::Const(ty, expr) => {
-            vis.visit_ty(ty);
-            vis.visit_expr(expr);
+            visit_opt(expr, |expr| vis.visit_expr(expr));
         }
         ItemKind::Fn(sig, generics, body) => {
             visit_fn_sig(sig, vis);
@@ -951,31 +947,43 @@ pub fn noop_flat_map_assoc_item<T: MutVisitor>(
     mut item: P<AssocItem>,
     visitor: &mut T,
 ) -> SmallVec<[P<AssocItem>; 1]> {
-    let AssocItem { id, ident, vis, defaultness: _, attrs, generics, kind, span, tokens: _ } =
+    let AssocItem { id, ident, vis, defaultness: _, attrs, kind, span, tokens: _ } =
         item.deref_mut();
+    walk_nested_item(visitor, id, span, ident, vis, attrs, kind);
+    smallvec![item]
+}
+
+pub fn walk_nested_item(
+    visitor: &mut impl MutVisitor,
+    id: &mut NodeId,
+    span: &mut Span,
+    ident: &mut Ident,
+    vis: &mut Visibility,
+    attrs: &mut Vec<Attribute>,
+    kind: &mut AssocItemKind,
+) {
     visitor.visit_id(id);
     visitor.visit_ident(ident);
     visitor.visit_vis(vis);
     visit_attrs(attrs, visitor);
-    visitor.visit_generics(generics);
     match kind {
-        AssocItemKind::Const(ty, expr) => {
+        AssocItemKind::Const(ty, expr) | AssocItemKind::Static(ty, _, expr) => {
             visitor.visit_ty(ty);
             visit_opt(expr, |expr| visitor.visit_expr(expr));
         }
-        AssocItemKind::Fn(sig, body) => {
+        AssocItemKind::Fn(sig, generics, body) => {
+            visitor.visit_generics(generics);
             visit_fn_sig(sig, visitor);
             visit_opt(body, |body| visitor.visit_block(body));
         }
-        AssocItemKind::TyAlias(bounds, ty) => {
+        AssocItemKind::TyAlias(generics, bounds, ty) => {
+            visitor.visit_generics(generics);
             visit_bounds(bounds, visitor);
             visit_opt(ty, |ty| visitor.visit_ty(ty));
         }
         AssocItemKind::Macro(mac) => visitor.visit_mac(mac),
     }
     visitor.visit_span(span);
-
-    smallvec![item]
 }
 
 pub fn noop_visit_fn_header<T: MutVisitor>(header: &mut FnHeader, vis: &mut T) {
@@ -1041,22 +1049,7 @@ pub fn noop_flat_map_foreign_item<T: MutVisitor>(
     visitor: &mut T,
 ) -> SmallVec<[P<ForeignItem>; 1]> {
     let ForeignItem { ident, attrs, id, kind, vis, span, tokens: _ } = item.deref_mut();
-    visitor.visit_ident(ident);
-    visit_attrs(attrs, visitor);
-    match kind {
-        ForeignItemKind::Fn(sig, generics, body) => {
-            visit_fn_sig(sig, visitor);
-            visitor.visit_generics(generics);
-            visit_opt(body, |body| visitor.visit_block(body));
-        }
-        ForeignItemKind::Static(t, _m) => visitor.visit_ty(t),
-        ForeignItemKind::Ty => {}
-        ForeignItemKind::Macro(mac) => visitor.visit_mac(mac),
-    }
-    visitor.visit_id(id);
-    visitor.visit_span(span);
-    visitor.visit_vis(vis);
-
+    walk_nested_item(visitor, id, span, ident, vis, attrs, kind);
     smallvec![item]
 }
 
diff --git a/src/libsyntax/visit.rs b/src/libsyntax/visit.rs
index 26f3773bea8..dedd42fe0f6 100644
--- a/src/libsyntax/visit.rs
+++ b/src/libsyntax/visit.rs
@@ -300,7 +300,7 @@ pub fn walk_item<'a, V: Visitor<'a>>(visitor: &mut V, item: &'a Item) {
         ItemKind::Use(ref use_tree) => visitor.visit_use_tree(use_tree, item.id, false),
         ItemKind::Static(ref typ, _, ref expr) | ItemKind::Const(ref typ, ref expr) => {
             visitor.visit_ty(typ);
-            visitor.visit_expr(expr);
+            walk_list!(visitor, visit_expr, expr);
         }
         ItemKind::Fn(ref sig, ref generics, ref body) => {
             visitor.visit_generics(generics);
@@ -525,21 +525,8 @@ pub fn walk_pat<'a, V: Visitor<'a>>(visitor: &mut V, pattern: &'a Pat) {
 }
 
 pub fn walk_foreign_item<'a, V: Visitor<'a>>(visitor: &mut V, item: &'a ForeignItem) {
-    visitor.visit_vis(&item.vis);
-    visitor.visit_ident(item.ident);
-
-    match item.kind {
-        ForeignItemKind::Fn(ref sig, ref generics, ref body) => {
-            visitor.visit_generics(generics);
-            let kind = FnKind::Fn(FnCtxt::Foreign, item.ident, sig, &item.vis, body.as_deref());
-            visitor.visit_fn(kind, item.span, item.id);
-        }
-        ForeignItemKind::Static(ref typ, _) => visitor.visit_ty(typ),
-        ForeignItemKind::Ty => (),
-        ForeignItemKind::Macro(ref mac) => visitor.visit_mac(mac),
-    }
-
-    walk_list!(visitor, visit_attribute, &item.attrs);
+    let ForeignItem { id, span, ident, vis, attrs, kind, tokens: _ } = item;
+    walk_nested_item(visitor, *id, *span, *ident, vis, attrs, kind, FnCtxt::Foreign);
 }
 
 pub fn walk_global_asm<'a, V: Visitor<'a>>(_: &mut V, _: &'a GlobalAsm) {
@@ -622,24 +609,39 @@ pub fn walk_fn<'a, V: Visitor<'a>>(visitor: &mut V, kind: FnKind<'a>, _span: Spa
 }
 
 pub fn walk_assoc_item<'a, V: Visitor<'a>>(visitor: &mut V, item: &'a AssocItem, ctxt: AssocCtxt) {
-    visitor.visit_vis(&item.vis);
-    visitor.visit_ident(item.ident);
-    walk_list!(visitor, visit_attribute, &item.attrs);
-    visitor.visit_generics(&item.generics);
-    match item.kind {
-        AssocItemKind::Const(ref ty, ref expr) => {
+    let AssocItem { id, span, ident, vis, attrs, kind, tokens: _, defaultness: _ } = item;
+    walk_nested_item(visitor, *id, *span, *ident, vis, attrs, kind, FnCtxt::Assoc(ctxt));
+}
+
+fn walk_nested_item<'a, V: Visitor<'a>>(
+    visitor: &mut V,
+    id: NodeId,
+    span: Span,
+    ident: Ident,
+    vis: &'a Visibility,
+    attrs: &'a [Attribute],
+    kind: &'a AssocItemKind,
+    ctxt: FnCtxt,
+) {
+    visitor.visit_vis(vis);
+    visitor.visit_ident(ident);
+    walk_list!(visitor, visit_attribute, attrs);
+    match kind {
+        AssocItemKind::Const(ty, expr) | AssocItemKind::Static(ty, _, expr) => {
             visitor.visit_ty(ty);
             walk_list!(visitor, visit_expr, expr);
         }
-        AssocItemKind::Fn(ref sig, ref body) => {
-            let kind = FnKind::Fn(FnCtxt::Assoc(ctxt), item.ident, sig, &item.vis, body.as_deref());
-            visitor.visit_fn(kind, item.span, item.id);
+        AssocItemKind::Fn(sig, generics, body) => {
+            visitor.visit_generics(generics);
+            let kind = FnKind::Fn(ctxt, ident, sig, vis, body.as_deref());
+            visitor.visit_fn(kind, span, id);
         }
-        AssocItemKind::TyAlias(ref bounds, ref ty) => {
+        AssocItemKind::TyAlias(generics, bounds, ty) => {
+            visitor.visit_generics(generics);
             walk_list!(visitor, visit_param_bound, bounds);
             walk_list!(visitor, visit_ty, ty);
         }
-        AssocItemKind::Macro(ref mac) => {
+        AssocItemKind::Macro(mac) => {
             visitor.visit_mac(mac);
         }
     }