Overhaul token collection.

This commit does the following.

- Renames `collect_tokens_trailing_token` as `collect_tokens`, because
  (a) it's annoying long, and (b) the `_trailing_token` bit is less
  accurate now that its types have changed.

- In `collect_tokens`, adds a `Option<CollectPos>` argument and a
  `UsePreAttrPos` in the return type of `f`. These are used in
  `parse_expr_force_collect` (for vanilla expressions) and in
  `parse_stmt_without_recovery` (for two different cases of expression
  statements). Together these ensure are enough to fix all the problems
  with token collection and assoc expressions. The changes to the
  `stringify.rs` test demonstrate some of these.

- Adds a new test. The code in this test was causing an assertion
  failure prior to this commit, due to an invalid `NodeRange`.

The extra complexity is annoying, but necessary to fix the existing
problems.

1 files changed, 76 insertions, 82 deletions

diff --git a/compiler/rustc_parse/src/parser/generics.rs b/compiler/rustc_parse/src/parser/generics.rs
index 47b801d1490..af3b6f740e3 100644
--- a/compiler/rustc_parse/src/parser/generics.rs
+++ b/compiler/rustc_parse/src/parser/generics.rs
@@ -7,7 +7,7 @@ use rustc_span::symbol::{kw, Ident};
 use rustc_span::Span;
 use thin_vec::ThinVec;
 
-use super::{ForceCollect, Parser, Trailing};
+use super::{ForceCollect, Parser, Trailing, UsePreAttrPos};
 use crate::errors::{
     self, MultipleWhereClauses, UnexpectedDefaultValueForLifetimeInGenericParameters,
     UnexpectedSelfInGenericParameters, WhereClauseBeforeTupleStructBody,
@@ -169,94 +169,88 @@ impl<'a> Parser<'a> {
         let mut done = false;
         while !done {
             let attrs = self.parse_outer_attributes()?;
-            let param =
-                self.collect_tokens_trailing_token(attrs, ForceCollect::No, |this, attrs| {
-                    if this.eat_keyword_noexpect(kw::SelfUpper) {
-                        // `Self` as a generic param is invalid. Here we emit the diagnostic and continue parsing
-                        // as if `Self` never existed.
-                        this.dcx().emit_err(UnexpectedSelfInGenericParameters {
-                            span: this.prev_token.span,
-                        });
+            let param = self.collect_tokens(None, attrs, ForceCollect::No, |this, attrs| {
+                if this.eat_keyword_noexpect(kw::SelfUpper) {
+                    // `Self` as a generic param is invalid. Here we emit the diagnostic and continue parsing
+                    // as if `Self` never existed.
+                    this.dcx()
+                        .emit_err(UnexpectedSelfInGenericParameters { span: this.prev_token.span });
+
+                    // Eat a trailing comma, if it exists.
+                    let _ = this.eat(&token::Comma);
+                }
+
+                let param = if this.check_lifetime() {
+                    let lifetime = this.expect_lifetime();
+                    // Parse lifetime parameter.
+                    let (colon_span, bounds) = if this.eat(&token::Colon) {
+                        (Some(this.prev_token.span), this.parse_lt_param_bounds())
+                    } else {
+                        (None, Vec::new())
+                    };
 
-                        // Eat a trailing comma, if it exists.
-                        let _ = this.eat(&token::Comma);
+                    if this.check_noexpect(&token::Eq) && this.look_ahead(1, |t| t.is_lifetime()) {
+                        let lo = this.token.span;
+                        // Parse `= 'lifetime`.
+                        this.bump(); // `=`
+                        this.bump(); // `'lifetime`
+                        let span = lo.to(this.prev_token.span);
+                        this.dcx().emit_err(UnexpectedDefaultValueForLifetimeInGenericParameters {
+                            span,
+                        });
                     }
 
-                    let param = if this.check_lifetime() {
-                        let lifetime = this.expect_lifetime();
-                        // Parse lifetime parameter.
-                        let (colon_span, bounds) = if this.eat(&token::Colon) {
-                            (Some(this.prev_token.span), this.parse_lt_param_bounds())
-                        } else {
-                            (None, Vec::new())
-                        };
-
-                        if this.check_noexpect(&token::Eq)
-                            && this.look_ahead(1, |t| t.is_lifetime())
-                        {
-                            let lo = this.token.span;
-                            // Parse `= 'lifetime`.
-                            this.bump(); // `=`
-                            this.bump(); // `'lifetime`
-                            let span = lo.to(this.prev_token.span);
-                            this.dcx().emit_err(
-                                UnexpectedDefaultValueForLifetimeInGenericParameters { span },
-                            );
+                    Some(ast::GenericParam {
+                        ident: lifetime.ident,
+                        id: lifetime.id,
+                        attrs,
+                        bounds,
+                        kind: ast::GenericParamKind::Lifetime,
+                        is_placeholder: false,
+                        colon_span,
+                    })
+                } else if this.check_keyword(kw::Const) {
+                    // Parse const parameter.
+                    Some(this.parse_const_param(attrs)?)
+                } else if this.check_ident() {
+                    // Parse type parameter.
+                    Some(this.parse_ty_param(attrs)?)
+                } else if this.token.can_begin_type() {
+                    // Trying to write an associated type bound? (#26271)
+                    let snapshot = this.create_snapshot_for_diagnostic();
+                    match this.parse_ty_where_predicate() {
+                        Ok(where_predicate) => {
+                            this.dcx().emit_err(errors::BadAssocTypeBounds {
+                                span: where_predicate.span(),
+                            });
+                            // FIXME - try to continue parsing other generics?
                         }
-
-                        Some(ast::GenericParam {
-                            ident: lifetime.ident,
-                            id: lifetime.id,
-                            attrs,
-                            bounds,
-                            kind: ast::GenericParamKind::Lifetime,
-                            is_placeholder: false,
-                            colon_span,
-                        })
-                    } else if this.check_keyword(kw::Const) {
-                        // Parse const parameter.
-                        Some(this.parse_const_param(attrs)?)
-                    } else if this.check_ident() {
-                        // Parse type parameter.
-                        Some(this.parse_ty_param(attrs)?)
-                    } else if this.token.can_begin_type() {
-                        // Trying to write an associated type bound? (#26271)
-                        let snapshot = this.create_snapshot_for_diagnostic();
-                        match this.parse_ty_where_predicate() {
-                            Ok(where_predicate) => {
-                                this.dcx().emit_err(errors::BadAssocTypeBounds {
-                                    span: where_predicate.span(),
-                                });
-                                // FIXME - try to continue parsing other generics?
-                                return Ok((None, Trailing::No));
-                            }
-                            Err(err) => {
-                                err.cancel();
-                                // FIXME - maybe we should overwrite 'self' outside of `collect_tokens`?
-                                this.restore_snapshot(snapshot);
-                                return Ok((None, Trailing::No));
-                            }
+                        Err(err) => {
+                            err.cancel();
+                            // FIXME - maybe we should overwrite 'self' outside of `collect_tokens`?
+                            this.restore_snapshot(snapshot);
                         }
-                    } else {
-                        // Check for trailing attributes and stop parsing.
-                        if !attrs.is_empty() {
-                            if !params.is_empty() {
-                                this.dcx()
-                                    .emit_err(errors::AttrAfterGeneric { span: attrs[0].span });
-                            } else {
-                                this.dcx()
-                                    .emit_err(errors::AttrWithoutGenerics { span: attrs[0].span });
-                            }
+                    }
+                    return Ok((None, Trailing::No, UsePreAttrPos::No));
+                } else {
+                    // Check for trailing attributes and stop parsing.
+                    if !attrs.is_empty() {
+                        if !params.is_empty() {
+                            this.dcx().emit_err(errors::AttrAfterGeneric { span: attrs[0].span });
+                        } else {
+                            this.dcx()
+                                .emit_err(errors::AttrWithoutGenerics { span: attrs[0].span });
                         }
-                        return Ok((None, Trailing::No));
-                    };
-
-                    if !this.eat(&token::Comma) {
-                        done = true;
                     }
-                    // We just ate the comma, so no need to capture the trailing token.
-                    Ok((param, Trailing::No))
-                })?;
+                    return Ok((None, Trailing::No, UsePreAttrPos::No));
+                };
+
+                if !this.eat(&token::Comma) {
+                    done = true;
+                }
+                // We just ate the comma, so no need to capture the trailing token.
+                Ok((param, Trailing::No, UsePreAttrPos::No))
+            })?;
 
             if let Some(param) = param {
                 params.push(param);