Auto merge of #49939 - kennytm:rollup, r=kennytm

Rollup of 14 pull requests

Successful merges: #49908, #49876, #49916, #49951, #49465, #49922, #49866, #49915, #49886, #49913, #49852, #49958, #49871, #49864

Failed merges:

1 files changed, 22 insertions, 9 deletions

diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs
index 35a745ccdee..0913ed86147 100644
--- a/src/libsyntax/parse/token.rs
+++ b/src/libsyntax/parse/token.rs
@@ -527,8 +527,17 @@ impl Token {
         // all span information.
         //
         // As a result, some AST nodes are annotated with the token
-        // stream they came from. Attempt to extract these lossless
-        // token streams before we fall back to the stringification.
+        // stream they came from. Here we attempt to extract these
+        // lossless token streams before we fall back to the
+        // stringification.
+        //
+        // During early phases of the compiler, though, the AST could
+        // get modified directly (e.g. attributes added or removed) and
+        // the internal cache of tokens my not be invalidated or
+        // updated. Consequently if the "lossless" token stream
+        // disagrees with our actuall stringification (which has
+        // historically been much more battle-tested) then we go with
+        // the lossy stream anyway (losing span information).
         let mut tokens = None;
 
         match nt.0 {
@@ -555,13 +564,17 @@ impl Token {
             _ => {}
         }
 
-        tokens.unwrap_or_else(|| {
-            nt.1.force(|| {
-                // FIXME(jseyfried): Avoid this pretty-print + reparse hack
-                let source = pprust::token_to_string(self);
-                parse_stream_from_source_str(FileName::MacroExpansion, source, sess, Some(span))
-            })
-        })
+        let tokens_for_real = nt.1.force(|| {
+            // FIXME(#43081): Avoid this pretty-print + reparse hack
+            let source = pprust::token_to_string(self);
+            parse_stream_from_source_str(FileName::MacroExpansion, source, sess, Some(span))
+        });
+        if let Some(tokens) = tokens {
+            if tokens.eq_unspanned(&tokens_for_real) {
+                return tokens
+            }
+        }
+        return tokens_for_real
     }
 }