rustc: Fix joint-ness of stringified token-streams

This commit fixes `StringReader`'s parsing of tokens which have been stringified
through procedural macros. Whether or not a token tree is joint is defined by
span information, but when working with procedural macros these spans are often
dummy and/or overridden which means that they end up considering all operators
joint if they can!

The fix here is to track the raw source span as opposed to the overridden span.
With this information we can more accurately classify `Punct` structs as either
joint or not.

Closes #50700

3 files changed, 42 insertions, 17 deletions

diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs
index bbece1ee5e3..89bf10244da 100644
--- a/src/libsyntax/parse/lexer/mod.rs
+++ b/src/libsyntax/parse/lexer/mod.rs
@@ -34,7 +34,10 @@ pub struct TokenAndSpan {
 
 impl Default for TokenAndSpan {
     fn default() -> Self {
-        TokenAndSpan { tok: token::Whitespace, sp: syntax_pos::DUMMY_SP }
+        TokenAndSpan {
+            tok: token::Whitespace,
+            sp: syntax_pos::DUMMY_SP,
+        }
     }
 }
 
@@ -54,8 +57,9 @@ pub struct StringReader<'a> {
     /// If part of a filemap is being re-lexed, this should be set to false.
     pub save_new_lines_and_multibyte: bool,
     // cached:
-    pub peek_tok: token::Token,
-    pub peek_span: Span,
+    peek_tok: token::Token,
+    peek_span: Span,
+    peek_span_src_raw: Span,
     pub fatal_errs: Vec<DiagnosticBuilder<'a>>,
     // cache a direct reference to the source text, so that we don't have to
     // retrieve it via `self.filemap.src.as_ref().unwrap()` all the time.
@@ -63,13 +67,20 @@ pub struct StringReader<'a> {
     /// Stack of open delimiters and their spans. Used for error message.
     token: token::Token,
     span: Span,
+    /// The raw source span which *does not* take `override_span` into account
+    span_src_raw: Span,
     open_braces: Vec<(token::DelimToken, Span)>,
     pub override_span: Option<Span>,
 }
 
 impl<'a> StringReader<'a> {
     fn mk_sp(&self, lo: BytePos, hi: BytePos) -> Span {
-        unwrap_or!(self.override_span, Span::new(lo, hi, NO_EXPANSION))
+        self.mk_sp_and_raw(lo, hi).0
+    }
+    fn mk_sp_and_raw(&self, lo: BytePos, hi: BytePos) -> (Span, Span) {
+        let raw = Span::new(lo, hi, NO_EXPANSION);
+        let real = unwrap_or!(self.override_span, raw);
+        (real, raw)
     }
     fn mk_ident(&self, string: &str) -> Ident {
         let mut ident = Ident::from_str(string);
@@ -121,6 +132,7 @@ impl<'a> StringReader<'a> {
             sp: self.peek_span,
         };
         self.advance_token()?;
+        self.span_src_raw = self.peek_span_src_raw;
         Ok(ret_val)
     }
 
@@ -180,10 +192,12 @@ impl<'a> StringReader<'a> {
             // dummy values; not read
             peek_tok: token::Eof,
             peek_span: syntax_pos::DUMMY_SP,
+            peek_span_src_raw: syntax_pos::DUMMY_SP,
             src,
             fatal_errs: Vec::new(),
             token: token::Eof,
             span: syntax_pos::DUMMY_SP,
+            span_src_raw: syntax_pos::DUMMY_SP,
             open_braces: Vec::new(),
             override_span: None,
         }
@@ -325,17 +339,25 @@ impl<'a> StringReader<'a> {
     fn advance_token(&mut self) -> Result<(), ()> {
         match self.scan_whitespace_or_comment() {
             Some(comment) => {
+                self.peek_span_src_raw = comment.sp;
                 self.peek_span = comment.sp;
                 self.peek_tok = comment.tok;
             }
             None => {
                 if self.is_eof() {
                     self.peek_tok = token::Eof;
-                    self.peek_span = self.mk_sp(self.filemap.end_pos, self.filemap.end_pos);
+                    let (real, raw) = self.mk_sp_and_raw(
+                        self.filemap.end_pos,
+                        self.filemap.end_pos,
+                    );
+                    self.peek_span = real;
+                    self.peek_span_src_raw = raw;
                 } else {
                     let start_bytepos = self.pos;
                     self.peek_tok = self.next_token_inner()?;
-                    self.peek_span = self.mk_sp(start_bytepos, self.pos);
+                    let (real, raw) = self.mk_sp_and_raw(start_bytepos, self.pos);
+                    self.peek_span = real;
+                    self.peek_span_src_raw = raw;
                 };
             }
         }
diff --git a/src/libsyntax/parse/lexer/tokentrees.rs b/src/libsyntax/parse/lexer/tokentrees.rs
index a2c81e24754..278b8c991f7 100644
--- a/src/libsyntax/parse/lexer/tokentrees.rs
+++ b/src/libsyntax/parse/lexer/tokentrees.rs
@@ -18,9 +18,7 @@ impl<'a> StringReader<'a> {
     pub fn parse_all_token_trees(&mut self) -> PResult<'a, TokenStream> {
         let mut tts = Vec::new();
         while self.token != token::Eof {
-            let tree = self.parse_token_tree()?;
-            let is_joint = tree.span().hi() == self.span.lo() && token::is_op(&self.token);
-            tts.push(if is_joint { tree.joint() } else { tree.into() });
+            tts.push(self.parse_token_tree()?);
         }
         Ok(TokenStream::concat(tts))
     }
@@ -32,19 +30,17 @@ impl<'a> StringReader<'a> {
             if let token::CloseDelim(..) = self.token {
                 return TokenStream::concat(tts);
             }
-            let tree = match self.parse_token_tree() {
-                Ok(tree) => tree,
+            match self.parse_token_tree() {
+                Ok(tree) => tts.push(tree),
                 Err(mut e) => {
                     e.emit();
                     return TokenStream::concat(tts);
                 }
-            };
-            let is_joint = tree.span().hi() == self.span.lo() && token::is_op(&self.token);
-            tts.push(if is_joint { tree.joint() } else { tree.into() });
+            }
         }
     }
 
-    fn parse_token_tree(&mut self) -> PResult<'a, TokenTree> {
+    fn parse_token_tree(&mut self) -> PResult<'a, TokenStream> {
         match self.token {
             token::Eof => {
                 let msg = "this file contains an un-closed delimiter";
@@ -115,7 +111,7 @@ impl<'a> StringReader<'a> {
                 Ok(TokenTree::Delimited(span, Delimited {
                     delim,
                     tts: tts.into(),
-                }))
+                }).into())
             },
             token::CloseDelim(_) => {
                 // An unexpected closing delimiter (i.e., there is no
@@ -127,8 +123,13 @@ impl<'a> StringReader<'a> {
             },
             _ => {
                 let tt = TokenTree::Token(self.span, self.token.clone());
+                // Note that testing for joint-ness here is done via the raw
+                // source span as the joint-ness is a property of the raw source
+                // rather than wanting to take `override_span` into account.
+                let raw = self.span_src_raw;
                 self.real_token();
-                Ok(tt)
+                let is_joint = raw.hi() == self.span_src_raw.lo() && token::is_op(&self.token);
+                Ok(if is_joint { tt.joint() } else { tt.into() })
             }
         }
     }
diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs
index 5575614a4d4..034be6a6864 100644
--- a/src/libsyntax/parse/token.rs
+++ b/src/libsyntax/parse/token.rs
@@ -581,6 +581,8 @@ impl Token {
             if tokens.probably_equal_for_proc_macro(&tokens_for_real) {
                 return tokens
             }
+            info!("cached tokens found, but they're not \"probably equal\", \
+                   going with stringified version");
         }
         return tokens_for_real
     }