From 850c3219fb8659608eb62cd43eaee29e9e354379 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Thu, 3 Sep 2020 15:22:07 +0200 Subject: Move jointness censoring to proc_macro Proc-macro API currently exposes jointness in `Punct` tokens. That is, `+` in `+one` is **non** joint. Our lexer produces jointness info for all tokens, so we need to censor it *somewhere* Previously we did this in a lexer, but it makes more sense to do this in a proc-macro server. --- compiler/rustc_parse/src/lexer/tokentrees.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'compiler/rustc_parse/src') diff --git a/compiler/rustc_parse/src/lexer/tokentrees.rs b/compiler/rustc_parse/src/lexer/tokentrees.rs index d5977ca3c7d..fb27ccfbd94 100644 --- a/compiler/rustc_parse/src/lexer/tokentrees.rs +++ b/compiler/rustc_parse/src/lexer/tokentrees.rs @@ -262,10 +262,7 @@ impl<'a> TokenTreesReader<'a> { } _ => { let tt = TokenTree::Token(self.token.take()); - let mut is_joint = self.bump(); - if !self.token.is_op() { - is_joint = NonJoint; - } + let is_joint = self.bump(); Ok((tt, is_joint)) } } -- cgit 1.4.1-3-g733a5 From 09d3db2e590030de8ae7d00589f8a174e5f51f03 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Thu, 3 Sep 2020 23:26:59 +0200 Subject: Optimize Cursor::look_ahead Cloning a tt is cheap, but not free (there's Arc inside). --- compiler/rustc_ast/src/tokenstream.rs | 4 ++-- compiler/rustc_expand/src/proc_macro_server.rs | 11 ++++++++--- compiler/rustc_parse/src/parser/mod.rs | 10 +++++----- 3 files changed, 15 insertions(+), 10 deletions(-) (limited to 'compiler/rustc_parse/src') diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index 151acddae84..fb98f55a215 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -403,8 +403,8 @@ impl Cursor { self.index = index; } - pub fn look_ahead(&self, n: usize) -> Option { - self.stream.0[self.index..].get(n).map(|(tree, _)| tree.clone()) + pub fn look_ahead(&self, n: usize) -> Option<&TokenTree> { + self.stream.0[self.index..].get(n).map(|(tree, _)| tree) } } diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index 19c87d08a13..765871a6396 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -47,13 +47,18 @@ impl ToInternal for Delimiter { } } -impl FromInternal<(TreeAndJoint, Option, &'_ ParseSess, &'_ mut Vec)> - for TokenTree +impl + FromInternal<( + TreeAndJoint, + Option<&'_ tokenstream::TokenTree>, + &'_ ParseSess, + &'_ mut Vec, + )> for TokenTree { fn from_internal( ((tree, is_joint), look_ahead, sess, stack): ( TreeAndJoint, - Option, + Option<&tokenstream::TokenTree>, &ParseSess, &mut Vec, ), diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index 84edfecad19..1b2067f8f25 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -822,15 +822,15 @@ impl<'a> Parser<'a> { } let frame = &self.token_cursor.frame; - looker(&match frame.tree_cursor.look_ahead(dist - 1) { + match frame.tree_cursor.look_ahead(dist - 1) { Some(tree) => match tree { - TokenTree::Token(token) => token, + TokenTree::Token(token) => looker(token), TokenTree::Delimited(dspan, delim, _) => { - Token::new(token::OpenDelim(delim), dspan.open) + looker(&Token::new(token::OpenDelim(delim.clone()), dspan.open)) } }, - None => Token::new(token::CloseDelim(frame.delim), frame.span.close), - }) + None => looker(&Token::new(token::CloseDelim(frame.delim), frame.span.close)), + } } /// Returns whether any of the given keywords are `dist` tokens ahead of the current one. -- cgit 1.4.1-3-g733a5