From deab741ab4c91c740b265db4d4950f31fc4f6cc3 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 21 Aug 2024 15:32:57 +1000 Subject: Clarify a comment. --- compiler/rustc_parse/src/parser/attr_wrapper.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'compiler/rustc_parse/src') diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index 49df2811d52..cfbac4876d5 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -510,9 +510,11 @@ fn make_attr_token_stream( } /// Tokens are needed if: -/// - any non-single-segment attributes (other than doc comments) are present; or -/// - any `cfg_attr` attributes are present; -/// - any single-segment, non-builtin attributes are present. +/// - any non-single-segment attributes (other than doc comments) are present, +/// e.g. `rustfmt::skip`; or +/// - any `cfg_attr` attributes are present; or +/// - any single-segment, non-builtin attributes are present, e.g. `derive`, +/// `test`, `global_allocator`. fn needs_tokens(attrs: &[ast::Attribute]) -> bool { attrs.iter().any(|attr| match attr.ident() { None => !attr.is_doc_comment(), -- cgit 1.4.1-3-g733a5 From 312ecdb2ed37249ad539bc0732278b80b4f3c7f8 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Tue, 20 Aug 2024 12:25:31 +1000 Subject: Avoid unnecessary `cloned`. --- compiler/rustc_parse/src/parser/attr_wrapper.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'compiler/rustc_parse/src') diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index cfbac4876d5..67d1e410c58 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -363,7 +363,7 @@ impl<'a> Parser<'a> { [parser_replacements_start..parser_replacements_end] .iter() .cloned() - .chain(inner_attr_parser_replacements.iter().cloned()) + .chain(inner_attr_parser_replacements.into_iter()) .map(|(parser_range, data)| { (NodeRange::new(parser_range, collect_pos.start_pos), data) }) -- cgit 1.4.1-3-g733a5 From 1ae521e9d57ba3a67b1007204da2836d8b19b4a2 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Tue, 20 Aug 2024 17:47:53 +1000 Subject: Return earlier in some cases in `collect_token`. This example triggers an assertion failure: ``` fn f() -> u32 { #[cfg_eval] #[cfg(not(FALSE))] 0 } ``` The sequence of events: - `configure_annotatable` calls `parse_expr_force_collect`, which calls `collect_tokens`. - Within that, we end up in `parse_expr_dot_or_call`, which again calls `collect_tokens`. - The return value of the `f` call is the expression `0`. - This inner call collects tokens for `0` (parser range 10..11) and creates a replacement covering `#[cfg(not(FALSE))] 0` (parser range 0..11). - We return to the outer `collect_tokens` call. The return value of the `f` call is *again* the expression `0`, again with the range 10..11, but the replacement from earlier covers the range 0..11. The code mistakenly assumes that any attributes from an inner `collect_tokens` call fit entirely within the body of the result of an outer `collect_tokens` call. So it adjusts the replacement parser range 0..11 to a node range by subtracting 10, resulting in -10..1. This is an invalid range and triggers an assertion failure. It's tricky to follow, but basically things get complicated when an AST node is returned from an inner `collect_tokens` call and then returned again from an outer `collect_token` node without being wrapped in any kind of additional layer. This commit changes `collect_tokens` to return early in some extra cases, avoiding the construction of lazy tokens. In the example above, the outer `collect_tokens` returns earlier because the `0` token already has tokens and `self.capture_state.capturing` is `Capturing::No`. This early return avoids the creation of the invalid range and the assertion failure. Fixes #129166. Note: these invalid ranges have been happening for a long time. #128725 looks like it's at fault only because it introduced the assertion that catches the invalid ranges. --- compiler/rustc_parse/src/parser/attr_wrapper.rs | 39 ++++++++++++---------- tests/crashes/129166.rs | 7 ---- .../invalid-node-range-issue-129166.rs | 11 ++++++ .../invalid-node-range-issue-129166.stderr | 8 +++++ 4 files changed, 41 insertions(+), 24 deletions(-) delete mode 100644 tests/crashes/129166.rs create mode 100644 tests/ui/conditional-compilation/invalid-node-range-issue-129166.rs create mode 100644 tests/ui/conditional-compilation/invalid-node-range-issue-129166.stderr (limited to 'compiler/rustc_parse/src') diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index 67d1e410c58..81b683705f3 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -234,6 +234,8 @@ impl<'a> Parser<'a> { force_collect: ForceCollect, f: impl FnOnce(&mut Self, AttrVec) -> PResult<'a, (R, Trailing, UsePreAttrPos)>, ) -> PResult<'a, R> { + let possible_capture_mode = self.capture_cfg; + // We must collect if anything could observe the collected tokens, i.e. // if any of the following conditions hold. // - We are force collecting tokens (because force collection requires @@ -244,9 +246,9 @@ impl<'a> Parser<'a> { // - Our target supports custom inner attributes (custom // inner attribute invocation might require token capturing). || R::SUPPORTS_CUSTOM_INNER_ATTRS - // - We are in `capture_cfg` mode (which requires tokens if + // - We are in "possible capture mode" (which requires tokens if // the parsed node has `#[cfg]` or `#[cfg_attr]` attributes). - || self.capture_cfg; + || possible_capture_mode; if !needs_collection { return Ok(f(self, attrs.attrs)?.0); } @@ -267,7 +269,7 @@ impl<'a> Parser<'a> { res? }; - // When we're not in `capture_cfg` mode, then skip collecting and + // When we're not in "definite capture mode", then skip collecting and // return early if either of the following conditions hold. // - `None`: Our target doesn't support tokens at all (e.g. `NtIdent`). // - `Some(Some(_))`: Our target already has tokens set (e.g. we've @@ -278,7 +280,10 @@ impl<'a> Parser<'a> { // Note that this check is independent of `force_collect`. There's no // need to collect tokens when we don't support tokens or already have // tokens. - if !self.capture_cfg && matches!(ret.tokens_mut(), None | Some(Some(_))) { + let definite_capture_mode = self.capture_cfg + && matches!(self.capture_state.capturing, Capturing::Yes) + && has_cfg_or_cfg_attr(ret.attrs()); + if !definite_capture_mode && matches!(ret.tokens_mut(), None | Some(Some(_))) { return Ok(ret); } @@ -298,11 +303,11 @@ impl<'a> Parser<'a> { // the earlier `needs_tokens` check, and we don't need to // check `R::SUPPORTS_CUSTOM_INNER_ATTRS`.) || needs_tokens(ret.attrs()) - // - We are in `capture_cfg` mode and there are `#[cfg]` or - // `#[cfg_attr]` attributes. (During normal non-`capture_cfg` - // parsing, we don't need any special capturing for those - // attributes, because they're builtin.) - || (self.capture_cfg && has_cfg_or_cfg_attr(ret.attrs())); + // - We are in "definite capture mode", which requires that there + // are `#[cfg]` or `#[cfg_attr]` attributes. (During normal + // non-`capture_cfg` parsing, we don't need any special capturing + // for those attributes, because they're builtin.) + || definite_capture_mode; if !needs_collection { return Ok(ret); } @@ -399,20 +404,18 @@ impl<'a> Parser<'a> { break_last_token: self.break_last_token, node_replacements, }); + let mut tokens_used = false; // If we support tokens and don't already have them, store the newly captured tokens. if let Some(target_tokens @ None) = ret.tokens_mut() { + tokens_used = true; *target_tokens = Some(tokens.clone()); } - // If `capture_cfg` is set and we're inside a recursive call to - // `collect_tokens`, then we need to register a replace range if we - // have `#[cfg]` or `#[cfg_attr]`. This allows us to run eager - // cfg-expansion on the captured token stream. - if self.capture_cfg - && matches!(self.capture_state.capturing, Capturing::Yes) - && has_cfg_or_cfg_attr(ret.attrs()) - { + // If in "definite capture mode" we need to register a replace range + // for the `#[cfg]` and/or `#[cfg_attr]` attrs. This allows us to run + // eager cfg-expansion on the captured token stream. + if definite_capture_mode { assert!(!self.break_last_token, "Should not have unglued last token with cfg attr"); // What is the status here when parsing the example code at the top of this method? @@ -430,6 +433,7 @@ impl<'a> Parser<'a> { let start_pos = if has_outer_attrs { attrs.start_pos.unwrap() } else { collect_pos.start_pos }; let target = AttrsTarget { attrs: ret.attrs().iter().cloned().collect(), tokens }; + tokens_used = true; self.capture_state .parser_replacements .push((ParserRange(start_pos..end_pos), Some(target))); @@ -439,6 +443,7 @@ impl<'a> Parser<'a> { self.capture_state.parser_replacements.clear(); self.capture_state.inner_attr_parser_ranges.clear(); } + assert!(tokens_used); // check we didn't create `tokens` unnecessarily Ok(ret) } } diff --git a/tests/crashes/129166.rs b/tests/crashes/129166.rs deleted file mode 100644 index d3635d410db..00000000000 --- a/tests/crashes/129166.rs +++ /dev/null @@ -1,7 +0,0 @@ -//@ known-bug: rust-lang/rust#129166 - -fn main() { - #[cfg_eval] - #[cfg] - 0 -} diff --git a/tests/ui/conditional-compilation/invalid-node-range-issue-129166.rs b/tests/ui/conditional-compilation/invalid-node-range-issue-129166.rs new file mode 100644 index 00000000000..794e6fad3fc --- /dev/null +++ b/tests/ui/conditional-compilation/invalid-node-range-issue-129166.rs @@ -0,0 +1,11 @@ +// This was triggering an assertion failure in `NodeRange::new`. + +#![feature(cfg_eval)] +#![feature(stmt_expr_attributes)] + +fn f() -> u32 { + #[cfg_eval] #[cfg(not(FALSE))] 0 + //~^ ERROR removing an expression is not supported in this position +} + +fn main() {} diff --git a/tests/ui/conditional-compilation/invalid-node-range-issue-129166.stderr b/tests/ui/conditional-compilation/invalid-node-range-issue-129166.stderr new file mode 100644 index 00000000000..0699e182bd5 --- /dev/null +++ b/tests/ui/conditional-compilation/invalid-node-range-issue-129166.stderr @@ -0,0 +1,8 @@ +error: removing an expression is not supported in this position + --> $DIR/invalid-node-range-issue-129166.rs:7:17 + | +LL | #[cfg_eval] #[cfg(not(FALSE))] 0 + | ^^^^^^^^^^^^^^^^^^ + +error: aborting due to 1 previous error + -- cgit 1.4.1-3-g733a5 From 0bae33fcd503473aec70aef28b0e08abce965557 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Tue, 20 Aug 2024 12:28:39 +1000 Subject: Avoid nested replacement ranges. In a case like this: ``` mod a { mod b { #[cfg_attr(unix, inline)] fn f() { #[cfg_attr(linux, inline)] fn g1() {} #[cfg_attr(linux, inline)] fn g2() {} } } } ``` We currently end up with the following replacement ranges. - The lazy tokens for `f` has replacement ranges for `g1` and `g2`. - The lazy tokens for `a` has replacement ranges for `f`, `g1`, and `g2`. I.e. the replacement ranges for `g1` and `g2` are duplicated. In general, replacement ranges for inner AST nodes are duplicated up the chain for each nested `collect_tokens` call. And the code that processes the replacements is careful about the ordering in which the replacements are applied, to ensure that inner replacements are applied before outer replacements. But all of this is unnecessary. If you apply an inner replacement and then an outer replacement, the outer replacement completely overwrites the inner replacement. This commit avoids the duplication by removing replacements from `self.capture_state.parser_replacements` when they are used. (The effect on the example above is that the lazy tokesn for `a` no longer include replacement ranges for `g1` and `g2`.) This eliminates the possibility of nested replacements on individual AST nodes, which avoids the need for careful ordering of replacements. --- compiler/rustc_parse/src/parser/attr_wrapper.rs | 28 +++++++------------------ 1 file changed, 7 insertions(+), 21 deletions(-) (limited to 'compiler/rustc_parse/src') diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index 81b683705f3..a74c87ca2a7 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -134,9 +134,8 @@ impl ToAttrTokenStream for LazyAttrTokenStreamImpl { node_replacements.array_windows() { assert!( - node_range.0.end <= next_node_range.0.start - || node_range.0.end >= next_node_range.0.end, - "Node ranges should be disjoint or nested: ({:?}, {:?}) ({:?}, {:?})", + node_range.0.end <= next_node_range.0.start, + "Node ranges should be disjoint: ({:?}, {:?}) ({:?}, {:?})", node_range, tokens, next_node_range, @@ -144,20 +143,8 @@ impl ToAttrTokenStream for LazyAttrTokenStreamImpl { ); } - // Process the replace ranges, starting from the highest start - // position and working our way back. If have tokens like: - // - // `#[cfg(FALSE)] struct Foo { #[cfg(FALSE)] field: bool }` - // - // Then we will generate replace ranges for both - // the `#[cfg(FALSE)] field: bool` and the entire - // `#[cfg(FALSE)] struct Foo { #[cfg(FALSE)] field: bool }` - // - // By starting processing from the replace range with the greatest - // start position, we ensure that any (outer) replace range which - // encloses another (inner) replace range will fully overwrite the - // inner range's replacement. - for (node_range, target) in node_replacements.into_iter().rev() { + // Process the replace ranges. + for (node_range, target) in node_replacements.into_iter() { assert!( !node_range.0.is_empty(), "Cannot replace an empty node range: {:?}", @@ -364,10 +351,9 @@ impl<'a> Parser<'a> { // from `ParserRange` form to `NodeRange` form. We will perform the actual // replacement only when we convert the `LazyAttrTokenStream` to an // `AttrTokenStream`. - self.capture_state.parser_replacements - [parser_replacements_start..parser_replacements_end] - .iter() - .cloned() + self.capture_state + .parser_replacements + .drain(parser_replacements_start..parser_replacements_end) .chain(inner_attr_parser_replacements.into_iter()) .map(|(parser_range, data)| { (NodeRange::new(parser_range, collect_pos.start_pos), data) -- cgit 1.4.1-3-g733a5 From 39b38a94e397bb2f55bb7b8e2a5cd69f649cadc1 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Thu, 22 Aug 2024 06:58:31 +1000 Subject: Split the assertion in `NodeRange::new`. --- compiler/rustc_parse/src/parser/mod.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'compiler/rustc_parse/src') diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index 61e3fa2e6c5..b72957ace52 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -238,7 +238,8 @@ impl NodeRange { // is the position of the function's start token. This gives // `NodeRange(10..15)`. fn new(ParserRange(parser_range): ParserRange, start_pos: u32) -> NodeRange { - assert!(parser_range.start >= start_pos && parser_range.end >= start_pos); + assert!(!parser_range.is_empty()); + assert!(parser_range.start >= start_pos); NodeRange((parser_range.start - start_pos)..(parser_range.end - start_pos)) } } -- cgit 1.4.1-3-g733a5 From 1fdabfbebbb12b2836f91aeec3157fd092f9a8ad Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 21 Aug 2024 14:16:42 +1000 Subject: Avoid double-handling of attributes in `collect_tokens`. By keeping track of attributes that have been previously processed. This fixes the `macro-rules-derive-cfg.stdout` test, and is necessary for #124141 which removes nonterminals. Also shrink the `SmallVec` inline size used in `IntervalSet`. 2 gives slightly better perf than 4 now that there's an `IntervalSet` in `Parser`, which is cloned reasonably often. --- Cargo.lock | 1 + compiler/rustc_index/src/interval.rs | 2 +- compiler/rustc_parse/Cargo.toml | 1 + compiler/rustc_parse/src/parser/attr_wrapper.rs | 23 ++++++++++++++--- compiler/rustc_parse/src/parser/mod.rs | 7 ++++- tests/ui/proc-macro/macro-rules-derive-cfg.stdout | 31 ++--------------------- 6 files changed, 30 insertions(+), 35 deletions(-) (limited to 'compiler/rustc_parse/src') diff --git a/Cargo.lock b/Cargo.lock index a18219b5683..4ce6029afe8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4172,6 +4172,7 @@ dependencies = [ "rustc_errors", "rustc_feature", "rustc_fluent_macro", + "rustc_index", "rustc_lexer", "rustc_macros", "rustc_session", diff --git a/compiler/rustc_index/src/interval.rs b/compiler/rustc_index/src/interval.rs index be028feca60..503470f896d 100644 --- a/compiler/rustc_index/src/interval.rs +++ b/compiler/rustc_index/src/interval.rs @@ -18,7 +18,7 @@ mod tests; #[derive(Debug, Clone)] pub struct IntervalSet { // Start, end - map: SmallVec<[(u32, u32); 4]>, + map: SmallVec<[(u32, u32); 2]>, domain: usize, _data: PhantomData, } diff --git a/compiler/rustc_parse/Cargo.toml b/compiler/rustc_parse/Cargo.toml index b33896154f2..c59ae48a07d 100644 --- a/compiler/rustc_parse/Cargo.toml +++ b/compiler/rustc_parse/Cargo.toml @@ -12,6 +12,7 @@ rustc_data_structures = { path = "../rustc_data_structures" } rustc_errors = { path = "../rustc_errors" } rustc_feature = { path = "../rustc_feature" } rustc_fluent_macro = { path = "../rustc_fluent_macro" } +rustc_index = { path = "../rustc_index" } rustc_lexer = { path = "../rustc_lexer" } rustc_macros = { path = "../rustc_macros" } rustc_session = { path = "../rustc_session" } diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index a74c87ca2a7..b2abe5464b4 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -256,6 +256,20 @@ impl<'a> Parser<'a> { res? }; + // Ignore any attributes we've previously processed. This happens when + // an inner call to `collect_tokens` returns an AST node and then an + // outer call ends up with the same AST node without any additional + // wrapping layer. + let ret_attrs: AttrVec = ret + .attrs() + .iter() + .cloned() + .filter(|attr| { + let is_unseen = self.capture_state.seen_attrs.insert(attr.id); + is_unseen + }) + .collect(); + // When we're not in "definite capture mode", then skip collecting and // return early if either of the following conditions hold. // - `None`: Our target doesn't support tokens at all (e.g. `NtIdent`). @@ -269,7 +283,7 @@ impl<'a> Parser<'a> { // tokens. let definite_capture_mode = self.capture_cfg && matches!(self.capture_state.capturing, Capturing::Yes) - && has_cfg_or_cfg_attr(ret.attrs()); + && has_cfg_or_cfg_attr(&ret_attrs); if !definite_capture_mode && matches!(ret.tokens_mut(), None | Some(Some(_))) { return Ok(ret); } @@ -289,7 +303,7 @@ impl<'a> Parser<'a> { // outer and inner attributes. So this check is more precise than // the earlier `needs_tokens` check, and we don't need to // check `R::SUPPORTS_CUSTOM_INNER_ATTRS`.) - || needs_tokens(ret.attrs()) + || needs_tokens(&ret_attrs) // - We are in "definite capture mode", which requires that there // are `#[cfg]` or `#[cfg_attr]` attributes. (During normal // non-`capture_cfg` parsing, we don't need any special capturing @@ -328,7 +342,7 @@ impl<'a> Parser<'a> { // `Parser::parse_inner_attributes`, and pair them in a `ParserReplacement` with `None`, // which means the relevant tokens will be removed. (More details below.) let mut inner_attr_parser_replacements = Vec::new(); - for attr in ret.attrs() { + for attr in ret_attrs.iter() { if attr.style == ast::AttrStyle::Inner { if let Some(inner_attr_parser_range) = self.capture_state.inner_attr_parser_ranges.remove(&attr.id) @@ -418,7 +432,7 @@ impl<'a> Parser<'a> { // cfg-expand this AST node. let start_pos = if has_outer_attrs { attrs.start_pos.unwrap() } else { collect_pos.start_pos }; - let target = AttrsTarget { attrs: ret.attrs().iter().cloned().collect(), tokens }; + let target = AttrsTarget { attrs: ret_attrs, tokens }; tokens_used = true; self.capture_state .parser_replacements @@ -428,6 +442,7 @@ impl<'a> Parser<'a> { // the outermost call to this method. self.capture_state.parser_replacements.clear(); self.capture_state.inner_attr_parser_ranges.clear(); + self.capture_state.seen_attrs.clear(); } assert!(tokens_used); // check we didn't create `tokens` unnecessarily Ok(ret) diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index b72957ace52..3ffc5fa4011 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -35,6 +35,7 @@ use rustc_ast_pretty::pprust; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::sync::Lrc; use rustc_errors::{Applicability, Diag, FatalError, MultiSpan, PResult}; +use rustc_index::interval::IntervalSet; use rustc_session::parse::ParseSess; use rustc_span::symbol::{kw, sym, Ident, Symbol}; use rustc_span::{Span, DUMMY_SP}; @@ -183,7 +184,7 @@ pub struct Parser<'a> { // This type is used a lot, e.g. it's cloned when matching many declarative macro rules with nonterminals. Make sure // it doesn't unintentionally get bigger. #[cfg(target_pointer_width = "64")] -rustc_data_structures::static_assert_size!(Parser<'_>, 256); +rustc_data_structures::static_assert_size!(Parser<'_>, 288); /// Stores span information about a closure. #[derive(Clone, Debug)] @@ -261,6 +262,9 @@ struct CaptureState { capturing: Capturing, parser_replacements: Vec, inner_attr_parser_ranges: FxHashMap, + // `IntervalSet` is good for perf because attrs are mostly added to this + // set in contiguous ranges. + seen_attrs: IntervalSet, } /// Iterator over a `TokenStream` that produces `Token`s. It's a bit odd that @@ -458,6 +462,7 @@ impl<'a> Parser<'a> { capturing: Capturing::No, parser_replacements: Vec::new(), inner_attr_parser_ranges: Default::default(), + seen_attrs: IntervalSet::new(u32::MAX as usize), }, current_closure: None, recovery: Recovery::Allowed, diff --git a/tests/ui/proc-macro/macro-rules-derive-cfg.stdout b/tests/ui/proc-macro/macro-rules-derive-cfg.stdout index 5205a4adf25..c1e46b50d40 100644 --- a/tests/ui/proc-macro/macro-rules-derive-cfg.stdout +++ b/tests/ui/proc-macro/macro-rules-derive-cfg.stdout @@ -1,11 +1,9 @@ PRINT-DERIVE INPUT (DISPLAY): struct Foo([bool; #[rustc_dummy(first)] #[rustc_dummy(second)] -{ #![rustc_dummy(third)] #[rustc_dummy(fourth)] #[rustc_dummy(fourth)] 30 }]); +{ #![rustc_dummy(third)] #[rustc_dummy(fourth)] 30 }]); PRINT-DERIVE DEEP-RE-COLLECTED (DISPLAY): struct Foo([bool; #[rustc_dummy(first)] #[rustc_dummy(second)] -{ - #! [rustc_dummy(third)] #[rustc_dummy(fourth)] #[rustc_dummy(fourth)] 30 -}]); +{ #! [rustc_dummy(third)] #[rustc_dummy(fourth)] 30 }]); PRINT-DERIVE INPUT (DEBUG): TokenStream [ Ident { ident: "struct", @@ -138,31 +136,6 @@ PRINT-DERIVE INPUT (DEBUG): TokenStream [ ], span: $DIR/macro-rules-derive-cfg.rs:25:6: 25:49 (#0), }, - Punct { - ch: '#', - spacing: Alone, - span: $DIR/macro-rules-derive-cfg.rs:25:5: 25:6 (#0), - }, - Group { - delimiter: Bracket, - stream: TokenStream [ - Ident { - ident: "rustc_dummy", - span: $DIR/macro-rules-derive-cfg.rs:25:28: 25:39 (#0), - }, - Group { - delimiter: Parenthesis, - stream: TokenStream [ - Ident { - ident: "fourth", - span: $DIR/macro-rules-derive-cfg.rs:25:40: 25:46 (#0), - }, - ], - span: $DIR/macro-rules-derive-cfg.rs:25:39: 25:47 (#0), - }, - ], - span: $DIR/macro-rules-derive-cfg.rs:25:6: 25:49 (#0), - }, Literal { kind: Integer, symbol: "30", -- cgit 1.4.1-3-g733a5 From d4bf28c014772f7233fc396992164c72c08fbfd9 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Thu, 22 Aug 2024 13:14:40 +1000 Subject: Optimize `collect_tokens` a little. Use `Cow` to avoid cloning `ret.attrs()` unless necessary. This requires moving some things around to satisfy the borrow checker. --- compiler/rustc_parse/src/parser/attr_wrapper.rs | 63 ++++++++++++++++--------- 1 file changed, 40 insertions(+), 23 deletions(-) (limited to 'compiler/rustc_parse/src') diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index b2abe5464b4..205cca830b2 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -1,3 +1,4 @@ +use std::borrow::Cow; use std::{iter, mem}; use rustc_ast::token::{Delimiter, Token, TokenKind}; @@ -6,6 +7,7 @@ use rustc_ast::tokenstream::{ Spacing, ToAttrTokenStream, }; use rustc_ast::{self as ast, AttrVec, Attribute, HasAttrs, HasTokens}; +use rustc_data_structures::fx::FxHashSet; use rustc_errors::PResult; use rustc_session::parse::ParseSess; use rustc_span::{sym, Span, DUMMY_SP}; @@ -256,27 +258,40 @@ impl<'a> Parser<'a> { res? }; + // - `None`: Our target doesn't support tokens at all (e.g. `NtIdent`). + // - `Some(None)`: Our target supports tokens and has none. + // - `Some(Some(_))`: Our target already has tokens set (e.g. we've + // parsed something like `#[my_attr] $item`). + let ret_can_hold_tokens = matches!(ret.tokens_mut(), Some(None)); + // Ignore any attributes we've previously processed. This happens when // an inner call to `collect_tokens` returns an AST node and then an // outer call ends up with the same AST node without any additional // wrapping layer. - let ret_attrs: AttrVec = ret - .attrs() - .iter() - .cloned() - .filter(|attr| { - let is_unseen = self.capture_state.seen_attrs.insert(attr.id); - is_unseen - }) - .collect(); + let mut seen_indices = FxHashSet::default(); + for (i, attr) in ret.attrs().iter().enumerate() { + let is_unseen = self.capture_state.seen_attrs.insert(attr.id); + if !is_unseen { + seen_indices.insert(i); + } + } + let ret_attrs: Cow<'_, [Attribute]> = + if seen_indices.is_empty() { + Cow::Borrowed(ret.attrs()) + } else { + let ret_attrs = + ret.attrs() + .iter() + .enumerate() + .filter_map(|(i, attr)| { + if seen_indices.contains(&i) { None } else { Some(attr.clone()) } + }) + .collect(); + Cow::Owned(ret_attrs) + }; // When we're not in "definite capture mode", then skip collecting and - // return early if either of the following conditions hold. - // - `None`: Our target doesn't support tokens at all (e.g. `NtIdent`). - // - `Some(Some(_))`: Our target already has tokens set (e.g. we've - // parsed something like `#[my_attr] $item`). The actual parsing code - // takes care of prepending any attributes to the nonterminal, so we - // don't need to modify the already captured tokens. + // return early if `ret` doesn't support tokens or already has some. // // Note that this check is independent of `force_collect`. There's no // need to collect tokens when we don't support tokens or already have @@ -284,7 +299,7 @@ impl<'a> Parser<'a> { let definite_capture_mode = self.capture_cfg && matches!(self.capture_state.capturing, Capturing::Yes) && has_cfg_or_cfg_attr(&ret_attrs); - if !definite_capture_mode && matches!(ret.tokens_mut(), None | Some(Some(_))) { + if !definite_capture_mode && !ret_can_hold_tokens { return Ok(ret); } @@ -406,12 +421,6 @@ impl<'a> Parser<'a> { }); let mut tokens_used = false; - // If we support tokens and don't already have them, store the newly captured tokens. - if let Some(target_tokens @ None) = ret.tokens_mut() { - tokens_used = true; - *target_tokens = Some(tokens.clone()); - } - // If in "definite capture mode" we need to register a replace range // for the `#[cfg]` and/or `#[cfg_attr]` attrs. This allows us to run // eager cfg-expansion on the captured token stream. @@ -432,7 +441,8 @@ impl<'a> Parser<'a> { // cfg-expand this AST node. let start_pos = if has_outer_attrs { attrs.start_pos.unwrap() } else { collect_pos.start_pos }; - let target = AttrsTarget { attrs: ret_attrs, tokens }; + let target = + AttrsTarget { attrs: ret_attrs.iter().cloned().collect(), tokens: tokens.clone() }; tokens_used = true; self.capture_state .parser_replacements @@ -444,6 +454,13 @@ impl<'a> Parser<'a> { self.capture_state.inner_attr_parser_ranges.clear(); self.capture_state.seen_attrs.clear(); } + + // If we support tokens and don't already have them, store the newly captured tokens. + if let Some(target_tokens @ None) = ret.tokens_mut() { + tokens_used = true; + *target_tokens = Some(tokens); + } + assert!(tokens_used); // check we didn't create `tokens` unnecessarily Ok(ret) } -- cgit 1.4.1-3-g733a5 From 9aaf873396186ea0c980cbea7d3449a355ccfeea Mon Sep 17 00:00:00 2001 From: Michael Goulet Date: Mon, 26 Aug 2024 11:11:13 -0400 Subject: Reserve prefix lifetimes too --- compiler/rustc_lexer/src/lib.rs | 26 +++++++++++++++++--------- compiler/rustc_parse/src/lexer/mod.rs | 10 ++++++++++ tests/ui/lexer/prefixed-lifetime.rs | 10 ++++++++++ tests/ui/lexer/prefixed-lifetime.stderr | 14 ++++++++++++++ 4 files changed, 51 insertions(+), 9 deletions(-) create mode 100644 tests/ui/lexer/prefixed-lifetime.rs create mode 100644 tests/ui/lexer/prefixed-lifetime.stderr (limited to 'compiler/rustc_parse/src') diff --git a/compiler/rustc_lexer/src/lib.rs b/compiler/rustc_lexer/src/lib.rs index 31fdd2d7cec..be3e151ea61 100644 --- a/compiler/rustc_lexer/src/lib.rs +++ b/compiler/rustc_lexer/src/lib.rs @@ -91,6 +91,12 @@ pub enum TokenKind { /// tokens. UnknownPrefix, + /// An unknown prefix in a lifetime, like `'foo#`. + /// + /// Note that like above, only the `'` and prefix are included in the token + /// and not the separator. + UnknownPrefixLifetime, + /// Similar to the above, but *always* an error on every edition. This is used /// for emoji identifier recovery, as those are not meant to be ever accepted. InvalidPrefix, @@ -688,15 +694,17 @@ impl Cursor<'_> { self.bump(); self.eat_while(is_id_continue); - // Check if after skipping literal contents we've met a closing - // single quote (which means that user attempted to create a - // string with single quotes). - if self.first() == '\'' { - self.bump(); - let kind = Char { terminated: true }; - Literal { kind, suffix_start: self.pos_within_token() } - } else { - Lifetime { starts_with_number } + match self.first() { + // Check if after skipping literal contents we've met a closing + // single quote (which means that user attempted to create a + // string with single quotes). + '\'' => { + self.bump(); + let kind = Char { terminated: true }; + Literal { kind, suffix_start: self.pos_within_token() } + } + '#' if !starts_with_number => UnknownPrefixLifetime, + _ => Lifetime { starts_with_number }, } } diff --git a/compiler/rustc_parse/src/lexer/mod.rs b/compiler/rustc_parse/src/lexer/mod.rs index f30939093c2..5d2d5a6010f 100644 --- a/compiler/rustc_parse/src/lexer/mod.rs +++ b/compiler/rustc_parse/src/lexer/mod.rs @@ -205,6 +205,16 @@ impl<'psess, 'src> StringReader<'psess, 'src> { self.report_unknown_prefix(start); self.ident(start) } + rustc_lexer::TokenKind::UnknownPrefixLifetime => { + self.report_unknown_prefix(start); + // Include the leading `'` in the real identifier, for macro + // expansion purposes. See #12512 for the gory details of why + // this is necessary. + let lifetime_name = self.str_from(start); + self.last_lifetime = Some(self.mk_sp(start, start + BytePos(1))); + let ident = Symbol::intern(lifetime_name); + token::Lifetime(ident, IdentIsRaw::No) + } rustc_lexer::TokenKind::InvalidIdent | rustc_lexer::TokenKind::InvalidPrefix // Do not recover an identifier with emoji if the codepoint is a confusable diff --git a/tests/ui/lexer/prefixed-lifetime.rs b/tests/ui/lexer/prefixed-lifetime.rs new file mode 100644 index 00000000000..6191b97d576 --- /dev/null +++ b/tests/ui/lexer/prefixed-lifetime.rs @@ -0,0 +1,10 @@ +//@ edition: 2021 + +macro_rules! w { + ($($tt:tt)*) => {}; +} + +w!('foo#lifetime); +//~^ ERROR prefix `'foo` is unknown + +fn main() {} diff --git a/tests/ui/lexer/prefixed-lifetime.stderr b/tests/ui/lexer/prefixed-lifetime.stderr new file mode 100644 index 00000000000..39e8b5a2a57 --- /dev/null +++ b/tests/ui/lexer/prefixed-lifetime.stderr @@ -0,0 +1,14 @@ +error: prefix `'foo` is unknown + --> $DIR/prefixed-lifetime.rs:7:4 + | +LL | w!('foo#lifetime); + | ^^^^ unknown prefix + | + = note: prefixed identifiers and literals are reserved since Rust 2021 +help: consider inserting whitespace here + | +LL | w!('foo #lifetime); + | + + +error: aborting due to 1 previous error + -- cgit 1.4.1-3-g733a5 From 3b3e43a386a9b89609fe529921bdd49ba3511fb8 Mon Sep 17 00:00:00 2001 From: Michael Goulet Date: Mon, 26 Aug 2024 11:26:36 -0400 Subject: Format lexer --- compiler/rustc_lexer/src/lib.rs | 2 +- compiler/rustc_parse/src/lexer/mod.rs | 41 +++++++++++++++++++---------------- 2 files changed, 23 insertions(+), 20 deletions(-) (limited to 'compiler/rustc_parse/src') diff --git a/compiler/rustc_lexer/src/lib.rs b/compiler/rustc_lexer/src/lib.rs index be3e151ea61..6561e416cfa 100644 --- a/compiler/rustc_lexer/src/lib.rs +++ b/compiler/rustc_lexer/src/lib.rs @@ -92,7 +92,7 @@ pub enum TokenKind { UnknownPrefix, /// An unknown prefix in a lifetime, like `'foo#`. - /// + /// /// Note that like above, only the `'` and prefix are included in the token /// and not the separator. UnknownPrefixLifetime, diff --git a/compiler/rustc_parse/src/lexer/mod.rs b/compiler/rustc_parse/src/lexer/mod.rs index 5d2d5a6010f..c0731cf31c3 100644 --- a/compiler/rustc_parse/src/lexer/mod.rs +++ b/compiler/rustc_parse/src/lexer/mod.rs @@ -188,9 +188,7 @@ impl<'psess, 'src> StringReader<'psess, 'src> { preceded_by_whitespace = true; continue; } - rustc_lexer::TokenKind::Ident => { - self.ident(start) - } + rustc_lexer::TokenKind::Ident => self.ident(start), rustc_lexer::TokenKind::RawIdent => { let sym = nfc_normalize(self.str_from(start + BytePos(2))); let span = self.mk_sp(start, self.pos); @@ -215,20 +213,21 @@ impl<'psess, 'src> StringReader<'psess, 'src> { let ident = Symbol::intern(lifetime_name); token::Lifetime(ident, IdentIsRaw::No) } - rustc_lexer::TokenKind::InvalidIdent - | rustc_lexer::TokenKind::InvalidPrefix + rustc_lexer::TokenKind::InvalidIdent | rustc_lexer::TokenKind::InvalidPrefix // Do not recover an identifier with emoji if the codepoint is a confusable // with a recoverable substitution token, like `➖`. - if !UNICODE_ARRAY - .iter() - .any(|&(c, _, _)| { - let sym = self.str_from(start); - sym.chars().count() == 1 && c == sym.chars().next().unwrap() - }) => + if !UNICODE_ARRAY.iter().any(|&(c, _, _)| { + let sym = self.str_from(start); + sym.chars().count() == 1 && c == sym.chars().next().unwrap() + }) => { let sym = nfc_normalize(self.str_from(start)); let span = self.mk_sp(start, self.pos); - self.psess.bad_unicode_identifiers.borrow_mut().entry(sym).or_default() + self.psess + .bad_unicode_identifiers + .borrow_mut() + .entry(sym) + .or_default() .push(span); token::Ident(sym, IdentIsRaw::No) } @@ -259,9 +258,9 @@ impl<'psess, 'src> StringReader<'psess, 'src> { let suffix = if suffix_start < self.pos { let string = self.str_from(suffix_start); if string == "_" { - self - .dcx() - .emit_err(errors::UnderscoreLiteralSuffix { span: self.mk_sp(suffix_start, self.pos) }); + self.dcx().emit_err(errors::UnderscoreLiteralSuffix { + span: self.mk_sp(suffix_start, self.pos), + }); None } else { Some(Symbol::intern(string)) @@ -279,7 +278,8 @@ impl<'psess, 'src> StringReader<'psess, 'src> { self.last_lifetime = Some(self.mk_sp(start, start + BytePos(1))); if starts_with_number { let span = self.mk_sp(start, self.pos); - self.dcx().struct_err("lifetimes cannot start with a number") + self.dcx() + .struct_err("lifetimes cannot start with a number") .with_span(span) .stash(span, StashKey::LifetimeIsChar); } @@ -341,16 +341,19 @@ impl<'psess, 'src> StringReader<'psess, 'src> { // first remove compound tokens like `<<` from `rustc_lexer`, and then add // fancier error recovery to it, as there will be less overall work to do this // way. - let (token, sugg) = unicode_chars::check_for_substitution(self, start, c, repeats+1); + let (token, sugg) = + unicode_chars::check_for_substitution(self, start, c, repeats + 1); self.dcx().emit_err(errors::UnknownTokenStart { span: self.mk_sp(start, self.pos + Pos::from_usize(repeats * c.len_utf8())), escaped: escaped_char(c), sugg, - null: if c == '\x00' {Some(errors::UnknownTokenNull)} else {None}, + null: if c == '\x00' { Some(errors::UnknownTokenNull) } else { None }, repeat: if repeats > 0 { swallow_next_invalid = repeats; Some(errors::UnknownTokenRepeat { repeats }) - } else {None} + } else { + None + }, }); if let Some(token) = token { -- cgit 1.4.1-3-g733a5 From 97910580aadab067ef13b5d0094a57e124c743ea Mon Sep 17 00:00:00 2001 From: Michael Goulet Date: Thu, 5 Sep 2024 05:43:55 -0400 Subject: Add initial support for raw lifetimes --- compiler/rustc_ast/src/mut_visit.rs | 4 +-- compiler/rustc_ast/src/token.rs | 18 ++++++------ compiler/rustc_ast/src/tokenstream.rs | 4 +-- compiler/rustc_ast_pretty/src/pprust/state.rs | 13 +++++++-- compiler/rustc_expand/src/mbe/macro_parser.rs | 6 ++-- compiler/rustc_expand/src/mbe/transcribe.rs | 4 +-- compiler/rustc_expand/src/proc_macro_server.rs | 9 +++--- compiler/rustc_lexer/src/lib.rs | 13 ++++++++- compiler/rustc_lint/messages.ftl | 4 +++ compiler/rustc_lint/src/context/diagnostics.rs | 4 +++ compiler/rustc_lint/src/lints.rs | 9 ++++++ compiler/rustc_lint_defs/src/lib.rs | 2 ++ compiler/rustc_parse/src/lexer/mod.rs | 39 ++++++++++++++++++++++++-- compiler/rustc_parse/src/parser/expr.rs | 6 ++-- compiler/rustc_parse/src/parser/mod.rs | 2 +- compiler/rustc_parse/src/parser/nonterminal.rs | 6 ++-- compiler/rustc_parse/src/parser/pat.rs | 6 ++-- compiler/rustc_parse/src/parser/ty.rs | 7 +++-- 18 files changed, 116 insertions(+), 40 deletions(-) (limited to 'compiler/rustc_parse/src') diff --git a/compiler/rustc_ast/src/mut_visit.rs b/compiler/rustc_ast/src/mut_visit.rs index 8a66894a356..7c8af6c4a55 100644 --- a/compiler/rustc_ast/src/mut_visit.rs +++ b/compiler/rustc_ast/src/mut_visit.rs @@ -752,7 +752,7 @@ fn visit_lazy_tts(vis: &mut T, lazy_tts: &mut Option(vis: &mut T, t: &mut Token) { let Token { kind, span } = t; match kind { - token::Ident(name, _ /*raw*/) | token::Lifetime(name) => { + token::Ident(name, _is_raw) | token::Lifetime(name, _is_raw) => { let mut ident = Ident::new(*name, *span); vis.visit_ident(&mut ident); *name = ident.name; @@ -762,7 +762,7 @@ pub fn visit_token(vis: &mut T, t: &mut Token) { token::NtIdent(ident, _is_raw) => { vis.visit_ident(ident); } - token::NtLifetime(ident) => { + token::NtLifetime(ident, _is_raw) => { vis.visit_ident(ident); } token::Interpolated(nt) => { diff --git a/compiler/rustc_ast/src/token.rs b/compiler/rustc_ast/src/token.rs index f1dddb3acac..a0082a41713 100644 --- a/compiler/rustc_ast/src/token.rs +++ b/compiler/rustc_ast/src/token.rs @@ -331,11 +331,11 @@ pub enum TokenKind { /// Do not forget about `NtLifetime` when you want to match on lifetime identifiers. /// It's recommended to use `Token::(lifetime,uninterpolate,uninterpolated_span)` to /// treat regular and interpolated lifetime identifiers in the same way. - Lifetime(Symbol), + Lifetime(Symbol, IdentIsRaw), /// This identifier (and its span) is the lifetime passed to the /// declarative macro. The span in the surrounding `Token` is the span of /// the `lifetime` metavariable in the macro's RHS. - NtLifetime(Ident), + NtLifetime(Ident, IdentIsRaw), /// An embedded AST node, as produced by a macro. This only exists for /// historical reasons. We'd like to get rid of it, for multiple reasons. @@ -458,7 +458,7 @@ impl Token { /// if they keep spans or perform edition checks. pub fn uninterpolated_span(&self) -> Span { match self.kind { - NtIdent(ident, _) | NtLifetime(ident) => ident.span, + NtIdent(ident, _) | NtLifetime(ident, _) => ident.span, Interpolated(ref nt) => nt.use_span(), _ => self.span, } @@ -661,7 +661,9 @@ impl Token { pub fn uninterpolate(&self) -> Cow<'_, Token> { match self.kind { NtIdent(ident, is_raw) => Cow::Owned(Token::new(Ident(ident.name, is_raw), ident.span)), - NtLifetime(ident) => Cow::Owned(Token::new(Lifetime(ident.name), ident.span)), + NtLifetime(ident, is_raw) => { + Cow::Owned(Token::new(Lifetime(ident.name, is_raw), ident.span)) + } _ => Cow::Borrowed(self), } } @@ -679,11 +681,11 @@ impl Token { /// Returns a lifetime identifier if this token is a lifetime. #[inline] - pub fn lifetime(&self) -> Option { + pub fn lifetime(&self) -> Option<(Ident, IdentIsRaw)> { // We avoid using `Token::uninterpolate` here because it's slow. match self.kind { - Lifetime(name) => Some(Ident::new(name, self.span)), - NtLifetime(ident) => Some(ident), + Lifetime(name, is_raw) => Some((Ident::new(name, self.span), is_raw)), + NtLifetime(ident, is_raw) => Some((ident, is_raw)), _ => None, } } @@ -865,7 +867,7 @@ impl Token { _ => return None, }, SingleQuote => match joint.kind { - Ident(name, IdentIsRaw::No) => Lifetime(Symbol::intern(&format!("'{name}"))), + Ident(name, is_raw) => Lifetime(Symbol::intern(&format!("'{name}")), is_raw), _ => return None, }, diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index 057b4455dca..fc1dd2caf68 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -482,11 +482,11 @@ impl TokenStream { token::NtIdent(ident, is_raw) => { TokenTree::Token(Token::new(token::Ident(ident.name, is_raw), ident.span), spacing) } - token::NtLifetime(ident) => TokenTree::Delimited( + token::NtLifetime(ident, is_raw) => TokenTree::Delimited( DelimSpan::from_single(token.span), DelimSpacing::new(Spacing::JointHidden, spacing), Delimiter::Invisible, - TokenStream::token_alone(token::Lifetime(ident.name), ident.span), + TokenStream::token_alone(token::Lifetime(ident.name, is_raw), ident.span), ), token::Interpolated(ref nt) => TokenTree::Delimited( DelimSpan::from_single(token.span), diff --git a/compiler/rustc_ast_pretty/src/pprust/state.rs b/compiler/rustc_ast_pretty/src/pprust/state.rs index c7ff39d23ed..3b1449d9a91 100644 --- a/compiler/rustc_ast_pretty/src/pprust/state.rs +++ b/compiler/rustc_ast_pretty/src/pprust/state.rs @@ -11,7 +11,9 @@ use std::borrow::Cow; use ast::TraitBoundModifiers; use rustc_ast::attr::AttrIdGenerator; use rustc_ast::ptr::P; -use rustc_ast::token::{self, BinOpToken, CommentKind, Delimiter, Nonterminal, Token, TokenKind}; +use rustc_ast::token::{ + self, BinOpToken, CommentKind, Delimiter, IdentIsRaw, Nonterminal, Token, TokenKind, +}; use rustc_ast::tokenstream::{Spacing, TokenStream, TokenTree}; use rustc_ast::util::classify; use rustc_ast::util::comments::{Comment, CommentStyle}; @@ -947,8 +949,13 @@ pub trait PrintState<'a>: std::ops::Deref + std::ops::Dere token::NtIdent(ident, is_raw) => { IdentPrinter::for_ast_ident(ident, is_raw.into()).to_string().into() } - token::Lifetime(name) => name.to_string().into(), - token::NtLifetime(ident) => ident.name.to_string().into(), + + token::Lifetime(name, IdentIsRaw::No) + | token::NtLifetime(Ident { name, .. }, IdentIsRaw::No) => name.to_string().into(), + token::Lifetime(name, IdentIsRaw::Yes) + | token::NtLifetime(Ident { name, .. }, IdentIsRaw::Yes) => { + format!("'r#{}", &name.as_str()[1..]).into() + } /* Other */ token::DocComment(comment_kind, attr_style, data) => { diff --git a/compiler/rustc_expand/src/mbe/macro_parser.rs b/compiler/rustc_expand/src/mbe/macro_parser.rs index 9011d02da33..501a2417fcf 100644 --- a/compiler/rustc_expand/src/mbe/macro_parser.rs +++ b/compiler/rustc_expand/src/mbe/macro_parser.rs @@ -398,8 +398,10 @@ pub(crate) enum NamedMatch { fn token_name_eq(t1: &Token, t2: &Token) -> bool { if let (Some((ident1, is_raw1)), Some((ident2, is_raw2))) = (t1.ident(), t2.ident()) { ident1.name == ident2.name && is_raw1 == is_raw2 - } else if let (Some(ident1), Some(ident2)) = (t1.lifetime(), t2.lifetime()) { - ident1.name == ident2.name + } else if let (Some((ident1, is_raw1)), Some((ident2, is_raw2))) = + (t1.lifetime(), t2.lifetime()) + { + ident1.name == ident2.name && is_raw1 == is_raw2 } else { t1.kind == t2.kind } diff --git a/compiler/rustc_expand/src/mbe/transcribe.rs b/compiler/rustc_expand/src/mbe/transcribe.rs index b06910595bb..39489a8df1b 100644 --- a/compiler/rustc_expand/src/mbe/transcribe.rs +++ b/compiler/rustc_expand/src/mbe/transcribe.rs @@ -283,9 +283,9 @@ pub(super) fn transcribe<'a>( let kind = token::NtIdent(*ident, *is_raw); TokenTree::token_alone(kind, sp) } - MatchedSingle(ParseNtResult::Lifetime(ident)) => { + MatchedSingle(ParseNtResult::Lifetime(ident, is_raw)) => { marker.visit_span(&mut sp); - let kind = token::NtLifetime(*ident); + let kind = token::NtLifetime(*ident, *is_raw); TokenTree::token_alone(kind, sp) } MatchedSingle(ParseNtResult::Nt(nt)) => { diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index 4ff5da1a4bd..5798bcedc22 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -229,15 +229,16 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec { + Lifetime(name, is_raw) => { let ident = symbol::Ident::new(name, span).without_first_quote(); trees.extend([ TokenTree::Punct(Punct { ch: b'\'', joint: true, span }), - TokenTree::Ident(Ident { sym: ident.name, is_raw: false, span }), + TokenTree::Ident(Ident { sym: ident.name, is_raw: is_raw.into(), span }), ]); } - NtLifetime(ident) => { - let stream = TokenStream::token_alone(token::Lifetime(ident.name), ident.span); + NtLifetime(ident, is_raw) => { + let stream = + TokenStream::token_alone(token::Lifetime(ident.name, is_raw), ident.span); trees.push(TokenTree::Group(Group { delimiter: pm::Delimiter::None, stream: Some(stream), diff --git a/compiler/rustc_lexer/src/lib.rs b/compiler/rustc_lexer/src/lib.rs index 6561e416cfa..60aab668cba 100644 --- a/compiler/rustc_lexer/src/lib.rs +++ b/compiler/rustc_lexer/src/lib.rs @@ -97,6 +97,9 @@ pub enum TokenKind { /// and not the separator. UnknownPrefixLifetime, + /// `'r#lt`, which in edition < 2021 is split into several tokens: `'r # lt`. + RawLifetime, + /// Similar to the above, but *always* an error on every edition. This is used /// for emoji identifier recovery, as those are not meant to be ever accepted. InvalidPrefix, @@ -683,9 +686,17 @@ impl Cursor<'_> { return Literal { kind, suffix_start }; } + if self.first() == 'r' && self.second() == '#' && is_id_start(self.third()) { + // Eat "r" and `#`, and identifier start characters. + self.bump(); + self.bump(); + self.bump(); + self.eat_while(is_id_continue); + return RawLifetime; + } + // Either a lifetime or a character literal with // length greater than 1. - let starts_with_number = self.first().is_ascii_digit(); // Skip the literal contents. diff --git a/compiler/rustc_lint/messages.ftl b/compiler/rustc_lint/messages.ftl index 759320b9eb6..f7d5c27baef 100644 --- a/compiler/rustc_lint/messages.ftl +++ b/compiler/rustc_lint/messages.ftl @@ -707,6 +707,10 @@ lint_range_endpoint_out_of_range = range endpoint is out of range for `{$ty}` lint_range_use_inclusive_range = use an inclusive range instead +lint_raw_prefix = prefix `'r` is reserved + .label = reserved prefix + .suggestion = insert whitespace here to avoid this being parsed as a prefix in Rust 2021 + lint_reason_must_be_string_literal = reason must be a string literal lint_reason_must_come_last = reason in lint attribute must come last diff --git a/compiler/rustc_lint/src/context/diagnostics.rs b/compiler/rustc_lint/src/context/diagnostics.rs index fd43afa1743..fbff210d87d 100644 --- a/compiler/rustc_lint/src/context/diagnostics.rs +++ b/compiler/rustc_lint/src/context/diagnostics.rs @@ -172,6 +172,10 @@ pub(super) fn decorate_lint(sess: &Session, diagnostic: BuiltinLintDiag, diag: & } .decorate_lint(diag); } + BuiltinLintDiag::RawPrefix(label_span) => { + lints::RawPrefix { label: label_span, suggestion: label_span.shrink_to_hi() } + .decorate_lint(diag); + } BuiltinLintDiag::UnusedBuiltinAttribute { attr_name, macro_name, invoc_span } => { lints::UnusedBuiltinAttribute { invoc_span, attr_name, macro_name }.decorate_lint(diag); } diff --git a/compiler/rustc_lint/src/lints.rs b/compiler/rustc_lint/src/lints.rs index 9050f36acba..8b62e716929 100644 --- a/compiler/rustc_lint/src/lints.rs +++ b/compiler/rustc_lint/src/lints.rs @@ -2772,6 +2772,15 @@ pub(crate) struct ReservedPrefix { pub prefix: String, } +#[derive(LintDiagnostic)] +#[diag(lint_raw_prefix)] +pub(crate) struct RawPrefix { + #[label] + pub label: Span, + #[suggestion(code = " ", applicability = "machine-applicable")] + pub suggestion: Span, +} + #[derive(LintDiagnostic)] #[diag(lint_unused_builtin_attribute)] pub(crate) struct UnusedBuiltinAttribute { diff --git a/compiler/rustc_lint_defs/src/lib.rs b/compiler/rustc_lint_defs/src/lib.rs index 6ee33041623..2b2a67dc846 100644 --- a/compiler/rustc_lint_defs/src/lib.rs +++ b/compiler/rustc_lint_defs/src/lib.rs @@ -609,6 +609,8 @@ pub enum BuiltinLintDiag { LegacyDeriveHelpers(Span), OrPatternsBackCompat(Span, String), ReservedPrefix(Span, String), + /// `'r#` in edition < 2021. + RawPrefix(Span), TrailingMacro(bool, Ident), BreakWithLabelAndLoop(Span), UnicodeTextFlow(Span, String), diff --git a/compiler/rustc_parse/src/lexer/mod.rs b/compiler/rustc_parse/src/lexer/mod.rs index c0731cf31c3..a680e2a3bc2 100644 --- a/compiler/rustc_parse/src/lexer/mod.rs +++ b/compiler/rustc_parse/src/lexer/mod.rs @@ -13,7 +13,6 @@ use rustc_session::lint::builtin::{ }; use rustc_session::lint::BuiltinLintDiag; use rustc_session::parse::ParseSess; -use rustc_span::edition::Edition; use rustc_span::symbol::Symbol; use rustc_span::{BytePos, Pos, Span}; use tracing::debug; @@ -284,7 +283,41 @@ impl<'psess, 'src> StringReader<'psess, 'src> { .stash(span, StashKey::LifetimeIsChar); } let ident = Symbol::intern(lifetime_name); - token::Lifetime(ident) + token::Lifetime(ident, IdentIsRaw::No) + } + rustc_lexer::TokenKind::RawLifetime => { + self.last_lifetime = Some(self.mk_sp(start, start + BytePos(1))); + + let ident_start = start + BytePos(3); + let prefix_span = self.mk_sp(start, ident_start); + + if prefix_span.at_least_rust_2021() { + let lifetime_name_without_tick = self.str_from(ident_start); + // Put the `'` back onto the lifetime name. + let mut lifetime_name = String::with_capacity(lifetime_name_without_tick.len() + 1); + lifetime_name.push('\''); + lifetime_name += lifetime_name_without_tick; + let sym = Symbol::intern(&lifetime_name); + + token::Lifetime(sym, IdentIsRaw::Yes) + } else { + // Otherwise, this is just `'r`. Warn about it though. + self.psess.buffer_lint( + RUST_2021_PREFIXES_INCOMPATIBLE_SYNTAX, + prefix_span, + ast::CRATE_NODE_ID, + BuiltinLintDiag::RawPrefix(prefix_span), + ); + + // Reset the state so we just lex the `'r`. + let lt_start = start + BytePos(2); + self.pos = lt_start; + self.cursor = Cursor::new(&str_before[2 as usize..]); + + let lifetime_name = self.str_from(start); + let ident = Symbol::intern(lifetime_name); + token::Lifetime(ident, IdentIsRaw::No) + } } rustc_lexer::TokenKind::Semi => token::Semi, rustc_lexer::TokenKind::Comma => token::Comma, @@ -712,7 +745,7 @@ impl<'psess, 'src> StringReader<'psess, 'src> { let expn_data = prefix_span.ctxt().outer_expn_data(); - if expn_data.edition >= Edition::Edition2021 { + if expn_data.edition.at_least_rust_2021() { // In Rust 2021, this is a hard error. let sugg = if prefix == "rb" { Some(errors::UnknownPrefixSugg::UseBr(prefix_span)) diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs index 84684e808d9..ecc4cd96faf 100644 --- a/compiler/rustc_parse/src/parser/expr.rs +++ b/compiler/rustc_parse/src/parser/expr.rs @@ -2050,7 +2050,7 @@ impl<'a> Parser<'a> { }; // On an error path, eagerly consider a lifetime to be an unclosed character lit, if that // makes sense. - if let Some(ident) = self.token.lifetime() + if let Some((ident, IdentIsRaw::No)) = self.token.lifetime() && could_be_unclosed_char_literal(ident) { let lt = self.expect_lifetime(); @@ -2925,9 +2925,9 @@ impl<'a> Parser<'a> { } pub(crate) fn eat_label(&mut self) -> Option