From 1fe2c0324006165b0c39ece0ccd7509e19583054 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Fri, 30 Nov 2018 10:02:04 +1100 Subject: Remove `tokenstream::Delimited`. Because it's an extra type layer that doesn't really help; in a couple of places it actively gets in the way, and overall removing it makes the code nicer. It does, however, move `tokenstream::TokenTree` further away from the `TokenTree` in `quote.rs`. More importantly, this change reduces the size of `TokenStream` from 48 bytes to 40 bytes on x86-64, which is enough to slightly reduce instruction counts on numerous benchmarks, the best by 1.5%. Note that `open_tt` and `close_tt` have gone from being methods on `Delimited` to associated methods of `TokenTree`. --- src/libsyntax/tokenstream.rs | 98 +++++++++++++++++++------------------------- 1 file changed, 42 insertions(+), 56 deletions(-) (limited to 'src/libsyntax/tokenstream.rs') diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index 242299f1b1f..90191c54126 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -34,52 +34,6 @@ use util::RcVec; use std::borrow::Cow; use std::{fmt, iter, mem}; -/// A delimited sequence of token trees -#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug)] -pub struct Delimited { - /// The type of delimiter - pub delim: DelimToken, - /// The delimited sequence of token trees - pub tts: ThinTokenStream, -} - -impl Delimited { - /// Returns the opening delimiter as a token. - pub fn open_token(&self) -> token::Token { - token::OpenDelim(self.delim) - } - - /// Returns the closing delimiter as a token. - pub fn close_token(&self) -> token::Token { - token::CloseDelim(self.delim) - } - - /// Returns the opening delimiter as a token tree. - pub fn open_tt(&self, span: Span) -> TokenTree { - let open_span = if span.is_dummy() { - span - } else { - span.with_hi(span.lo() + BytePos(self.delim.len() as u32)) - }; - TokenTree::Token(open_span, self.open_token()) - } - - /// Returns the closing delimiter as a token tree. - pub fn close_tt(&self, span: Span) -> TokenTree { - let close_span = if span.is_dummy() { - span - } else { - span.with_lo(span.hi() - BytePos(self.delim.len() as u32)) - }; - TokenTree::Token(close_span, self.close_token()) - } - - /// Returns the token trees inside the delimiters. - pub fn stream(&self) -> TokenStream { - self.tts.clone().into() - } -} - /// When the main rust parser encounters a syntax-extension invocation, it /// parses the arguments to the invocation as a token-tree. This is a very /// loose structure, such that all sorts of different AST-fragments can @@ -97,7 +51,7 @@ pub enum TokenTree { /// A single token Token(Span, token::Token), /// A delimited sequence of token trees - Delimited(DelimSpan, Delimited), + Delimited(DelimSpan, DelimToken, ThinTokenStream), } impl TokenTree { @@ -116,9 +70,10 @@ impl TokenTree { pub fn eq_unspanned(&self, other: &TokenTree) -> bool { match (self, other) { (&TokenTree::Token(_, ref tk), &TokenTree::Token(_, ref tk2)) => tk == tk2, - (&TokenTree::Delimited(_, ref dl), &TokenTree::Delimited(_, ref dl2)) => { - dl.delim == dl2.delim && - dl.stream().eq_unspanned(&dl2.stream()) + (&TokenTree::Delimited(_, delim, ref tts), + &TokenTree::Delimited(_, delim2, ref tts2)) => { + delim == delim2 && + tts.stream().eq_unspanned(&tts2.stream()) } (_, _) => false, } @@ -134,9 +89,10 @@ impl TokenTree { (&TokenTree::Token(_, ref tk), &TokenTree::Token(_, ref tk2)) => { tk.probably_equal_for_proc_macro(tk2) } - (&TokenTree::Delimited(_, ref dl), &TokenTree::Delimited(_, ref dl2)) => { - dl.delim == dl2.delim && - dl.stream().probably_equal_for_proc_macro(&dl2.stream()) + (&TokenTree::Delimited(_, delim, ref tts), + &TokenTree::Delimited(_, delim2, ref tts2)) => { + delim == delim2 && + tts.stream().probably_equal_for_proc_macro(&tts2.stream()) } (_, _) => false, } @@ -146,7 +102,7 @@ impl TokenTree { pub fn span(&self) -> Span { match *self { TokenTree::Token(sp, _) => sp, - TokenTree::Delimited(sp, _) => sp.entire(), + TokenTree::Delimited(sp, ..) => sp.entire(), } } @@ -154,7 +110,7 @@ impl TokenTree { pub fn set_span(&mut self, span: Span) { match *self { TokenTree::Token(ref mut sp, _) => *sp = span, - TokenTree::Delimited(ref mut sp, _) => *sp = DelimSpan::from_single(span), + TokenTree::Delimited(ref mut sp, ..) => *sp = DelimSpan::from_single(span), } } @@ -169,6 +125,26 @@ impl TokenTree { pub fn joint(self) -> TokenStream { TokenStream { kind: TokenStreamKind::JointTree(self) } } + + /// Returns the opening delimiter as a token tree. + pub fn open_tt(span: Span, delim: DelimToken) -> TokenTree { + let open_span = if span.is_dummy() { + span + } else { + span.with_hi(span.lo() + BytePos(delim.len() as u32)) + }; + TokenTree::Token(open_span, token::OpenDelim(delim)) + } + + /// Returns the closing delimiter as a token tree. + pub fn close_tt(span: Span, delim: DelimToken) -> TokenTree { + let close_span = if span.is_dummy() { + span + } else { + span.with_lo(span.hi() - BytePos(delim.len() as u32)) + }; + TokenTree::Token(close_span, token::CloseDelim(delim)) + } } /// # Token Streams @@ -182,6 +158,10 @@ pub struct TokenStream { kind: TokenStreamKind, } +// `TokenStream` is used a lot. Make sure it doesn't unintentionally get bigger. +#[cfg(target_arch = "x86_64")] +static_assert!(MEM_SIZE_OF_TOKEN_STREAM: mem::size_of::() == 40); + impl TokenStream { /// Given a `TokenStream` with a `Stream` of only two arguments, return a new `TokenStream` /// separating the two arguments with a comma for diagnostic suggestions. @@ -198,7 +178,7 @@ impl TokenStream { continue; } (TokenStreamKind::Tree(TokenTree::Token(sp, _)), _) => *sp, - (TokenStreamKind::Tree(TokenTree::Delimited(sp, _)), _) => sp.entire(), + (TokenStreamKind::Tree(TokenTree::Delimited(sp, ..)), _) => sp.entire(), _ => continue, }; let sp = sp.shrink_to_hi(); @@ -678,6 +658,12 @@ impl Cursor { #[derive(Debug, Clone)] pub struct ThinTokenStream(Option>); +impl ThinTokenStream { + pub fn stream(&self) -> TokenStream { + self.clone().into() + } +} + impl From for ThinTokenStream { fn from(stream: TokenStream) -> ThinTokenStream { ThinTokenStream(match stream.kind { -- cgit 1.4.1-3-g733a5 From 67f8fb5960551fbe80cc49ace1103a9fe393f9f4 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 12 Dec 2018 09:01:39 +1100 Subject: Use `Lrc>` instead of `RcVec`. This shrinks: - ThinTokenStream: 16 to 8 bytes - TokenTree: 32 to 24 bytes - TokenStream: 40 to 32 bytes The only downside is that in a couple of places this requires using `to_vec()` (which allocates) instead of `sub_slice()`. But those places are rarely executed, so it doesn't hurt perf. Overall, this reduces instruction counts on numerous benchmarks by up to 3%. --- src/libsyntax/tokenstream.rs | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) (limited to 'src/libsyntax/tokenstream.rs') diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index 90191c54126..da7f5029d9d 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -28,8 +28,8 @@ use ext::tt::{macro_parser, quoted}; use parse::Directory; use parse::token::{self, DelimToken, Token}; use print::pprust; +use rustc_data_structures::sync::Lrc; use serialize::{Decoder, Decodable, Encoder, Encodable}; -use util::RcVec; use std::borrow::Cow; use std::{fmt, iter, mem}; @@ -160,7 +160,7 @@ pub struct TokenStream { // `TokenStream` is used a lot. Make sure it doesn't unintentionally get bigger. #[cfg(target_arch = "x86_64")] -static_assert!(MEM_SIZE_OF_TOKEN_STREAM: mem::size_of::() == 40); +static_assert!(MEM_SIZE_OF_TOKEN_STREAM: mem::size_of::() == 32); impl TokenStream { /// Given a `TokenStream` with a `Stream` of only two arguments, return a new `TokenStream` @@ -194,7 +194,7 @@ impl TokenStream { new_slice.extend_from_slice(parts.0); new_slice.push(comma); new_slice.extend_from_slice(parts.1); - let slice = RcVec::new(new_slice); + let slice = Lrc::new(new_slice); return Some((TokenStream { kind: TokenStreamKind::Stream(slice) }, sp)); } } @@ -207,7 +207,7 @@ enum TokenStreamKind { Empty, Tree(TokenTree), JointTree(TokenTree), - Stream(RcVec), + Stream(Lrc>), } impl From for TokenStream { @@ -246,7 +246,7 @@ impl Extend for TokenStream { vec.push(TokenStream { kind }); vec } - TokenStreamKind::Stream(rc_vec) => match RcVec::try_unwrap(rc_vec) { + TokenStreamKind::Stream(rc_vec) => match Lrc::try_unwrap(rc_vec) { Ok(mut vec) => { // Extend in place using the existing capacity if possible. // This is the fast path for libraries like `quote` that @@ -277,7 +277,7 @@ impl Extend for TokenStream { *self = match tts.len() { 0 => TokenStream::empty(), 1 => tts.pop().unwrap(), - _ => TokenStream::concat_rc_vec(RcVec::new_preserving_capacity(tts)), + _ => TokenStream::concat_rc_vec(Lrc::new(tts)), }; } } @@ -314,11 +314,11 @@ impl TokenStream { match streams.len() { 0 => TokenStream::empty(), 1 => streams.pop().unwrap(), - _ => TokenStream::concat_rc_vec(RcVec::new(streams)), + _ => TokenStream::concat_rc_vec(Lrc::new(streams)), } } - fn concat_rc_vec(streams: RcVec) -> TokenStream { + fn concat_rc_vec(streams: Lrc>) -> TokenStream { TokenStream { kind: TokenStreamKind::Stream(streams) } } @@ -483,7 +483,7 @@ impl TokenStreamBuilder { match len { 1 => {} 2 => self.0.push(streams[0].clone().into()), - _ => self.0.push(TokenStream::concat_rc_vec(streams.sub_slice(0 .. len - 1))), + _ => self.0.push(TokenStream::concat_rc_vec( Lrc::new(streams[0 .. len - 1].to_vec()))), } self.push_all_but_last_tree(&streams[len - 1]) } @@ -495,7 +495,7 @@ impl TokenStreamBuilder { match len { 1 => {} 2 => self.0.push(streams[1].clone().into()), - _ => self.0.push(TokenStream::concat_rc_vec(streams.sub_slice(1 .. len))), + _ => self.0.push(TokenStream::concat_rc_vec(Lrc::new(streams[1 .. len].to_vec()))), } self.push_all_but_first_tree(&streams[0]) } @@ -515,13 +515,13 @@ enum CursorKind { #[derive(Clone)] struct StreamCursor { - stream: RcVec, + stream: Lrc>, index: usize, - stack: Vec<(RcVec, usize)>, + stack: Vec<(Lrc>, usize)>, } impl StreamCursor { - fn new(stream: RcVec) -> Self { + fn new(stream: Lrc>) -> Self { StreamCursor { stream: stream, index: 0, stack: Vec::new() } } @@ -544,7 +544,7 @@ impl StreamCursor { } } - fn insert(&mut self, stream: RcVec) { + fn insert(&mut self, stream: Lrc>) { self.stack.push((mem::replace(&mut self.stream, stream), mem::replace(&mut self.index, 0))); } @@ -656,7 +656,7 @@ impl Cursor { /// `ThinTokenStream` is smaller, but needs to allocate to represent a single `TokenTree`. /// We must use `ThinTokenStream` in `TokenTree::Delimited` to avoid infinite size due to recursion. #[derive(Debug, Clone)] -pub struct ThinTokenStream(Option>); +pub struct ThinTokenStream(Option>>); impl ThinTokenStream { pub fn stream(&self) -> TokenStream { @@ -668,8 +668,8 @@ impl From for ThinTokenStream { fn from(stream: TokenStream) -> ThinTokenStream { ThinTokenStream(match stream.kind { TokenStreamKind::Empty => None, - TokenStreamKind::Tree(tree) => Some(RcVec::new(vec![tree.into()])), - TokenStreamKind::JointTree(tree) => Some(RcVec::new(vec![tree.joint()])), + TokenStreamKind::Tree(tree) => Some(Lrc::new(vec![tree.into()])), + TokenStreamKind::JointTree(tree) => Some(Lrc::new(vec![tree.joint()])), TokenStreamKind::Stream(stream) => Some(stream), }) } -- cgit 1.4.1-3-g733a5 From 3c9aef1f452d9b7063d908fc40012e39ae1d2d16 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 12 Dec 2018 09:40:22 +1100 Subject: Use `TokenStream::concat` more. It's a better choice in a few places. --- src/libsyntax/tokenstream.rs | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) (limited to 'src/libsyntax/tokenstream.rs') diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index da7f5029d9d..e433f8d8dfd 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -167,9 +167,9 @@ impl TokenStream { /// separating the two arguments with a comma for diagnostic suggestions. pub(crate) fn add_comma(&self) -> Option<(TokenStream, Span)> { // Used to suggest if a user writes `foo!(a b);` - if let TokenStreamKind::Stream(ref slice) = self.kind { + if let TokenStreamKind::Stream(ref stream) = self.kind { let mut suggestion = None; - let mut iter = slice.iter().enumerate().peekable(); + let mut iter = stream.iter().enumerate().peekable(); while let Some((pos, ts)) = iter.next() { if let Some((_, next)) = iter.peek() { let sp = match (&ts.kind, &next.kind) { @@ -189,13 +189,12 @@ impl TokenStream { } } if let Some((pos, comma, sp)) = suggestion { - let mut new_slice = vec![]; - let parts = slice.split_at(pos + 1); - new_slice.extend_from_slice(parts.0); - new_slice.push(comma); - new_slice.extend_from_slice(parts.1); - let slice = Lrc::new(new_slice); - return Some((TokenStream { kind: TokenStreamKind::Stream(slice) }, sp)); + let mut new_stream = vec![]; + let parts = stream.split_at(pos + 1); + new_stream.extend_from_slice(parts.0); + new_stream.push(comma); + new_stream.extend_from_slice(parts.1); + return Some((TokenStream::concat(new_stream), sp)); } } None @@ -273,12 +272,7 @@ impl Extend for TokenStream { // Build the resulting token stream. If it contains more than one token, // preserve capacity in the vector in anticipation of the caller // performing additional calls to extend. - let mut tts = builder.0; - *self = match tts.len() { - 0 => TokenStream::empty(), - 1 => tts.pop().unwrap(), - _ => TokenStream::concat_rc_vec(Lrc::new(tts)), - }; + *self = TokenStream::concat(builder.0); } } @@ -483,7 +477,7 @@ impl TokenStreamBuilder { match len { 1 => {} 2 => self.0.push(streams[0].clone().into()), - _ => self.0.push(TokenStream::concat_rc_vec( Lrc::new(streams[0 .. len - 1].to_vec()))), + _ => self.0.push(TokenStream::concat(streams[0 .. len - 1].to_vec())), } self.push_all_but_last_tree(&streams[len - 1]) } @@ -495,7 +489,7 @@ impl TokenStreamBuilder { match len { 1 => {} 2 => self.0.push(streams[1].clone().into()), - _ => self.0.push(TokenStream::concat_rc_vec(Lrc::new(streams[1 .. len].to_vec()))), + _ => self.0.push(TokenStream::concat(streams[1 .. len].to_vec())), } self.push_all_but_first_tree(&streams[0]) } -- cgit 1.4.1-3-g733a5 From 07c12fa89e297bf5940f903dea8186a0a33f9d82 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 12 Dec 2018 09:55:43 +1100 Subject: Merge `TokenStreamKind` into `TokenStream`. Because the distinction provides little value, and removing it cleans up the code quite a bit. --- src/libsyntax/tokenstream.rs | 137 ++++++++++++++++++++----------------------- 1 file changed, 65 insertions(+), 72 deletions(-) (limited to 'src/libsyntax/tokenstream.rs') diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index e433f8d8dfd..e31bde4082d 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -123,7 +123,7 @@ impl TokenTree { } pub fn joint(self) -> TokenStream { - TokenStream { kind: TokenStreamKind::JointTree(self) } + TokenStream::JointTree(self) } /// Returns the opening delimiter as a token tree. @@ -154,8 +154,11 @@ impl TokenTree { /// instead of a representation of the abstract syntax tree. /// Today's `TokenTree`s can still contain AST via `Token::Interpolated` for back-compat. #[derive(Clone, Debug)] -pub struct TokenStream { - kind: TokenStreamKind, +pub enum TokenStream { + Empty, + Tree(TokenTree), + JointTree(TokenTree), + Stream(Lrc>), } // `TokenStream` is used a lot. Make sure it doesn't unintentionally get bigger. @@ -167,24 +170,22 @@ impl TokenStream { /// separating the two arguments with a comma for diagnostic suggestions. pub(crate) fn add_comma(&self) -> Option<(TokenStream, Span)> { // Used to suggest if a user writes `foo!(a b);` - if let TokenStreamKind::Stream(ref stream) = self.kind { + if let TokenStream::Stream(ref stream) = self { let mut suggestion = None; let mut iter = stream.iter().enumerate().peekable(); while let Some((pos, ts)) = iter.next() { if let Some((_, next)) = iter.peek() { - let sp = match (&ts.kind, &next.kind) { - (TokenStreamKind::Tree(TokenTree::Token(_, token::Token::Comma)), _) | - (_, TokenStreamKind::Tree(TokenTree::Token(_, token::Token::Comma))) => { + let sp = match (&ts, &next) { + (TokenStream::Tree(TokenTree::Token(_, token::Token::Comma)), _) | + (_, TokenStream::Tree(TokenTree::Token(_, token::Token::Comma))) => { continue; } - (TokenStreamKind::Tree(TokenTree::Token(sp, _)), _) => *sp, - (TokenStreamKind::Tree(TokenTree::Delimited(sp, ..)), _) => sp.entire(), + (TokenStream::Tree(TokenTree::Token(sp, _)), _) => *sp, + (TokenStream::Tree(TokenTree::Delimited(sp, ..)), _) => sp.entire(), _ => continue, }; let sp = sp.shrink_to_hi(); - let comma = TokenStream { - kind: TokenStreamKind::Tree(TokenTree::Token(sp, token::Comma)), - }; + let comma = TokenStream::Tree(TokenTree::Token(sp, token::Comma)); suggestion = Some((pos, comma, sp)); } } @@ -201,17 +202,9 @@ impl TokenStream { } } -#[derive(Clone, Debug)] -enum TokenStreamKind { - Empty, - Tree(TokenTree), - JointTree(TokenTree), - Stream(Lrc>), -} - impl From for TokenStream { fn from(tt: TokenTree) -> TokenStream { - TokenStream { kind: TokenStreamKind::Tree(tt) } + TokenStream::Tree(tt) } } @@ -230,22 +223,22 @@ impl> iter::FromIterator for TokenStream { impl Extend for TokenStream { fn extend>(&mut self, iter: I) { let iter = iter.into_iter(); - let kind = mem::replace(&mut self.kind, TokenStreamKind::Empty); + let this = mem::replace(self, TokenStream::Empty); // Vector of token streams originally in self. - let tts: Vec = match kind { - TokenStreamKind::Empty => { + let tts: Vec = match this { + TokenStream::Empty => { let mut vec = Vec::new(); vec.reserve(iter.size_hint().0); vec } - TokenStreamKind::Tree(_) | TokenStreamKind::JointTree(_) => { + TokenStream::Tree(_) | TokenStream::JointTree(_) => { let mut vec = Vec::new(); vec.reserve(1 + iter.size_hint().0); - vec.push(TokenStream { kind }); + vec.push(this); vec } - TokenStreamKind::Stream(rc_vec) => match Lrc::try_unwrap(rc_vec) { + TokenStream::Stream(rc_vec) => match Lrc::try_unwrap(rc_vec) { Ok(mut vec) => { // Extend in place using the existing capacity if possible. // This is the fast path for libraries like `quote` that @@ -286,7 +279,7 @@ impl PartialEq for TokenStream { impl TokenStream { pub fn len(&self) -> usize { - if let TokenStreamKind::Stream(ref slice) = self.kind { + if let TokenStream::Stream(ref slice) = self { slice.len() } else { 0 @@ -294,12 +287,12 @@ impl TokenStream { } pub fn empty() -> TokenStream { - TokenStream { kind: TokenStreamKind::Empty } + TokenStream::Empty } pub fn is_empty(&self) -> bool { - match self.kind { - TokenStreamKind::Empty => true, + match self { + TokenStream::Empty => true, _ => false, } } @@ -313,7 +306,7 @@ impl TokenStream { } fn concat_rc_vec(streams: Lrc>) -> TokenStream { - TokenStream { kind: TokenStreamKind::Stream(streams) } + TokenStream::Stream(streams) } pub fn trees(&self) -> Cursor { @@ -377,9 +370,9 @@ impl TokenStream { /// Precondition: `self` consists of a single token tree. /// Returns true if the token tree is a joint operation w.r.t. `proc_macro::TokenNode`. pub fn as_tree(self) -> (TokenTree, bool /* joint? */) { - match self.kind { - TokenStreamKind::Tree(tree) => (tree, false), - TokenStreamKind::JointTree(tree) => (tree, true), + match self { + TokenStream::Tree(tree) => (tree, false), + TokenStream::JointTree(tree) => (tree, true), _ => unreachable!(), } } @@ -389,9 +382,9 @@ impl TokenStream { let mut result = Vec::new(); let mut i = 0; while let Some(stream) = trees.next_as_stream() { - result.push(match stream.kind { - TokenStreamKind::Tree(tree) => f(i, tree).into(), - TokenStreamKind::JointTree(tree) => f(i, tree).joint(), + result.push(match stream { + TokenStream::Tree(tree) => f(i, tree).into(), + TokenStream::JointTree(tree) => f(i, tree).joint(), _ => unreachable!() }); i += 1; @@ -403,9 +396,9 @@ impl TokenStream { let mut trees = self.into_trees(); let mut result = Vec::new(); while let Some(stream) = trees.next_as_stream() { - result.push(match stream.kind { - TokenStreamKind::Tree(tree) => f(tree).into(), - TokenStreamKind::JointTree(tree) => f(tree).joint(), + result.push(match stream { + TokenStream::Tree(tree) => f(tree).into(), + TokenStream::JointTree(tree) => f(tree).joint(), _ => unreachable!() }); } @@ -413,19 +406,19 @@ impl TokenStream { } fn first_tree_and_joint(&self) -> Option<(TokenTree, bool)> { - match self.kind { - TokenStreamKind::Empty => None, - TokenStreamKind::Tree(ref tree) => Some((tree.clone(), false)), - TokenStreamKind::JointTree(ref tree) => Some((tree.clone(), true)), - TokenStreamKind::Stream(ref stream) => stream.first().unwrap().first_tree_and_joint(), + match self { + TokenStream::Empty => None, + TokenStream::Tree(ref tree) => Some((tree.clone(), false)), + TokenStream::JointTree(ref tree) => Some((tree.clone(), true)), + TokenStream::Stream(ref stream) => stream.first().unwrap().first_tree_and_joint(), } } fn last_tree_if_joint(&self) -> Option { - match self.kind { - TokenStreamKind::Empty | TokenStreamKind::Tree(..) => None, - TokenStreamKind::JointTree(ref tree) => Some(tree.clone()), - TokenStreamKind::Stream(ref stream) => stream.last().unwrap().last_tree_if_joint(), + match self { + TokenStream::Empty | TokenStream::Tree(..) => None, + TokenStream::JointTree(ref tree) => Some(tree.clone()), + TokenStream::Stream(ref stream) => stream.last().unwrap().last_tree_if_joint(), } } } @@ -472,7 +465,7 @@ impl TokenStreamBuilder { } fn push_all_but_last_tree(&mut self, stream: &TokenStream) { - if let TokenStreamKind::Stream(ref streams) = stream.kind { + if let TokenStream::Stream(ref streams) = stream { let len = streams.len(); match len { 1 => {} @@ -484,7 +477,7 @@ impl TokenStreamBuilder { } fn push_all_but_first_tree(&mut self, stream: &TokenStream) { - if let TokenStreamKind::Stream(ref streams) = stream.kind { + if let TokenStream::Stream(ref streams) = stream { let len = streams.len(); match len { 1 => {} @@ -524,10 +517,10 @@ impl StreamCursor { if self.index < self.stream.len() { self.index += 1; let next = self.stream[self.index - 1].clone(); - match next.kind { - TokenStreamKind::Tree(..) | TokenStreamKind::JointTree(..) => return Some(next), - TokenStreamKind::Stream(stream) => self.insert(stream), - TokenStreamKind::Empty => {} + match next { + TokenStream::Tree(..) | TokenStream::JointTree(..) => return Some(next), + TokenStream::Stream(stream) => self.insert(stream), + TokenStream::Empty => {} } } else if let Some((stream, index)) = self.stack.pop() { self.stream = stream; @@ -548,8 +541,8 @@ impl Iterator for Cursor { type Item = TokenTree; fn next(&mut self) -> Option { - self.next_as_stream().map(|stream| match stream.kind { - TokenStreamKind::Tree(tree) | TokenStreamKind::JointTree(tree) => tree, + self.next_as_stream().map(|stream| match stream { + TokenStream::Tree(tree) | TokenStream::JointTree(tree) => tree, _ => unreachable!() }) } @@ -557,11 +550,11 @@ impl Iterator for Cursor { impl Cursor { fn new(stream: TokenStream) -> Self { - Cursor(match stream.kind { - TokenStreamKind::Empty => CursorKind::Empty, - TokenStreamKind::Tree(tree) => CursorKind::Tree(tree, false), - TokenStreamKind::JointTree(tree) => CursorKind::JointTree(tree, false), - TokenStreamKind::Stream(stream) => CursorKind::Stream(StreamCursor::new(stream)), + Cursor(match stream { + TokenStream::Empty => CursorKind::Empty, + TokenStream::Tree(tree) => CursorKind::Tree(tree, false), + TokenStream::JointTree(tree) => CursorKind::JointTree(tree, false), + TokenStream::Stream(stream) => CursorKind::Stream(StreamCursor::new(stream)), }) } @@ -610,11 +603,11 @@ impl Cursor { pub fn look_ahead(&self, n: usize) -> Option { fn look_ahead(streams: &[TokenStream], mut n: usize) -> Result { for stream in streams { - n = match stream.kind { - TokenStreamKind::Tree(ref tree) | TokenStreamKind::JointTree(ref tree) + n = match stream { + TokenStream::Tree(ref tree) | TokenStream::JointTree(ref tree) if n == 0 => return Ok(tree.clone()), - TokenStreamKind::Tree(..) | TokenStreamKind::JointTree(..) => n - 1, - TokenStreamKind::Stream(ref stream) => match look_ahead(stream, n) { + TokenStream::Tree(..) | TokenStream::JointTree(..) => n - 1, + TokenStream::Stream(ref stream) => match look_ahead(stream, n) { Ok(tree) => return Ok(tree), Err(n) => n, }, @@ -660,11 +653,11 @@ impl ThinTokenStream { impl From for ThinTokenStream { fn from(stream: TokenStream) -> ThinTokenStream { - ThinTokenStream(match stream.kind { - TokenStreamKind::Empty => None, - TokenStreamKind::Tree(tree) => Some(Lrc::new(vec![tree.into()])), - TokenStreamKind::JointTree(tree) => Some(Lrc::new(vec![tree.joint()])), - TokenStreamKind::Stream(stream) => Some(stream), + ThinTokenStream(match stream { + TokenStream::Empty => None, + TokenStream::Tree(tree) => Some(Lrc::new(vec![tree.into()])), + TokenStream::JointTree(tree) => Some(Lrc::new(vec![tree.joint()])), + TokenStream::Stream(stream) => Some(stream), }) } } -- cgit 1.4.1-3-g733a5 From e80c7ddb05ee584c12131a4713173a3eafc49f4a Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 12 Dec 2018 10:01:08 +1100 Subject: Rename `TokenStream::concat` and remove `TokenStream::concat_rc_vec`. `TokenStream::new` is a better name for the former, and the latter is now just equivalent to `TokenStream::Stream`. --- src/libsyntax/attr/mod.rs | 6 +++--- src/libsyntax/ext/quote.rs | 2 +- src/libsyntax/ext/tt/transcribe.rs | 4 ++-- src/libsyntax/parse/attr.rs | 2 +- src/libsyntax/parse/lexer/tokentrees.rs | 6 +++--- src/libsyntax/parse/mod.rs | 6 +++--- src/libsyntax/parse/parser.rs | 4 ++-- src/libsyntax/tokenstream.rs | 34 +++++++++++++++------------------ 8 files changed, 30 insertions(+), 34 deletions(-) (limited to 'src/libsyntax/tokenstream.rs') diff --git a/src/libsyntax/attr/mod.rs b/src/libsyntax/attr/mod.rs index 7723c15a266..73cbe49f43b 100644 --- a/src/libsyntax/attr/mod.rs +++ b/src/libsyntax/attr/mod.rs @@ -483,7 +483,7 @@ impl MetaItem { last_pos = segment.ident.span.hi(); } idents.push(self.node.tokens(self.span)); - TokenStream::concat(idents) + TokenStream::new(idents) } fn from_tokens(tokens: &mut iter::Peekable) -> Option @@ -539,7 +539,7 @@ impl MetaItemKind { match *self { MetaItemKind::Word => TokenStream::empty(), MetaItemKind::NameValue(ref lit) => { - TokenStream::concat(vec![TokenTree::Token(span, Token::Eq).into(), lit.tokens()]) + TokenStream::new(vec![TokenTree::Token(span, Token::Eq).into(), lit.tokens()]) } MetaItemKind::List(ref list) => { let mut tokens = Vec::new(); @@ -552,7 +552,7 @@ impl MetaItemKind { TokenTree::Delimited( DelimSpan::from_single(span), token::Paren, - TokenStream::concat(tokens).into(), + TokenStream::new(tokens).into(), ).into() } } diff --git a/src/libsyntax/ext/quote.rs b/src/libsyntax/ext/quote.rs index c3497a17806..5820b49ab62 100644 --- a/src/libsyntax/ext/quote.rs +++ b/src/libsyntax/ext/quote.rs @@ -247,7 +247,7 @@ pub mod rt { let delim_span = DelimSpan::from_single(self.span); r.push(TokenTree::Delimited( - delim_span, token::Bracket, TokenStream::concat(inner).into() + delim_span, token::Bracket, TokenStream::new(inner).into() )); r } diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/tt/transcribe.rs index a76779ffebd..a63abd40495 100644 --- a/src/libsyntax/ext/tt/transcribe.rs +++ b/src/libsyntax/ext/tt/transcribe.rs @@ -103,12 +103,12 @@ pub fn transcribe(cx: &ExtCtxt, } Frame::Delimited { forest, span, .. } => { if result_stack.is_empty() { - return TokenStream::concat(result); + return TokenStream::new(result); } let tree = TokenTree::Delimited( span, forest.delim, - TokenStream::concat(result).into(), + TokenStream::new(result).into(), ); result = result_stack.pop().unwrap(); result.push(tree.into()); diff --git a/src/libsyntax/parse/attr.rs b/src/libsyntax/parse/attr.rs index 4ff6048e821..1bd0656846b 100644 --- a/src/libsyntax/parse/attr.rs +++ b/src/libsyntax/parse/attr.rs @@ -170,7 +170,7 @@ impl<'a> Parser<'a> { token::CloseDelim(_) | token::Eof => self.unexpected()?, _ => self.parse_token_tree(), }; - TokenStream::concat(vec![eq.into(), tree.into()]) + TokenStream::new(vec![eq.into(), tree.into()]) } else { TokenStream::empty() }; diff --git a/src/libsyntax/parse/lexer/tokentrees.rs b/src/libsyntax/parse/lexer/tokentrees.rs index 86c87cf898d..0906c25cab3 100644 --- a/src/libsyntax/parse/lexer/tokentrees.rs +++ b/src/libsyntax/parse/lexer/tokentrees.rs @@ -22,7 +22,7 @@ impl<'a> StringReader<'a> { tts.push(self.parse_token_tree()?); } - Ok(TokenStream::concat(tts)) + Ok(TokenStream::new(tts)) } // Parse a stream of tokens into a list of `TokenTree`s, up to a `CloseDelim`. @@ -30,14 +30,14 @@ impl<'a> StringReader<'a> { let mut tts = vec![]; loop { if let token::CloseDelim(..) = self.token { - return TokenStream::concat(tts); + return TokenStream::new(tts); } match self.parse_token_tree() { Ok(tree) => tts.push(tree), Err(mut e) => { e.emit(); - return TokenStream::concat(tts); + return TokenStream::new(tts); } } } diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index eb71003d3d0..a1685d537c8 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -842,13 +842,13 @@ mod tests { with_globals(|| { let tts = string_to_stream("fn a (b : i32) { b; }".to_string()); - let expected = TokenStream::concat(vec![ + let expected = TokenStream::new(vec![ TokenTree::Token(sp(0, 2), token::Ident(Ident::from_str("fn"), false)).into(), TokenTree::Token(sp(3, 4), token::Ident(Ident::from_str("a"), false)).into(), TokenTree::Delimited( DelimSpan::from_pair(sp(5, 6), sp(13, 14)), token::DelimToken::Paren, - TokenStream::concat(vec![ + TokenStream::new(vec![ TokenTree::Token(sp(6, 7), token::Ident(Ident::from_str("b"), false)).into(), TokenTree::Token(sp(8, 9), token::Colon).into(), @@ -859,7 +859,7 @@ mod tests { TokenTree::Delimited( DelimSpan::from_pair(sp(15, 16), sp(20, 21)), token::DelimToken::Brace, - TokenStream::concat(vec![ + TokenStream::new(vec![ TokenTree::Token(sp(17, 18), token::Ident(Ident::from_str("b"), false)).into(), TokenTree::Token(sp(18, 19), token::Semi).into(), diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index ded6da9f3ad..4e209f58024 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -2928,7 +2928,7 @@ impl<'a> Parser<'a> { _ => result.push(self.parse_token_tree().into()), } } - TokenStream::concat(result) + TokenStream::new(result) } /// Parse a prefix-unary-operator expr @@ -4624,7 +4624,7 @@ impl<'a> Parser<'a> { self.unexpected()?; unreachable!() }; - TokenStream::concat(vec![ + TokenStream::new(vec![ args.into(), TokenTree::Token(token_lo.to(self.prev_span), token::FatArrow).into(), body.into(), diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index e31bde4082d..c11ef33f931 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -195,7 +195,7 @@ impl TokenStream { new_stream.extend_from_slice(parts.0); new_stream.push(comma); new_stream.extend_from_slice(parts.1); - return Some((TokenStream::concat(new_stream), sp)); + return Some((TokenStream::new(new_stream), sp)); } } None @@ -216,7 +216,7 @@ impl From for TokenStream { impl> iter::FromIterator for TokenStream { fn from_iter>(iter: I) -> Self { - TokenStream::concat(iter.into_iter().map(Into::into).collect::>()) + TokenStream::new(iter.into_iter().map(Into::into).collect::>()) } } @@ -265,7 +265,7 @@ impl Extend for TokenStream { // Build the resulting token stream. If it contains more than one token, // preserve capacity in the vector in anticipation of the caller // performing additional calls to extend. - *self = TokenStream::concat(builder.0); + *self = TokenStream::new(builder.0); } } @@ -297,18 +297,14 @@ impl TokenStream { } } - pub fn concat(mut streams: Vec) -> TokenStream { + pub fn new(mut streams: Vec) -> TokenStream { match streams.len() { 0 => TokenStream::empty(), 1 => streams.pop().unwrap(), - _ => TokenStream::concat_rc_vec(Lrc::new(streams)), + _ => TokenStream::Stream(Lrc::new(streams)), } } - fn concat_rc_vec(streams: Lrc>) -> TokenStream { - TokenStream::Stream(streams) - } - pub fn trees(&self) -> Cursor { self.clone().into_trees() } @@ -389,7 +385,7 @@ impl TokenStream { }); i += 1; } - TokenStream::concat(result) + TokenStream::new(result) } pub fn map TokenTree>(self, mut f: F) -> TokenStream { @@ -402,7 +398,7 @@ impl TokenStream { _ => unreachable!() }); } - TokenStream::concat(result) + TokenStream::new(result) } fn first_tree_and_joint(&self) -> Option<(TokenTree, bool)> { @@ -461,7 +457,7 @@ impl TokenStreamBuilder { } pub fn build(self) -> TokenStream { - TokenStream::concat(self.0) + TokenStream::new(self.0) } fn push_all_but_last_tree(&mut self, stream: &TokenStream) { @@ -470,7 +466,7 @@ impl TokenStreamBuilder { match len { 1 => {} 2 => self.0.push(streams[0].clone().into()), - _ => self.0.push(TokenStream::concat(streams[0 .. len - 1].to_vec())), + _ => self.0.push(TokenStream::new(streams[0 .. len - 1].to_vec())), } self.push_all_but_last_tree(&streams[len - 1]) } @@ -482,7 +478,7 @@ impl TokenStreamBuilder { match len { 1 => {} 2 => self.0.push(streams[1].clone().into()), - _ => self.0.push(TokenStream::concat(streams[1 .. len].to_vec())), + _ => self.0.push(TokenStream::new(streams[1 .. len].to_vec())), } self.push_all_but_first_tree(&streams[0]) } @@ -577,7 +573,7 @@ impl Cursor { _ if stream.is_empty() => return, CursorKind::Empty => *self = stream.trees(), CursorKind::Tree(_, consumed) | CursorKind::JointTree(_, consumed) => { - *self = TokenStream::concat(vec![self.original_stream(), stream]).trees(); + *self = TokenStream::new(vec![self.original_stream(), stream]).trees(); if consumed { self.next(); } @@ -593,10 +589,10 @@ impl Cursor { CursorKind::Empty => TokenStream::empty(), CursorKind::Tree(ref tree, _) => tree.clone().into(), CursorKind::JointTree(ref tree, _) => tree.clone().joint(), - CursorKind::Stream(ref cursor) => TokenStream::concat_rc_vec({ + CursorKind::Stream(ref cursor) => TokenStream::Stream( cursor.stack.get(0).cloned().map(|(stream, _)| stream) .unwrap_or_else(|| cursor.stream.clone()) - }), + ), } } @@ -664,7 +660,7 @@ impl From for ThinTokenStream { impl From for TokenStream { fn from(stream: ThinTokenStream) -> TokenStream { - stream.0.map(TokenStream::concat_rc_vec).unwrap_or_else(TokenStream::empty) + stream.0.map(TokenStream::Stream).unwrap_or_else(TokenStream::empty) } } @@ -763,7 +759,7 @@ mod tests { let test_res = string_to_ts("foo::bar::baz"); let test_fst = string_to_ts("foo::bar"); let test_snd = string_to_ts("::baz"); - let eq_res = TokenStream::concat(vec![test_fst, test_snd]); + let eq_res = TokenStream::new(vec![test_fst, test_snd]); assert_eq!(test_res.trees().count(), 5); assert_eq!(eq_res.trees().count(), 5); assert_eq!(test_res.eq_unspanned(&eq_res), true); -- cgit 1.4.1-3-g733a5 From f756257fb7a5a6ff9fedd83ef03ed09a34bfef13 Mon Sep 17 00:00:00 2001 From: Vadim Petrochenkov Date: Tue, 11 Dec 2018 02:26:31 +0300 Subject: Do not interpret mismatches from pretty-printed `$crate` as token stream invalidation --- src/libsyntax/parse/token.rs | 4 +- src/libsyntax/tokenstream.rs | 4 +- src/test/ui/proc-macro/dollar-crate.stderr | 10 +++- src/test/ui/proc-macro/dollar-crate.stdout | 92 ++++++++++++------------------ 4 files changed, 50 insertions(+), 60 deletions(-) (limited to 'src/libsyntax/tokenstream.rs') diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index ed746657459..badcc4ed876 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -633,7 +633,9 @@ impl Token { (&Shebang(a), &Shebang(b)) => a == b, (&Lifetime(a), &Lifetime(b)) => a.name == b.name, - (&Ident(a, b), &Ident(c, d)) => a.name == c.name && b == d, + (&Ident(a, b), &Ident(c, d)) => b == d && (a.name == c.name || + a.name == keywords::DollarCrate.name() || + c.name == keywords::DollarCrate.name()), (&Literal(ref a, b), &Literal(ref c, d)) => { b == d && a.probably_equal_for_proc_macro(c) diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index c11ef33f931..013ecd3d343 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -348,7 +348,9 @@ impl TokenStream { | TokenTree::Token(_, Token::Semi) // The pretty printer collapses whitespace arbitrarily and can // introduce whitespace from `NoDelim`. - | TokenTree::Token(_, Token::Whitespace) => false, + | TokenTree::Token(_, Token::Whitespace) + // The pretty printer can turn `$crate` into `::crate_name` + | TokenTree::Token(_, Token::ModSep) => false, _ => true } } diff --git a/src/test/ui/proc-macro/dollar-crate.stderr b/src/test/ui/proc-macro/dollar-crate.stderr index 171562a5aff..75d3e17802e 100644 --- a/src/test/ui/proc-macro/dollar-crate.stderr +++ b/src/test/ui/proc-macro/dollar-crate.stderr @@ -2,7 +2,10 @@ error[E0428]: the name `D` is defined multiple times --> $DIR/dollar-crate.rs:23:13 | LL | struct D($crate::S); //~ ERROR the name `D` is defined multiple times - | ^^^^^^^^^^^^^^^^^^^^ `D` redefined here + | ^^^^^^^^^^^^^^^^^^^^ + | | + | `D` redefined here + | previous definition of the type `D` here ... LL | local!(); | --------- in this macro invocation @@ -13,7 +16,10 @@ error[E0428]: the name `D` is defined multiple times --> $DIR/dollar-crate.rs:31:5 | LL | dollar_crate_external::external!(); //~ ERROR the name `D` is defined multiple times - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `D` redefined here + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | | + | `D` redefined here + | previous definition of the type `D` here | = note: `D` must be defined only once in the type namespace of this module = note: this error originates in a macro outside of the current crate (in Nightly builds, run with -Z external-macro-backtrace for more info) diff --git a/src/test/ui/proc-macro/dollar-crate.stdout b/src/test/ui/proc-macro/dollar-crate.stdout index 8e254854e32..c47b3603f41 100644 --- a/src/test/ui/proc-macro/dollar-crate.stdout +++ b/src/test/ui/proc-macro/dollar-crate.stdout @@ -42,80 +42,80 @@ ATTRIBUTE INPUT (PRETTY-PRINTED): struct A(crate::S); ATTRIBUTE INPUT: TokenStream [ Ident { ident: "struct", - span: #0 bytes(0..0) + span: #2 bytes(LO..HI) }, Ident { ident: "A", - span: #0 bytes(0..0) + span: #2 bytes(LO..HI) }, Group { delimiter: Parenthesis, stream: TokenStream [ Ident { - ident: "crate", - span: #0 bytes(0..0) + ident: "$crate", + span: #2 bytes(LO..HI) }, Punct { ch: ':', spacing: Joint, - span: #0 bytes(0..0) + span: #2 bytes(LO..HI) }, Punct { ch: ':', spacing: Alone, - span: #0 bytes(0..0) + span: #2 bytes(LO..HI) }, Ident { ident: "S", - span: #0 bytes(0..0) + span: #2 bytes(LO..HI) } ], - span: #0 bytes(0..0) + span: #2 bytes(LO..HI) }, Punct { ch: ';', spacing: Alone, - span: #0 bytes(0..0) + span: #2 bytes(LO..HI) } ] DERIVE INPUT (PRETTY-PRINTED): struct D(crate::S); DERIVE INPUT: TokenStream [ Ident { ident: "struct", - span: #0 bytes(0..0) + span: #2 bytes(LO..HI) }, Ident { ident: "D", - span: #0 bytes(0..0) + span: #2 bytes(LO..HI) }, Group { delimiter: Parenthesis, stream: TokenStream [ Ident { - ident: "crate", - span: #0 bytes(0..0) + ident: "$crate", + span: #2 bytes(LO..HI) }, Punct { ch: ':', spacing: Joint, - span: #0 bytes(0..0) + span: #2 bytes(LO..HI) }, Punct { ch: ':', spacing: Alone, - span: #0 bytes(0..0) + span: #2 bytes(LO..HI) }, Ident { ident: "S", - span: #0 bytes(0..0) + span: #2 bytes(LO..HI) } ], - span: #0 bytes(0..0) + span: #2 bytes(LO..HI) }, Punct { ch: ';', spacing: Alone, - span: #0 bytes(0..0) + span: #2 bytes(LO..HI) } ] PROC MACRO INPUT (PRETTY-PRINTED): struct M ( $crate :: S ) ; @@ -162,99 +162,79 @@ ATTRIBUTE INPUT (PRETTY-PRINTED): struct A(::dollar_crate_external::S); ATTRIBUTE INPUT: TokenStream [ Ident { ident: "struct", - span: #0 bytes(0..0) + span: #10 bytes(LO..HI) }, Ident { ident: "A", - span: #0 bytes(0..0) + span: #10 bytes(LO..HI) }, Group { delimiter: Parenthesis, stream: TokenStream [ - Punct { - ch: ':', - spacing: Joint, - span: #0 bytes(0..0) - }, - Punct { - ch: ':', - spacing: Alone, - span: #0 bytes(0..0) - }, Ident { - ident: "dollar_crate_external", - span: #0 bytes(0..0) + ident: "$crate", + span: #10 bytes(LO..HI) }, Punct { ch: ':', spacing: Joint, - span: #0 bytes(0..0) + span: #10 bytes(LO..HI) }, Punct { ch: ':', spacing: Alone, - span: #0 bytes(0..0) + span: #10 bytes(LO..HI) }, Ident { ident: "S", - span: #0 bytes(0..0) + span: #10 bytes(LO..HI) } ], - span: #0 bytes(0..0) + span: #10 bytes(LO..HI) }, Punct { ch: ';', spacing: Alone, - span: #0 bytes(0..0) + span: #10 bytes(LO..HI) } ] DERIVE INPUT (PRETTY-PRINTED): struct D(::dollar_crate_external::S); DERIVE INPUT: TokenStream [ Ident { ident: "struct", - span: #0 bytes(0..0) + span: #10 bytes(LO..HI) }, Ident { ident: "D", - span: #0 bytes(0..0) + span: #10 bytes(LO..HI) }, Group { delimiter: Parenthesis, stream: TokenStream [ - Punct { - ch: ':', - spacing: Joint, - span: #0 bytes(0..0) - }, - Punct { - ch: ':', - spacing: Alone, - span: #0 bytes(0..0) - }, Ident { - ident: "dollar_crate_external", - span: #0 bytes(0..0) + ident: "$crate", + span: #10 bytes(LO..HI) }, Punct { ch: ':', spacing: Joint, - span: #0 bytes(0..0) + span: #10 bytes(LO..HI) }, Punct { ch: ':', spacing: Alone, - span: #0 bytes(0..0) + span: #10 bytes(LO..HI) }, Ident { ident: "S", - span: #0 bytes(0..0) + span: #10 bytes(LO..HI) } ], - span: #0 bytes(0..0) + span: #10 bytes(LO..HI) }, Punct { ch: ';', spacing: Alone, - span: #0 bytes(0..0) + span: #10 bytes(LO..HI) } ] -- cgit 1.4.1-3-g733a5