diff options
Diffstat (limited to 'compiler/rustc_ast/src')
| -rw-r--r-- | compiler/rustc_ast/src/ast.rs | 66 | ||||
| -rw-r--r-- | compiler/rustc_ast/src/attr/mod.rs | 31 | ||||
| -rw-r--r-- | compiler/rustc_ast/src/lib.rs | 1 | ||||
| -rw-r--r-- | compiler/rustc_ast/src/mut_visit.rs | 40 | ||||
| -rw-r--r-- | compiler/rustc_ast/src/token.rs | 21 | ||||
| -rw-r--r-- | compiler/rustc_ast/src/tokenstream.rs | 11 | ||||
| -rw-r--r-- | compiler/rustc_ast/src/util/lev_distance.rs | 109 | ||||
| -rw-r--r-- | compiler/rustc_ast/src/util/lev_distance/tests.rs | 59 | ||||
| -rw-r--r-- | compiler/rustc_ast/src/visit.rs | 6 |
9 files changed, 123 insertions, 221 deletions
diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs index 328086af183..220bbed7e78 100644 --- a/compiler/rustc_ast/src/ast.rs +++ b/compiler/rustc_ast/src/ast.rs @@ -24,7 +24,7 @@ pub use UnsafeSource::*; use crate::ptr::P; use crate::token::{self, CommentKind, DelimToken}; -use crate::tokenstream::{DelimSpan, LazyTokenStream, TokenStream, TokenTree}; +use crate::tokenstream::{DelimSpan, LazyTokenStream, TokenStream}; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_data_structures::stack::ensure_sufficient_stack; @@ -39,7 +39,6 @@ use rustc_span::{Span, DUMMY_SP}; use std::cmp::Ordering; use std::convert::TryFrom; use std::fmt; -use std::iter; #[cfg(test)] mod tests; @@ -901,10 +900,39 @@ pub struct Stmt { pub id: NodeId, pub kind: StmtKind, pub span: Span, - pub tokens: Option<LazyTokenStream>, } impl Stmt { + pub fn tokens(&self) -> Option<&LazyTokenStream> { + match self.kind { + StmtKind::Local(ref local) => local.tokens.as_ref(), + StmtKind::Item(ref item) => item.tokens.as_ref(), + StmtKind::Expr(ref expr) | StmtKind::Semi(ref expr) => expr.tokens.as_ref(), + StmtKind::Empty => None, + StmtKind::MacCall(ref mac) => mac.tokens.as_ref(), + } + } + + pub fn tokens_mut(&mut self) -> Option<&mut LazyTokenStream> { + match self.kind { + StmtKind::Local(ref mut local) => local.tokens.as_mut(), + StmtKind::Item(ref mut item) => item.tokens.as_mut(), + StmtKind::Expr(ref mut expr) | StmtKind::Semi(ref mut expr) => expr.tokens.as_mut(), + StmtKind::Empty => None, + StmtKind::MacCall(ref mut mac) => mac.tokens.as_mut(), + } + } + + pub fn set_tokens(&mut self, tokens: Option<LazyTokenStream>) { + match self.kind { + StmtKind::Local(ref mut local) => local.tokens = tokens, + StmtKind::Item(ref mut item) => item.tokens = tokens, + StmtKind::Expr(ref mut expr) | StmtKind::Semi(ref mut expr) => expr.tokens = tokens, + StmtKind::Empty => {} + StmtKind::MacCall(ref mut mac) => mac.tokens = tokens, + } + } + pub fn has_trailing_semicolon(&self) -> bool { match &self.kind { StmtKind::Semi(_) => true, @@ -912,18 +940,25 @@ impl Stmt { _ => false, } } + + /// Converts a parsed `Stmt` to a `Stmt` with + /// a trailing semicolon. + /// + /// This only modifies the parsed AST struct, not the attached + /// `LazyTokenStream`. The parser is responsible for calling + /// `CreateTokenStream::add_trailing_semi` when there is actually + /// a semicolon in the tokenstream. pub fn add_trailing_semicolon(mut self) -> Self { self.kind = match self.kind { StmtKind::Expr(expr) => StmtKind::Semi(expr), StmtKind::MacCall(mac) => { - StmtKind::MacCall(mac.map(|MacCallStmt { mac, style: _, attrs }| MacCallStmt { - mac, - style: MacStmtStyle::Semicolon, - attrs, + StmtKind::MacCall(mac.map(|MacCallStmt { mac, style: _, attrs, tokens }| { + MacCallStmt { mac, style: MacStmtStyle::Semicolon, attrs, tokens } })) } kind => kind, }; + self } @@ -963,6 +998,7 @@ pub struct MacCallStmt { pub mac: MacCall, pub style: MacStmtStyle, pub attrs: AttrVec, + pub tokens: Option<LazyTokenStream>, } #[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug)] @@ -988,6 +1024,7 @@ pub struct Local { pub init: Option<P<Expr>>, pub span: Span, pub attrs: AttrVec, + pub tokens: Option<LazyTokenStream>, } /// An arm of a 'match'. @@ -1476,20 +1513,6 @@ impl MacArgs { } } - /// Tokens together with the delimiters or `=`. - /// Use of this method generally means that something suboptimal or hacky is happening. - pub fn outer_tokens(&self) -> TokenStream { - match *self { - MacArgs::Empty => TokenStream::default(), - MacArgs::Delimited(dspan, delim, ref tokens) => { - TokenTree::Delimited(dspan, delim.to_token(), tokens.clone()).into() - } - MacArgs::Eq(eq_span, ref tokens) => { - iter::once(TokenTree::token(token::Eq, eq_span)).chain(tokens.trees()).collect() - } - } - } - /// Whether a macro with these arguments needs a semicolon /// when used as a standalone item or statement. pub fn need_semicolon(&self) -> bool { @@ -1845,6 +1868,7 @@ impl UintTy { pub struct AssocTyConstraint { pub id: NodeId, pub ident: Ident, + pub gen_args: Option<GenericArgs>, pub kind: AssocTyConstraintKind, pub span: Span, } diff --git a/compiler/rustc_ast/src/attr/mod.rs b/compiler/rustc_ast/src/attr/mod.rs index 2ff65737444..19c7c479f04 100644 --- a/compiler/rustc_ast/src/attr/mod.rs +++ b/compiler/rustc_ast/src/attr/mod.rs @@ -115,6 +115,10 @@ impl NestedMetaItem { pub fn is_meta_item_list(&self) -> bool { self.meta_item_list().is_some() } + + pub fn name_value_literal_span(&self) -> Option<Span> { + self.meta_item()?.name_value_literal_span() + } } impl Attribute { @@ -175,6 +179,22 @@ impl Attribute { pub fn is_value_str(&self) -> bool { self.value_str().is_some() } + + /// This is used in case you want the value span instead of the whole attribute. Example: + /// + /// ```text + /// #[doc(alias = "foo")] + /// ``` + /// + /// In here, it'll return a span for `"foo"`. + pub fn name_value_literal_span(&self) -> Option<Span> { + match self.kind { + AttrKind::Normal(ref item, _) => { + item.meta(self.span).and_then(|meta| meta.name_value_literal_span()) + } + AttrKind::DocComment(..) => None, + } + } } impl MetaItem { @@ -227,6 +247,17 @@ impl MetaItem { pub fn is_value_str(&self) -> bool { self.value_str().is_some() } + + /// This is used in case you want the value span instead of the whole attribute. Example: + /// + /// ```text + /// #[doc(alias = "foo")] + /// ``` + /// + /// In here, it'll return a span for `"foo"`. + pub fn name_value_literal_span(&self) -> Option<Span> { + Some(self.name_value_literal()?.span) + } } impl AttrItem { diff --git a/compiler/rustc_ast/src/lib.rs b/compiler/rustc_ast/src/lib.rs index 6e47ff7d740..8a20dd79685 100644 --- a/compiler/rustc_ast/src/lib.rs +++ b/compiler/rustc_ast/src/lib.rs @@ -34,7 +34,6 @@ macro_rules! unwrap_or { pub mod util { pub mod classify; pub mod comments; - pub mod lev_distance; pub mod literal; pub mod parser; } diff --git a/compiler/rustc_ast/src/mut_visit.rs b/compiler/rustc_ast/src/mut_visit.rs index ddae0ab03e4..3889ede7f4c 100644 --- a/compiler/rustc_ast/src/mut_visit.rs +++ b/compiler/rustc_ast/src/mut_visit.rs @@ -371,20 +371,15 @@ pub fn visit_mac_args<T: MutVisitor>(args: &mut MacArgs, vis: &mut T) { // The value in `#[key = VALUE]` must be visited as an expression for backward // compatibility, so that macros can be expanded in that position. if !vis.token_visiting_enabled() { - if let Some(TokenTree::Token(token)) = tokens.trees_ref().next() { - if let token::Interpolated(..) = token.kind { - // ^^ Do not `make_mut` unless we have to. - match Lrc::make_mut(&mut tokens.0).get_mut(0) { - Some((TokenTree::Token(token), _spacing)) => match &mut token.kind { - token::Interpolated(nt) => match Lrc::make_mut(nt) { - token::NtExpr(expr) => vis.visit_expr(expr), - t => panic!("unexpected token in key-value attribute: {:?}", t), - }, - t => panic!("unexpected token in key-value attribute: {:?}", t), - }, + match Lrc::make_mut(&mut tokens.0).get_mut(0) { + Some((TokenTree::Token(token), _spacing)) => match &mut token.kind { + token::Interpolated(nt) => match Lrc::make_mut(nt) { + token::NtExpr(expr) => vis.visit_expr(expr), t => panic!("unexpected token in key-value attribute: {:?}", t), - } - } + }, + t => panic!("unexpected token in key-value attribute: {:?}", t), + }, + t => panic!("unexpected token in key-value attribute: {:?}", t), } } } @@ -441,11 +436,14 @@ pub fn noop_flat_map_arm<T: MutVisitor>(mut arm: Arm, vis: &mut T) -> SmallVec<[ } pub fn noop_visit_ty_constraint<T: MutVisitor>( - AssocTyConstraint { id, ident, kind, span }: &mut AssocTyConstraint, + AssocTyConstraint { id, ident, gen_args, kind, span }: &mut AssocTyConstraint, vis: &mut T, ) { vis.visit_id(id); vis.visit_ident(ident); + if let Some(ref mut gen_args) = gen_args { + vis.visit_generic_args(gen_args); + } match kind { AssocTyConstraintKind::Equality { ref mut ty } => { vis.visit_ty(ty); @@ -576,13 +574,14 @@ pub fn noop_visit_parenthesized_parameter_data<T: MutVisitor>( } pub fn noop_visit_local<T: MutVisitor>(local: &mut P<Local>, vis: &mut T) { - let Local { id, pat, ty, init, span, attrs } = local.deref_mut(); + let Local { id, pat, ty, init, span, attrs, tokens } = local.deref_mut(); vis.visit_id(id); vis.visit_pat(pat); visit_opt(ty, |ty| vis.visit_ty(ty)); visit_opt(init, |init| vis.visit_expr(init)); vis.visit_span(span); visit_thin_attrs(attrs, vis); + visit_lazy_tts(tokens, vis); } pub fn noop_visit_attribute<T: MutVisitor>(attr: &mut Attribute, vis: &mut T) { @@ -1325,16 +1324,12 @@ pub fn noop_filter_map_expr<T: MutVisitor>(mut e: P<Expr>, vis: &mut T) -> Optio } pub fn noop_flat_map_stmt<T: MutVisitor>( - Stmt { kind, mut span, mut id, mut tokens }: Stmt, + Stmt { kind, mut span, mut id }: Stmt, vis: &mut T, ) -> SmallVec<[Stmt; 1]> { vis.visit_id(&mut id); vis.visit_span(&mut span); - visit_lazy_tts(&mut tokens, vis); - noop_flat_map_stmt_kind(kind, vis) - .into_iter() - .map(|kind| Stmt { id, kind, span, tokens: tokens.clone() }) - .collect() + noop_flat_map_stmt_kind(kind, vis).into_iter().map(|kind| Stmt { id, kind, span }).collect() } pub fn noop_flat_map_stmt_kind<T: MutVisitor>( @@ -1351,9 +1346,10 @@ pub fn noop_flat_map_stmt_kind<T: MutVisitor>( StmtKind::Semi(expr) => vis.filter_map_expr(expr).into_iter().map(StmtKind::Semi).collect(), StmtKind::Empty => smallvec![StmtKind::Empty], StmtKind::MacCall(mut mac) => { - let MacCallStmt { mac: mac_, style: _, attrs } = mac.deref_mut(); + let MacCallStmt { mac: mac_, style: _, attrs, tokens } = mac.deref_mut(); vis.visit_mac_call(mac_); visit_thin_attrs(attrs, vis); + visit_lazy_tts(tokens, vis); smallvec![StmtKind::MacCall(mac)] } } diff --git a/compiler/rustc_ast/src/token.rs b/compiler/rustc_ast/src/token.rs index 2bba7e618c0..f583825fbb3 100644 --- a/compiler/rustc_ast/src/token.rs +++ b/compiler/rustc_ast/src/token.rs @@ -785,13 +785,20 @@ impl Nonterminal { /// See issue #73345 for more details. /// FIXME(#73933): Remove this eventually. pub fn pretty_printing_compatibility_hack(&self) -> bool { - if let NtItem(item) = self { - let name = item.ident.name; - if name == sym::ProceduralMasqueradeDummyType || name == sym::ProcMacroHack { - if let ast::ItemKind::Enum(enum_def, _) = &item.kind { - if let [variant] = &*enum_def.variants { - return variant.ident.name == sym::Input; - } + let item = match self { + NtItem(item) => item, + NtStmt(stmt) => match &stmt.kind { + ast::StmtKind::Item(item) => item, + _ => return false, + }, + _ => return false, + }; + + let name = item.ident.name; + if name == sym::ProceduralMasqueradeDummyType || name == sym::ProcMacroHack { + if let ast::ItemKind::Enum(enum_def, _) = &item.kind { + if let [variant] = &*enum_def.variants { + return variant.ident.name == sym::Input; } } } diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index fe67b905bf3..b2207f22816 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -121,10 +121,14 @@ where } pub trait CreateTokenStream: sync::Send + sync::Sync { + fn add_trailing_semi(&self) -> Box<dyn CreateTokenStream>; fn create_token_stream(&self) -> TokenStream; } impl CreateTokenStream for TokenStream { + fn add_trailing_semi(&self) -> Box<dyn CreateTokenStream> { + panic!("Cannot call `add_trailing_semi` on a `TokenStream`!"); + } fn create_token_stream(&self) -> TokenStream { self.clone() } @@ -141,6 +145,13 @@ impl LazyTokenStream { LazyTokenStream(Lrc::new(Box::new(inner))) } + /// Extends the captured stream by one token, + /// which must be a trailing semicolon. This + /// affects the `TokenStream` created by `make_tokenstream`. + pub fn add_trailing_semi(&self) -> LazyTokenStream { + LazyTokenStream(Lrc::new(self.0.add_trailing_semi())) + } + pub fn create_token_stream(&self) -> TokenStream { self.0.create_token_stream() } diff --git a/compiler/rustc_ast/src/util/lev_distance.rs b/compiler/rustc_ast/src/util/lev_distance.rs deleted file mode 100644 index 21c2c925bc4..00000000000 --- a/compiler/rustc_ast/src/util/lev_distance.rs +++ /dev/null @@ -1,109 +0,0 @@ -// FIXME(Centril): Move to rustc_span? - -use rustc_span::symbol::Symbol; -use std::cmp; - -#[cfg(test)] -mod tests; - -/// Finds the Levenshtein distance between two strings -pub fn lev_distance(a: &str, b: &str) -> usize { - // cases which don't require further computation - if a.is_empty() { - return b.chars().count(); - } else if b.is_empty() { - return a.chars().count(); - } - - let mut dcol: Vec<_> = (0..=b.len()).collect(); - let mut t_last = 0; - - for (i, sc) in a.chars().enumerate() { - let mut current = i; - dcol[0] = current + 1; - - for (j, tc) in b.chars().enumerate() { - let next = dcol[j + 1]; - if sc == tc { - dcol[j + 1] = current; - } else { - dcol[j + 1] = cmp::min(current, next); - dcol[j + 1] = cmp::min(dcol[j + 1], dcol[j]) + 1; - } - current = next; - t_last = j; - } - } - dcol[t_last + 1] -} - -/// Finds the best match for a given word in the given iterator -/// -/// As a loose rule to avoid the obviously incorrect suggestions, it takes -/// an optional limit for the maximum allowable edit distance, which defaults -/// to one-third of the given word. -/// -/// Besides Levenshtein, we use case insensitive comparison to improve accuracy on an edge case with -/// a lower(upper)case letters mismatch. -pub fn find_best_match_for_name<'a, T>( - iter_names: T, - lookup: Symbol, - dist: Option<usize>, -) -> Option<Symbol> -where - T: Iterator<Item = &'a Symbol>, -{ - let lookup = &lookup.as_str(); - let max_dist = dist.unwrap_or_else(|| cmp::max(lookup.len(), 3) / 3); - let name_vec: Vec<&Symbol> = iter_names.collect(); - - let (case_insensitive_match, levenshtein_match) = name_vec - .iter() - .filter_map(|&name| { - let dist = lev_distance(lookup, &name.as_str()); - if dist <= max_dist { Some((name, dist)) } else { None } - }) - // Here we are collecting the next structure: - // (case_insensitive_match, (levenshtein_match, levenshtein_distance)) - .fold((None, None), |result, (candidate, dist)| { - ( - if candidate.as_str().to_uppercase() == lookup.to_uppercase() { - Some(candidate) - } else { - result.0 - }, - match result.1 { - None => Some((candidate, dist)), - Some((c, d)) => Some(if dist < d { (candidate, dist) } else { (c, d) }), - }, - ) - }); - // Priority of matches: - // 1. Exact case insensitive match - // 2. Levenshtein distance match - // 3. Sorted word match - if let Some(candidate) = case_insensitive_match { - Some(*candidate) - } else if levenshtein_match.is_some() { - levenshtein_match.map(|(candidate, _)| *candidate) - } else { - find_match_by_sorted_words(name_vec, lookup) - } -} - -fn find_match_by_sorted_words<'a>(iter_names: Vec<&'a Symbol>, lookup: &str) -> Option<Symbol> { - iter_names.iter().fold(None, |result, candidate| { - if sort_by_words(&candidate.as_str()) == sort_by_words(lookup) { - Some(**candidate) - } else { - result - } - }) -} - -fn sort_by_words(name: &str) -> String { - let mut split_words: Vec<&str> = name.split('_').collect(); - // We are sorting primitive &strs and can use unstable sort here - split_words.sort_unstable(); - split_words.join("_") -} diff --git a/compiler/rustc_ast/src/util/lev_distance/tests.rs b/compiler/rustc_ast/src/util/lev_distance/tests.rs deleted file mode 100644 index 7ebedbcb76a..00000000000 --- a/compiler/rustc_ast/src/util/lev_distance/tests.rs +++ /dev/null @@ -1,59 +0,0 @@ -use super::*; - -#[test] -fn test_lev_distance() { - use std::char::{from_u32, MAX}; - // Test bytelength agnosticity - for c in (0..MAX as u32).filter_map(|i| from_u32(i)).map(|i| i.to_string()) { - assert_eq!(lev_distance(&c[..], &c[..]), 0); - } - - let a = "\nMäry häd ä little lämb\n\nLittle lämb\n"; - let b = "\nMary häd ä little lämb\n\nLittle lämb\n"; - let c = "Mary häd ä little lämb\n\nLittle lämb\n"; - assert_eq!(lev_distance(a, b), 1); - assert_eq!(lev_distance(b, a), 1); - assert_eq!(lev_distance(a, c), 2); - assert_eq!(lev_distance(c, a), 2); - assert_eq!(lev_distance(b, c), 1); - assert_eq!(lev_distance(c, b), 1); -} - -#[test] -fn test_find_best_match_for_name() { - use rustc_span::with_default_session_globals; - with_default_session_globals(|| { - let input = vec![Symbol::intern("aaab"), Symbol::intern("aaabc")]; - assert_eq!( - find_best_match_for_name(input.iter(), Symbol::intern("aaaa"), None), - Some(Symbol::intern("aaab")) - ); - - assert_eq!( - find_best_match_for_name(input.iter(), Symbol::intern("1111111111"), None), - None - ); - - let input = vec![Symbol::intern("aAAA")]; - assert_eq!( - find_best_match_for_name(input.iter(), Symbol::intern("AAAA"), None), - Some(Symbol::intern("aAAA")) - ); - - let input = vec![Symbol::intern("AAAA")]; - // Returns None because `lev_distance > max_dist / 3` - assert_eq!(find_best_match_for_name(input.iter(), Symbol::intern("aaaa"), None), None); - - let input = vec![Symbol::intern("AAAA")]; - assert_eq!( - find_best_match_for_name(input.iter(), Symbol::intern("aaaa"), Some(4)), - Some(Symbol::intern("AAAA")) - ); - - let input = vec![Symbol::intern("a_longer_variable_name")]; - assert_eq!( - find_best_match_for_name(input.iter(), Symbol::intern("a_variable_longer_name"), None), - Some(Symbol::intern("a_longer_variable_name")) - ); - }) -} diff --git a/compiler/rustc_ast/src/visit.rs b/compiler/rustc_ast/src/visit.rs index 560064182e1..a420bb56350 100644 --- a/compiler/rustc_ast/src/visit.rs +++ b/compiler/rustc_ast/src/visit.rs @@ -485,6 +485,9 @@ pub fn walk_assoc_ty_constraint<'a, V: Visitor<'a>>( constraint: &'a AssocTyConstraint, ) { visitor.visit_ident(constraint.ident); + if let Some(ref gen_args) = constraint.gen_args { + visitor.visit_generic_args(gen_args.span(), gen_args); + } match constraint.kind { AssocTyConstraintKind::Equality { ref ty } => { visitor.visit_ty(ty); @@ -686,7 +689,7 @@ pub fn walk_stmt<'a, V: Visitor<'a>>(visitor: &mut V, statement: &'a Stmt) { StmtKind::Expr(ref expr) | StmtKind::Semi(ref expr) => visitor.visit_expr(expr), StmtKind::Empty => {} StmtKind::MacCall(ref mac) => { - let MacCallStmt { ref mac, style: _, ref attrs } = **mac; + let MacCallStmt { ref mac, style: _, ref attrs, tokens: _ } = **mac; visitor.visit_mac_call(mac); for attr in attrs.iter() { visitor.visit_attribute(attr); @@ -903,7 +906,6 @@ pub fn walk_mac_args<'a, V: Visitor<'a>>(visitor: &mut V, args: &'a MacArgs) { token::NtExpr(expr) => visitor.visit_expr(expr), t => panic!("unexpected token in key-value attribute: {:?}", t), }, - token::Literal(..) | token::Ident(..) => {} t => panic!("unexpected token in key-value attribute: {:?}", t), }, t => panic!("unexpected token in key-value attribute: {:?}", t), |