From d131abefc3aa53fc6376593729e6931326270503 Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Tue, 8 Oct 2019 21:50:09 +0200 Subject: ast: don't use pprust in Debug --- src/libsyntax/ast.rs | 47 ++++++----------------------------------------- 1 file changed, 6 insertions(+), 41 deletions(-) (limited to 'src/libsyntax') diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs index 023952042e6..1c78672da1a 100644 --- a/src/libsyntax/ast.rs +++ b/src/libsyntax/ast.rs @@ -70,7 +70,7 @@ impl fmt::Display for Lifetime { /// along with a bunch of supporting information. /// /// E.g., `std::cmp::PartialEq`. -#[derive(Clone, RustcEncodable, RustcDecodable)] +#[derive(Clone, RustcEncodable, RustcDecodable, Debug)] pub struct Path { pub span: Span, /// The segments in the path: the things separated by `::`. @@ -86,12 +86,6 @@ impl PartialEq for Path { } } -impl fmt::Debug for Path { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "path({})", pprust::path_to_string(self)) - } -} - impl fmt::Display for Path { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", pprust::path_to_string(self)) @@ -507,19 +501,13 @@ pub struct Block { pub span: Span, } -#[derive(Clone, RustcEncodable, RustcDecodable)] +#[derive(Clone, RustcEncodable, RustcDecodable, Debug)] pub struct Pat { pub id: NodeId, pub kind: PatKind, pub span: Span, } -impl fmt::Debug for Pat { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "pat({}: {})", self.id, pprust::pat_to_string(self)) - } -} - impl Pat { /// Attempt reparsing the pattern as a type. /// This is intended for use by diagnostics. @@ -831,7 +819,7 @@ impl UnOp { } /// A statement -#[derive(Clone, RustcEncodable, RustcDecodable)] +#[derive(Clone, RustcEncodable, RustcDecodable, Debug)] pub struct Stmt { pub id: NodeId, pub kind: StmtKind, @@ -865,18 +853,7 @@ impl Stmt { } } -impl fmt::Debug for Stmt { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!( - f, - "stmt({}: {})", - self.id.to_string(), - pprust::stmt_to_string(self) - ) - } -} - -#[derive(Clone, RustcEncodable, RustcDecodable)] +#[derive(Clone, RustcEncodable, RustcDecodable, Debug)] pub enum StmtKind { /// A local (let) binding. Local(P), @@ -973,7 +950,7 @@ pub struct AnonConst { } /// An expression. -#[derive(Clone, RustcEncodable, RustcDecodable)] +#[derive(Clone, RustcEncodable, RustcDecodable, Debug)] pub struct Expr { pub id: NodeId, pub kind: ExprKind, @@ -1100,12 +1077,6 @@ impl Expr { } } -impl fmt::Debug for Expr { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "expr({}: {})", self.id, pprust::expr_to_string(self)) - } -} - /// Limit types of a range (inclusive or exclusive) #[derive(Copy, Clone, PartialEq, RustcEncodable, RustcDecodable, Debug)] pub enum RangeLimits { @@ -1660,19 +1631,13 @@ pub enum AssocTyConstraintKind { }, } -#[derive(Clone, RustcEncodable, RustcDecodable)] +#[derive(Clone, RustcEncodable, RustcDecodable, Debug)] pub struct Ty { pub id: NodeId, pub kind: TyKind, pub span: Span, } -impl fmt::Debug for Ty { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "type({})", pprust::ty_to_string(self)) - } -} - #[derive(Clone, RustcEncodable, RustcDecodable, Debug)] pub struct BareFnTy { pub unsafety: Unsafety, -- cgit 1.4.1-3-g733a5 From 742ec4b9bf7ae7d693da7fe75e5f974e0fafb9d0 Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Tue, 8 Oct 2019 22:17:46 +0200 Subject: ast: remove implicit pprust dependency via Display. Instead just use `pprust::path_to_string(..)` where needed. This has two benefits: a) The AST definition is now independent of printing it. (Therefore we get closer to extracting a data-crate.) b) Debugging should be easier as program flow is clearer. --- src/librustc/lint/levels.rs | 3 ++- src/librustc_lint/builtin.rs | 5 +++-- src/librustc_mir/dataflow/mod.rs | 6 ++--- src/librustc_passes/ast_validation.rs | 3 ++- src/librustc_resolve/build_reduced_graph.rs | 3 ++- src/librustc_resolve/lib.rs | 5 +++-- src/librustc_resolve/macros.rs | 10 ++++++--- src/librustdoc/html/render.rs | 3 ++- src/libsyntax/ast.rs | 7 ------ src/libsyntax/attr/builtin.rs | 34 +++++++++++++++++++++-------- src/libsyntax/ext/expand.rs | 13 +++++++---- src/libsyntax/parse/diagnostics.rs | 2 +- src/libsyntax/parse/parser.rs | 6 +++-- src/libsyntax/parse/parser/expr.rs | 7 ++++-- src/libsyntax_ext/proc_macro_harness.rs | 28 +++++++++++++++++------- 15 files changed, 88 insertions(+), 47 deletions(-) (limited to 'src/libsyntax') diff --git a/src/librustc/lint/levels.rs b/src/librustc/lint/levels.rs index 28afe9730a0..60b1b192d10 100644 --- a/src/librustc/lint/levels.rs +++ b/src/librustc/lint/levels.rs @@ -12,6 +12,7 @@ use rustc_data_structures::stable_hasher::{HashStable, ToStableHashKey, StableHa use syntax::ast; use syntax::attr; use syntax::feature_gate; +use syntax::print::pprust; use syntax::source_map::MultiSpan; use syntax::symbol::{Symbol, sym}; @@ -285,7 +286,7 @@ impl<'a> LintLevelsBuilder<'a> { tool_ident.span, E0710, "an unknown tool name found in scoped lint: `{}`", - meta_item.path + pprust::path_to_string(&meta_item.path), ); continue; } diff --git a/src/librustc_lint/builtin.rs b/src/librustc_lint/builtin.rs index d0a7eab071c..9a16d0a0715 100644 --- a/src/librustc_lint/builtin.rs +++ b/src/librustc_lint/builtin.rs @@ -45,7 +45,7 @@ use syntax::feature_gate::{Stability, deprecated_attributes}; use syntax_pos::{BytePos, Span}; use syntax::symbol::{Symbol, kw, sym}; use syntax::errors::{Applicability, DiagnosticBuilder}; -use syntax::print::pprust::expr_to_string; +use syntax::print::pprust::{self, expr_to_string}; use syntax::visit::FnKind; use rustc::hir::{self, GenericParamKind, PatKind}; @@ -701,7 +701,8 @@ impl EarlyLintPass for DeprecatedAttr { } } if attr.check_name(sym::no_start) || attr.check_name(sym::crate_id) { - let msg = format!("use of deprecated attribute `{}`: no longer used.", attr.path); + let path_str = pprust::path_to_string(&attr.path); + let msg = format!("use of deprecated attribute `{}`: no longer used.", path_str); lint_deprecated_attr(cx, attr, &msg, None); } } diff --git a/src/librustc_mir/dataflow/mod.rs b/src/librustc_mir/dataflow/mod.rs index 06999abdc8b..ad0f75d7725 100644 --- a/src/librustc_mir/dataflow/mod.rs +++ b/src/librustc_mir/dataflow/mod.rs @@ -1,4 +1,5 @@ use syntax::ast::{self, MetaItem}; +use syntax::print::pprust; use syntax::symbol::{Symbol, sym}; use rustc_index::bit_set::{BitSet, HybridBitSet}; @@ -159,9 +160,8 @@ where if let Some(s) = item.value_str() { return Some(s.to_string()) } else { - sess.span_err( - item.span, - &format!("{} attribute requires a path", item.path)); + let path = pprust::path_to_string(&item.path); + sess.span_err(item.span, &format!("{} attribute requires a path", path)); return None; } } diff --git a/src/librustc_passes/ast_validation.rs b/src/librustc_passes/ast_validation.rs index 0339b85ca55..43c4f720ad4 100644 --- a/src/librustc_passes/ast_validation.rs +++ b/src/librustc_passes/ast_validation.rs @@ -263,7 +263,8 @@ impl<'a> AstValidator<'a> { let mut err = self.err_handler().struct_span_err(poly.span, &format!("`?Trait` is not permitted in {}", where_)); if is_trait { - err.note(&format!("traits are `?{}` by default", poly.trait_ref.path)); + let path_str = pprust::path_to_string(&poly.trait_ref.path); + err.note(&format!("traits are `?{}` by default", path_str)); } err.emit(); } diff --git a/src/librustc_resolve/build_reduced_graph.rs b/src/librustc_resolve/build_reduced_graph.rs index 030f9b97eb8..2b8965aa5d4 100644 --- a/src/librustc_resolve/build_reduced_graph.rs +++ b/src/librustc_resolve/build_reduced_graph.rs @@ -37,6 +37,7 @@ use syntax::ext::expand::AstFragment; use syntax::ext::hygiene::ExpnId; use syntax::feature_gate::is_builtin_attr; use syntax::parse::token::{self, Token}; +use syntax::print::pprust; use syntax::{span_err, struct_span_err}; use syntax::source_map::{respan, Spanned}; use syntax::symbol::{kw, sym}; @@ -228,7 +229,7 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> { .span_suggestion( path.span, "try", - format!("crate::{}", path), + format!("crate::{}", pprust::path_to_string(&path)), Applicability::MaybeIncorrect, ) .emit(); diff --git a/src/librustc_resolve/lib.rs b/src/librustc_resolve/lib.rs index 6aa85820ba8..ff8f7fc7aa4 100644 --- a/src/librustc_resolve/lib.rs +++ b/src/librustc_resolve/lib.rs @@ -38,6 +38,7 @@ use rustc_metadata::cstore::CStore; use syntax::ext::hygiene::{ExpnId, Transparency, SyntaxContext}; use syntax::ast::{self, Name, NodeId, Ident, FloatTy, IntTy, UintTy}; use syntax::ext::base::{SyntaxExtension, MacroKind, SpecialDerives}; +use syntax::print::pprust; use syntax::symbol::{kw, sym}; use syntax::visit::{self, Visitor}; @@ -2011,13 +2012,13 @@ impl<'a> Resolver<'a> { let mut candidates = self.lookup_import_candidates(ident, TypeNS, is_mod); candidates.sort_by_cached_key(|c| { - (c.path.segments.len(), c.path.to_string()) + (c.path.segments.len(), pprust::path_to_string(&c.path)) }); if let Some(candidate) = candidates.get(0) { ( String::from("unresolved import"), Some(( - vec![(ident.span, candidate.path.to_string())], + vec![(ident.span, pprust::path_to_string(&candidate.path))], String::from("a similar path exists"), Applicability::MaybeIncorrect, )), diff --git a/src/librustc_resolve/macros.rs b/src/librustc_resolve/macros.rs index 2b87bba8386..49186088fa0 100644 --- a/src/librustc_resolve/macros.rs +++ b/src/librustc_resolve/macros.rs @@ -21,6 +21,7 @@ use syntax::ext::hygiene::{self, ExpnId, ExpnData, ExpnKind}; use syntax::ext::compile_declarative_macro; use syntax::feature_gate::{emit_feature_err, is_builtin_attr_name}; use syntax::feature_gate::GateIssue; +use syntax::print::pprust; use syntax::symbol::{Symbol, kw, sym}; use syntax_pos::{Span, DUMMY_SP}; @@ -324,7 +325,8 @@ impl<'a> Resolver<'a> { Ok(if ext.macro_kind() != kind { let expected = kind.descr_expected(); - let msg = format!("expected {}, found {} `{}`", expected, res.descr(), path); + let path_str = pprust::path_to_string(path); + let msg = format!("expected {}, found {} `{}`", expected, res.descr(), path_str); self.session.struct_span_err(path.span, &msg) .span_label(path.span, format!("not {} {}", kind.article(), expected)) .emit(); @@ -805,14 +807,16 @@ impl<'a> Resolver<'a> { } } if let Some(depr) = &stability.rustc_depr { - let (message, lint) = stability::rustc_deprecation_message(depr, &path.to_string()); + let path = pprust::path_to_string(path); + let (message, lint) = stability::rustc_deprecation_message(depr, &path); stability::early_report_deprecation( self.session, &message, depr.suggestion, lint, span ); } } if let Some(depr) = &ext.deprecation { - let (message, lint) = stability::deprecation_message(depr, &path.to_string()); + let path = pprust::path_to_string(&path); + let (message, lint) = stability::deprecation_message(depr, &path); stability::early_report_deprecation(self.session, &message, None, lint, span); } } diff --git a/src/librustdoc/html/render.rs b/src/librustdoc/html/render.rs index 1ff71a0024b..b726ad1e0d6 100644 --- a/src/librustdoc/html/render.rs +++ b/src/librustdoc/html/render.rs @@ -46,6 +46,7 @@ use serialize::json::{ToJson, Json, as_json}; use syntax::ast; use syntax::edition::Edition; use syntax::ext::base::MacroKind; +use syntax::print::pprust; use syntax::source_map::FileName; use syntax::feature_gate::UnstableFeatures; use syntax::symbol::{Symbol, sym}; @@ -2957,7 +2958,7 @@ fn item_enum(w: &mut Buffer, cx: &Context, it: &clean::Item, e: &clean::Enum) { } fn render_attribute(attr: &ast::MetaItem) -> Option { - let path = attr.path.to_string(); + let path = pprust::path_to_string(&attr.path); if attr.is_word() { Some(path) diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs index 1c78672da1a..79d91125ec4 100644 --- a/src/libsyntax/ast.rs +++ b/src/libsyntax/ast.rs @@ -7,7 +7,6 @@ pub use crate::util::parser::ExprPrecedence; use crate::ext::hygiene::ExpnId; use crate::parse::token::{self, DelimToken}; -use crate::print::pprust; use crate::ptr::P; use crate::source_map::{dummy_spanned, respan, Spanned}; use crate::symbol::{kw, sym, Symbol}; @@ -86,12 +85,6 @@ impl PartialEq for Path { } } -impl fmt::Display for Path { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", pprust::path_to_string(self)) - } -} - impl Path { // Convert a span and an identifier to the corresponding // one-segment path. diff --git a/src/libsyntax/attr/builtin.rs b/src/libsyntax/attr/builtin.rs index 2a8e6b2cc95..efc48256e7c 100644 --- a/src/libsyntax/attr/builtin.rs +++ b/src/libsyntax/attr/builtin.rs @@ -5,6 +5,7 @@ use crate::early_buffered_lints::BufferedEarlyLintId; use crate::ext::base::ExtCtxt; use crate::feature_gate::{Features, GatedCfg}; use crate::parse::ParseSess; +use crate::print::pprust; use errors::{Applicability, Handler}; use syntax_pos::hygiene::Transparency; @@ -243,7 +244,11 @@ fn find_stability_generic<'a, I>(sess: &ParseSess, let meta = meta.as_ref().unwrap(); let get = |meta: &MetaItem, item: &mut Option| { if item.is_some() { - handle_errors(sess, meta.span, AttrError::MultipleItem(meta.path.to_string())); + handle_errors( + sess, + meta.span, + AttrError::MultipleItem(pprust::path_to_string(&meta.path)), + ); return false } if let Some(v) = meta.value_str() { @@ -271,7 +276,10 @@ fn find_stability_generic<'a, I>(sess: &ParseSess, handle_errors( sess, mi.span, - AttrError::UnknownMetaItem(mi.path.to_string(), expected), + AttrError::UnknownMetaItem( + pprust::path_to_string(&mi.path), + expected, + ), ); continue 'outer } @@ -362,7 +370,7 @@ fn find_stability_generic<'a, I>(sess: &ParseSess, sess, meta.span(), AttrError::UnknownMetaItem( - mi.path.to_string(), + pprust::path_to_string(&mi.path), &["feature", "reason", "issue", "soft"] ), ); @@ -434,7 +442,8 @@ fn find_stability_generic<'a, I>(sess: &ParseSess, sess, meta.span(), AttrError::UnknownMetaItem( - mi.path.to_string(), &["since", "note"], + pprust::path_to_string(&mi.path), + &["since", "note"], ), ); continue 'outer @@ -597,8 +606,11 @@ pub fn eval_condition(cfg: &ast::MetaItem, sess: &ParseSess, eval: &mut F) !eval_condition(mis[0].meta_item().unwrap(), sess, eval) }, _ => { - span_err!(sess.span_diagnostic, cfg.span, E0537, - "invalid predicate `{}`", cfg.path); + span_err!( + sess.span_diagnostic, cfg.span, E0537, + "invalid predicate `{}`", + pprust::path_to_string(&cfg.path) + ); false } } @@ -653,7 +665,9 @@ fn find_deprecation_generic<'a, I>(sess: &ParseSess, let get = |meta: &MetaItem, item: &mut Option| { if item.is_some() { handle_errors( - sess, meta.span, AttrError::MultipleItem(meta.path.to_string()) + sess, + meta.span, + AttrError::MultipleItem(pprust::path_to_string(&meta.path)), ); return false } @@ -691,8 +705,10 @@ fn find_deprecation_generic<'a, I>(sess: &ParseSess, handle_errors( sess, meta.span(), - AttrError::UnknownMetaItem(mi.path.to_string(), - &["since", "note"]), + AttrError::UnknownMetaItem( + pprust::path_to_string(&mi.path), + &["since", "note"], + ), ); continue 'outer } diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index bbd8da2acef..1e44f3dd5e5 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -13,6 +13,7 @@ use crate::mut_visit::*; use crate::parse::{DirectoryOwnership, PResult, ParseSess}; use crate::parse::token; use crate::parse::parser::Parser; +use crate::print::pprust; use crate::ptr::P; use crate::symbol::{sym, Symbol}; use crate::tokenstream::{TokenStream, TokenTree}; @@ -388,7 +389,8 @@ impl<'a, 'b> MacroExpander<'a, 'b> { "`derive` may only be applied to structs, enums and unions"); if let ast::AttrStyle::Inner = attr.style { let trait_list = derives.iter() - .map(|t| t.to_string()).collect::>(); + .map(|t| pprust::path_to_string(t)) + .collect::>(); let suggestion = format!("#[derive({})]", trait_list.join(", ")); err.span_suggestion( span, "try an outer attribute", suggestion, @@ -587,8 +589,11 @@ impl<'a, 'b> MacroExpander<'a, 'b> { let result = if let Some(result) = fragment_kind.make_from(tok_result) { result } else { - let msg = format!("non-{kind} macro in {kind} position: {path}", - kind = fragment_kind.name(), path = mac.path); + let msg = format!( + "non-{kind} macro in {kind} position: {path}", + kind = fragment_kind.name(), + path = pprust::path_to_string(&mac.path), + ); self.cx.span_err(span, &msg); self.cx.trace_macros_diag(); fragment_kind.dummy(span) @@ -878,7 +883,7 @@ impl<'a> Parser<'a> { err.span_label(span, "caused by the macro expansion here"); let msg = format!( "the usage of `{}!` is likely invalid in {} context", - macro_path, + pprust::path_to_string(¯o_path), kind_name, ); err.note(&msg); diff --git a/src/libsyntax/parse/diagnostics.rs b/src/libsyntax/parse/diagnostics.rs index f376c19a66c..df744e403d8 100644 --- a/src/libsyntax/parse/diagnostics.rs +++ b/src/libsyntax/parse/diagnostics.rs @@ -854,7 +854,7 @@ impl<'a> Parser<'a> { // This is a best-effort recovery. path.span, "try", - format!("<{}>::{}", ty_str, path), + format!("<{}>::{}", ty_str, pprust::path_to_string(&path)), Applicability::MaybeIncorrect, ) .emit(); diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index 4a457f5a43c..a19380657fd 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -1454,12 +1454,14 @@ impl<'a> Parser<'a> { `pub(super)`: visible only in the current module's parent `pub(in path::to::module)`: visible only on the specified path"##; + let path_str = pprust::path_to_string(&path); + struct_span_err!(self.sess.span_diagnostic, path.span, E0704, "{}", msg) .help(suggestion) .span_suggestion( path.span, - &format!("make this visible only to module `{}` with `in`", path), - format!("in {}", path), + &format!("make this visible only to module `{}` with `in`", path_str), + format!("in {}", path_str), Applicability::MachineApplicable, ) .emit(); diff --git a/src/libsyntax/parse/parser/expr.rs b/src/libsyntax/parse/parser/expr.rs index b459782d237..660bf9b7d65 100644 --- a/src/libsyntax/parse/parser/expr.rs +++ b/src/libsyntax/parse/parser/expr.rs @@ -552,8 +552,11 @@ impl<'a> Parser<'a> { // Report non-fatal diagnostics, keep `x as usize` as an expression // in AST and continue parsing. - let msg = format!("`<` is interpreted as a start of generic \ - arguments for `{}`, not a {}", path, op_noun); + let msg = format!( + "`<` is interpreted as a start of generic arguments for `{}`, not a {}", + pprust::path_to_string(&path), + op_noun, + ); let span_after_type = parser_snapshot_after_type.token.span; let expr = mk_expr(self, P(Ty { span: path.span, diff --git a/src/libsyntax_ext/proc_macro_harness.rs b/src/libsyntax_ext/proc_macro_harness.rs index 9b53bcb841c..1d48e8231b4 100644 --- a/src/libsyntax_ext/proc_macro_harness.rs +++ b/src/libsyntax_ext/proc_macro_harness.rs @@ -7,6 +7,7 @@ use syntax::ext::base::ExtCtxt; use syntax::ext::expand::{AstFragment, ExpansionConfig}; use syntax::ext::proc_macro::is_proc_macro_attr; use syntax::parse::ParseSess; +use syntax::print::pprust; use syntax::ptr::P; use syntax::symbol::{kw, sym}; use syntax::visit::{self, Visitor}; @@ -248,13 +249,20 @@ impl<'a> Visitor<'a> for CollectProcMacros<'a> { for attr in &item.attrs { if is_proc_macro_attr(&attr) { if let Some(prev_attr) = found_attr { + let path_str = pprust::path_to_string(&attr.path); let msg = if attr.path.segments[0].ident.name == prev_attr.path.segments[0].ident.name { - format!("only one `#[{}]` attribute is allowed on any given function", - attr.path) + format!( + "only one `#[{}]` attribute is allowed on any given function", + path_str, + ) } else { - format!("`#[{}]` and `#[{}]` attributes cannot both be applied \ - to the same function", attr.path, prev_attr.path) + format!( + "`#[{}]` and `#[{}]` attributes cannot both be applied + to the same function", + path_str, + pprust::path_to_string(&prev_attr.path), + ) }; self.handler.struct_span_err(attr.span, &msg) @@ -280,8 +288,10 @@ impl<'a> Visitor<'a> for CollectProcMacros<'a> { }; if !is_fn { - let msg = format!("the `#[{}]` attribute may only be used on bare functions", - attr.path); + let msg = format!( + "the `#[{}]` attribute may only be used on bare functions", + pprust::path_to_string(&attr.path), + ); self.handler.span_err(attr.span, &msg); return; @@ -292,8 +302,10 @@ impl<'a> Visitor<'a> for CollectProcMacros<'a> { } if !self.is_proc_macro_crate { - let msg = format!("the `#[{}]` attribute is only usable with crates of the \ - `proc-macro` crate type", attr.path); + let msg = format!( + "the `#[{}]` attribute is only usable with crates of the `proc-macro` crate type", + pprust::path_to_string(&attr.path), + ); self.handler.span_err(attr.span, &msg); return; -- cgit 1.4.1-3-g733a5 From ab8105ee9703882534203237b3a6494842d65a75 Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Wed, 9 Oct 2019 01:45:40 +0200 Subject: tokenstream: don't depend on pprust --- src/libsyntax/ext/mbe/macro_rules.rs | 6 ++++-- src/libsyntax/ext/proc_macro_server.rs | 3 ++- src/libsyntax/tokenstream.rs | 9 +-------- 3 files changed, 7 insertions(+), 11 deletions(-) (limited to 'src/libsyntax') diff --git a/src/libsyntax/ext/mbe/macro_rules.rs b/src/libsyntax/ext/mbe/macro_rules.rs index aec4a683141..e4fc269d147 100644 --- a/src/libsyntax/ext/mbe/macro_rules.rs +++ b/src/libsyntax/ext/mbe/macro_rules.rs @@ -174,7 +174,8 @@ fn generic_extension<'cx>( rhses: &[mbe::TokenTree], ) -> Box { if cx.trace_macros() { - trace_macros_note(cx, sp, format!("expanding `{}! {{ {} }}`", name, arg)); + let msg = format!("expanding `{}! {{ {} }}`", name, pprust::tts_to_string(arg.clone())); + trace_macros_note(cx, sp, msg); } // Which arm's failure should we report? (the one furthest along) @@ -212,7 +213,8 @@ fn generic_extension<'cx>( } if cx.trace_macros() { - trace_macros_note(cx, sp, format!("to `{}`", tts)); + let msg = format!("to `{}`", pprust::tts_to_string(tts.clone())); + trace_macros_note(cx, sp, msg); } let directory = Directory { diff --git a/src/libsyntax/ext/proc_macro_server.rs b/src/libsyntax/ext/proc_macro_server.rs index 021ec46d987..3ed6f4114ae 100644 --- a/src/libsyntax/ext/proc_macro_server.rs +++ b/src/libsyntax/ext/proc_macro_server.rs @@ -2,6 +2,7 @@ use crate::ast; use crate::ext::base::ExtCtxt; use crate::parse::{self, token, ParseSess}; use crate::parse::lexer::comments; +use crate::print::pprust; use crate::tokenstream::{self, DelimSpan, IsJoint::*, TokenStream, TreeAndJoint}; use errors::Diagnostic; @@ -407,7 +408,7 @@ impl server::TokenStream for Rustc<'_> { ) } fn to_string(&mut self, stream: &Self::TokenStream) -> String { - stream.to_string() + pprust::tts_to_string(stream.clone()) } fn from_token_tree( &mut self, diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index bef12ed4fad..970bacdde13 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -14,7 +14,6 @@ //! ownership of the original. use crate::parse::token::{self, DelimToken, Token, TokenKind}; -use crate::print::pprust; use syntax_pos::{BytePos, Span, DUMMY_SP}; #[cfg(target_arch = "x86_64")] @@ -23,7 +22,7 @@ use rustc_data_structures::sync::Lrc; use rustc_serialize::{Decoder, Decodable, Encoder, Encodable}; use smallvec::{SmallVec, smallvec}; -use std::{fmt, iter, mem}; +use std::{iter, mem}; #[cfg(test)] mod tests; @@ -507,12 +506,6 @@ impl Cursor { } } -impl fmt::Display for TokenStream { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(&pprust::tts_to_string(self.clone())) - } -} - impl Encodable for TokenStream { fn encode(&self, encoder: &mut E) -> Result<(), E::Error> { self.trees().collect::>().encode(encoder) -- cgit 1.4.1-3-g733a5 From 4a0c487d1bfe483a9341a8e477475dbc4a72b3a6 Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Fri, 11 Oct 2019 10:54:26 +0200 Subject: syntax: consolidate function parsing in `item.rs` --- src/libsyntax/parse/parser.rs | 285 +-------------- src/libsyntax/parse/parser/item.rs | 690 ++++++++++++++++++++++++++----------- src/libsyntax/parse/parser/ty.rs | 3 +- 3 files changed, 491 insertions(+), 487 deletions(-) (limited to 'src/libsyntax') diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index 4a457f5a43c..920208c8fd9 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -9,20 +9,19 @@ mod path; pub use path::PathStyle; mod stmt; mod generics; +use super::diagnostics::Error; use crate::ast::{ - self, DUMMY_NODE_ID, AttrStyle, Attribute, BindingMode, CrateSugar, Ident, - IsAsync, MacDelimiter, Mutability, Param, StrStyle, SelfKind, TyKind, Visibility, - VisibilityKind, Unsafety, + self, DUMMY_NODE_ID, AttrStyle, Attribute, CrateSugar, Ident, + IsAsync, MacDelimiter, Mutability, StrStyle, Visibility, VisibilityKind, Unsafety, }; use crate::parse::{ParseSess, PResult, Directory, DirectoryOwnership, SeqSep, literal, token}; -use crate::parse::diagnostics::{Error, dummy_arg}; use crate::parse::lexer::UnmatchedBrace; use crate::parse::lexer::comments::{doc_comment_style, strip_doc_comment_decoration}; use crate::parse::token::{Token, TokenKind, DelimToken}; use crate::print::pprust; use crate::ptr::P; -use crate::source_map::{self, respan}; +use crate::source_map::respan; use crate::symbol::{kw, sym, Symbol}; use crate::tokenstream::{self, DelimSpan, TokenTree, TokenStream, TreeAndJoint}; use crate::ThinVec; @@ -56,17 +55,6 @@ crate enum BlockMode { Ignore, } -/// The parsing configuration used to parse a parameter list (see `parse_fn_params`). -struct ParamCfg { - /// Is `self` is allowed as the first parameter? - is_self_allowed: bool, - /// Is `...` allowed as the tail of the parameter list? - allow_c_variadic: bool, - /// `is_name_required` decides if, per-parameter, - /// the parameter must have a pattern or just a type. - is_name_required: fn(&token::Token) -> bool, -} - /// Like `maybe_whole_expr`, but for things other than expressions. #[macro_export] macro_rules! maybe_whole { @@ -1105,271 +1093,6 @@ impl<'a> Parser<'a> { res } - /// Parses the parameter list of a function, including the `(` and `)` delimiters. - fn parse_fn_params(&mut self, mut cfg: ParamCfg) -> PResult<'a, Vec> { - let sp = self.token.span; - let is_trait_item = cfg.is_self_allowed; - let mut c_variadic = false; - // Parse the arguments, starting out with `self` being possibly allowed... - let (params, _) = self.parse_paren_comma_seq(|p| { - let param = p.parse_param_general(&cfg, is_trait_item); - // ...now that we've parsed the first argument, `self` is no longer allowed. - cfg.is_self_allowed = false; - - match param { - Ok(param) => Ok( - if let TyKind::CVarArgs = param.ty.kind { - c_variadic = true; - if p.token != token::CloseDelim(token::Paren) { - p.span_err( - p.token.span, - "`...` must be the last argument of a C-variadic function", - ); - // FIXME(eddyb) this should probably still push `CVarArgs`. - // Maybe AST validation/HIR lowering should emit the above error? - None - } else { - Some(param) - } - } else { - Some(param) - } - ), - Err(mut e) => { - e.emit(); - let lo = p.prev_span; - // Skip every token until next possible arg or end. - p.eat_to_tokens(&[&token::Comma, &token::CloseDelim(token::Paren)]); - // Create a placeholder argument for proper arg count (issue #34264). - let span = lo.to(p.prev_span); - Ok(Some(dummy_arg(Ident::new(kw::Invalid, span)))) - } - } - })?; - - let mut params: Vec<_> = params.into_iter().filter_map(|x| x).collect(); - - // Replace duplicated recovered params with `_` pattern to avoid unecessary errors. - self.deduplicate_recovered_params_names(&mut params); - - if c_variadic && params.len() <= 1 { - self.span_err( - sp, - "C-variadic function must be declared with at least one named argument", - ); - } - - Ok(params) - } - - /// Skips unexpected attributes and doc comments in this position and emits an appropriate - /// error. - /// This version of parse param doesn't necessarily require identifier names. - fn parse_param_general(&mut self, cfg: &ParamCfg, is_trait_item: bool) -> PResult<'a, Param> { - let lo = self.token.span; - let attrs = self.parse_outer_attributes()?; - - // Possibly parse `self`. Recover if we parsed it and it wasn't allowed here. - if let Some(mut param) = self.parse_self_param()? { - param.attrs = attrs.into(); - return if cfg.is_self_allowed { - Ok(param) - } else { - self.recover_bad_self_param(param, is_trait_item) - }; - } - - let is_name_required = match self.token.kind { - token::DotDotDot => false, - _ => (cfg.is_name_required)(&self.token), - }; - let (pat, ty) = if is_name_required || self.is_named_param() { - debug!("parse_param_general parse_pat (is_name_required:{})", is_name_required); - - let pat = self.parse_fn_param_pat()?; - if let Err(mut err) = self.expect(&token::Colon) { - return if let Some(ident) = self.parameter_without_type( - &mut err, - pat, - is_name_required, - cfg.is_self_allowed, - is_trait_item, - ) { - err.emit(); - Ok(dummy_arg(ident)) - } else { - Err(err) - }; - } - - self.eat_incorrect_doc_comment_for_param_type(); - (pat, self.parse_ty_common(true, true, cfg.allow_c_variadic)?) - } else { - debug!("parse_param_general ident_to_pat"); - let parser_snapshot_before_ty = self.clone(); - self.eat_incorrect_doc_comment_for_param_type(); - let mut ty = self.parse_ty_common(true, true, cfg.allow_c_variadic); - if ty.is_ok() && self.token != token::Comma && - self.token != token::CloseDelim(token::Paren) { - // This wasn't actually a type, but a pattern looking like a type, - // so we are going to rollback and re-parse for recovery. - ty = self.unexpected(); - } - match ty { - Ok(ty) => { - let ident = Ident::new(kw::Invalid, self.prev_span); - let bm = BindingMode::ByValue(Mutability::Immutable); - let pat = self.mk_pat_ident(ty.span, bm, ident); - (pat, ty) - } - // If this is a C-variadic argument and we hit an error, return the error. - Err(err) if self.token == token::DotDotDot => return Err(err), - // Recover from attempting to parse the argument as a type without pattern. - Err(mut err) => { - err.cancel(); - mem::replace(self, parser_snapshot_before_ty); - self.recover_arg_parse()? - } - } - }; - - let span = lo.to(self.token.span); - - Ok(Param { - attrs: attrs.into(), - id: ast::DUMMY_NODE_ID, - is_placeholder: false, - pat, - span, - ty, - }) - } - - /// Returns the parsed optional self parameter and whether a self shortcut was used. - /// - /// See `parse_self_param_with_attrs` to collect attributes. - fn parse_self_param(&mut self) -> PResult<'a, Option> { - // Extract an identifier *after* having confirmed that the token is one. - let expect_self_ident = |this: &mut Self| { - match this.token.kind { - // Preserve hygienic context. - token::Ident(name, _) => { - let span = this.token.span; - this.bump(); - Ident::new(name, span) - } - _ => unreachable!(), - } - }; - // Is `self` `n` tokens ahead? - let is_isolated_self = |this: &Self, n| { - this.is_keyword_ahead(n, &[kw::SelfLower]) - && this.look_ahead(n + 1, |t| t != &token::ModSep) - }; - // Is `mut self` `n` tokens ahead? - let is_isolated_mut_self = |this: &Self, n| { - this.is_keyword_ahead(n, &[kw::Mut]) - && is_isolated_self(this, n + 1) - }; - // Parse `self` or `self: TYPE`. We already know the current token is `self`. - let parse_self_possibly_typed = |this: &mut Self, m| { - let eself_ident = expect_self_ident(this); - let eself_hi = this.prev_span; - let eself = if this.eat(&token::Colon) { - SelfKind::Explicit(this.parse_ty()?, m) - } else { - SelfKind::Value(m) - }; - Ok((eself, eself_ident, eself_hi)) - }; - // Recover for the grammar `*self`, `*const self`, and `*mut self`. - let recover_self_ptr = |this: &mut Self| { - let msg = "cannot pass `self` by raw pointer"; - let span = this.token.span; - this.struct_span_err(span, msg) - .span_label(span, msg) - .emit(); - - Ok((SelfKind::Value(Mutability::Immutable), expect_self_ident(this), this.prev_span)) - }; - - // Parse optional `self` parameter of a method. - // Only a limited set of initial token sequences is considered `self` parameters; anything - // else is parsed as a normal function parameter list, so some lookahead is required. - let eself_lo = self.token.span; - let (eself, eself_ident, eself_hi) = match self.token.kind { - token::BinOp(token::And) => { - let eself = if is_isolated_self(self, 1) { - // `&self` - self.bump(); - SelfKind::Region(None, Mutability::Immutable) - } else if is_isolated_mut_self(self, 1) { - // `&mut self` - self.bump(); - self.bump(); - SelfKind::Region(None, Mutability::Mutable) - } else if self.look_ahead(1, |t| t.is_lifetime()) && is_isolated_self(self, 2) { - // `&'lt self` - self.bump(); - let lt = self.expect_lifetime(); - SelfKind::Region(Some(lt), Mutability::Immutable) - } else if self.look_ahead(1, |t| t.is_lifetime()) && is_isolated_mut_self(self, 2) { - // `&'lt mut self` - self.bump(); - let lt = self.expect_lifetime(); - self.bump(); - SelfKind::Region(Some(lt), Mutability::Mutable) - } else { - // `¬_self` - return Ok(None); - }; - (eself, expect_self_ident(self), self.prev_span) - } - // `*self` - token::BinOp(token::Star) if is_isolated_self(self, 1) => { - self.bump(); - recover_self_ptr(self)? - } - // `*mut self` and `*const self` - token::BinOp(token::Star) if - self.look_ahead(1, |t| t.is_mutability()) - && is_isolated_self(self, 2) => - { - self.bump(); - self.bump(); - recover_self_ptr(self)? - } - // `self` and `self: TYPE` - token::Ident(..) if is_isolated_self(self, 0) => { - parse_self_possibly_typed(self, Mutability::Immutable)? - } - // `mut self` and `mut self: TYPE` - token::Ident(..) if is_isolated_mut_self(self, 0) => { - self.bump(); - parse_self_possibly_typed(self, Mutability::Mutable)? - } - _ => return Ok(None), - }; - - let eself = source_map::respan(eself_lo.to(eself_hi), eself); - Ok(Some(Param::from_self(ThinVec::default(), eself, eself_ident))) - } - - fn is_named_param(&self) -> bool { - let offset = match self.token.kind { - token::Interpolated(ref nt) => match **nt { - token::NtPat(..) => return self.look_ahead(1, |t| t == &token::Colon), - _ => 0, - } - token::BinOp(token::And) | token::AndAnd => 1, - _ if self.token.is_keyword(kw::Mut) => 1, - _ => 0, - }; - - self.look_ahead(offset, |t| t.is_ident()) && - self.look_ahead(offset + 1, |t| t == &token::Colon) - } - fn is_crate_vis(&self) -> bool { self.token.is_keyword(kw::Crate) && self.look_ahead(1, |t| t != &token::ModSep) } diff --git a/src/libsyntax/parse/parser/item.rs b/src/libsyntax/parse/parser/item.rs index 3c60c88e2aa..08c624b5539 100644 --- a/src/libsyntax/parse/parser/item.rs +++ b/src/libsyntax/parse/parser/item.rs @@ -1,28 +1,23 @@ -use super::{Parser, PResult, PathStyle, SemiColonMode, BlockMode, ParamCfg}; - +use super::{Parser, PResult, PathStyle, SemiColonMode, BlockMode}; use crate::maybe_whole; use crate::ptr::P; -use crate::ast::{ - self, DUMMY_NODE_ID, Ident, Attribute, AttrStyle, - Item, ItemKind, ImplItem, ImplItemKind, TraitItem, TraitItemKind, - UseTree, UseTreeKind, PathSegment, - IsAuto, Constness, IsAsync, Unsafety, Defaultness, - Visibility, VisibilityKind, Mutability, FnDecl, FnHeader, MethodSig, Block, - ForeignItem, ForeignItemKind, - Ty, TyKind, Generics, GenericBounds, TraitRef, - EnumDef, VariantData, StructField, AnonConst, - Mac, MacDelimiter, -}; +use crate::ast::{self, DUMMY_NODE_ID, Ident, Attribute, AttrStyle, AnonConst, Item, ItemKind}; +use crate::ast::{ImplItem, ImplItemKind, TraitItem, TraitItemKind, UseTree, UseTreeKind}; +use crate::ast::{PathSegment, IsAuto, Constness, IsAsync, Unsafety, Defaultness}; +use crate::ast::{Visibility, VisibilityKind, Mutability, FnHeader, ForeignItem, ForeignItemKind}; +use crate::ast::{Ty, TyKind, Generics, GenericBounds, TraitRef, EnumDef, VariantData, StructField}; +use crate::ast::{Mac, MacDelimiter, Block, BindingMode, FnDecl, MethodSig, SelfKind, Param}; use crate::ext::base::DummyResult; use crate::parse::token; use crate::parse::parser::maybe_append; -use crate::parse::diagnostics::Error; +use crate::parse::diagnostics::{Error, dummy_arg}; use crate::tokenstream::{TokenTree, TokenStream}; -use crate::source_map::{respan, Span}; use crate::symbol::{kw, sym}; +use crate::source_map::{self, respan, Span}; +use crate::ThinVec; -use std::mem; use log::debug; +use std::mem; use rustc_target::spec::abi::Abi; use errors::{Applicability, DiagnosticBuilder, DiagnosticId, StashKey}; @@ -412,7 +407,7 @@ impl<'a> Parser<'a> { self.parse_macro_use_or_failure(attrs, macros_allowed, attributes_allowed, lo, vis) } - fn mk_item_with_info( + pub(super) fn mk_item_with_info( &self, attrs: Vec, lo: Span, @@ -425,16 +420,6 @@ impl<'a> Parser<'a> { Ok(Some(self.mk_item(span, ident, item, vis, attrs))) } - fn recover_first_param(&mut self) -> &'static str { - match self.parse_outer_attributes() - .and_then(|_| self.parse_self_param()) - .map_err(|mut e| e.cancel()) - { - Ok(Some(_)) => "method", - _ => "function", - } - } - /// This is the fall-through for parsing items. fn parse_macro_use_or_failure( &mut self, @@ -707,9 +692,11 @@ impl<'a> Parser<'a> { Ok(item) } - fn parse_impl_item_(&mut self, - at_end: &mut bool, - mut attrs: Vec) -> PResult<'a, ImplItem> { + fn parse_impl_item_( + &mut self, + at_end: &mut bool, + mut attrs: Vec, + ) -> PResult<'a, ImplItem> { let lo = self.token.span; let vis = self.parse_visibility(false)?; let defaultness = self.parse_defaultness(); @@ -722,8 +709,11 @@ impl<'a> Parser<'a> { (name, kind, generics) } else if self.is_const_item() { self.parse_impl_const()? + } else if let Some(mac) = self.parse_assoc_macro_invoc("impl", Some(&vis), at_end)? { + // FIXME: code copied from `parse_macro_use_or_failure` -- use abstraction! + (Ident::invalid(), ast::ImplItemKind::Macro(mac), Generics::default()) } else { - let (name, inner_attrs, generics, kind) = self.parse_impl_method(&vis, at_end)?; + let (name, inner_attrs, generics, kind) = self.parse_impl_method(at_end)?; attrs.extend(inner_attrs); (name, kind, generics) }; @@ -783,71 +773,6 @@ impl<'a> Parser<'a> { Ok((name, ImplItemKind::Const(typ, expr), Generics::default())) } - /// Parses a method or a macro invocation in a trait impl. - fn parse_impl_method( - &mut self, - vis: &Visibility, - at_end: &mut bool - ) -> PResult<'a, (Ident, Vec, Generics, ImplItemKind)> { - // FIXME: code copied from `parse_macro_use_or_failure` -- use abstraction! - if let Some(mac) = self.parse_assoc_macro_invoc("impl", Some(vis), at_end)? { - // method macro - Ok((Ident::invalid(), vec![], Generics::default(), ast::ImplItemKind::Macro(mac))) - } else { - let (ident, sig, generics) = self.parse_method_sig(|_| true)?; - *at_end = true; - let (inner_attrs, body) = self.parse_inner_attrs_and_block()?; - Ok((ident, inner_attrs, generics, ast::ImplItemKind::Method(sig, body))) - } - } - - /// Parse the "signature", including the identifier, parameters, and generics - /// of a method. The body is not parsed as that differs between `trait`s and `impl`s. - fn parse_method_sig( - &mut self, - is_name_required: fn(&token::Token) -> bool, - ) -> PResult<'a, (Ident, MethodSig, Generics)> { - let header = self.parse_fn_front_matter()?; - let (ident, decl, generics) = self.parse_fn_sig(ParamCfg { - is_self_allowed: true, - allow_c_variadic: false, - is_name_required, - })?; - Ok((ident, MethodSig { header, decl }, generics)) - } - - /// Parses all the "front matter" for a `fn` declaration, up to - /// and including the `fn` keyword: - /// - /// - `const fn` - /// - `unsafe fn` - /// - `const unsafe fn` - /// - `extern fn` - /// - etc. - fn parse_fn_front_matter(&mut self) -> PResult<'a, FnHeader> { - let is_const_fn = self.eat_keyword(kw::Const); - let const_span = self.prev_span; - let asyncness = self.parse_asyncness(); - if let IsAsync::Async { .. } = asyncness { - self.ban_async_in_2015(self.prev_span); - } - let asyncness = respan(self.prev_span, asyncness); - let unsafety = self.parse_unsafety(); - let (constness, unsafety, abi) = if is_const_fn { - (respan(const_span, Constness::Const), unsafety, Abi::Rust) - } else { - let abi = self.parse_extern_abi()?; - (respan(self.prev_span, Constness::NotConst), unsafety, abi) - }; - if !self.eat_keyword(kw::Fn) { - // It is possible for `expect_one_of` to recover given the contents of - // `self.expected_tokens`, therefore, do not use `self.unexpected()` which doesn't - // account for this. - if !self.expect_one_of(&[], &[])? { unreachable!() } - } - Ok(FnHeader { constness, unsafety, asyncness, abi }) - } - /// Parses `auto? trait Foo { ... }` or `trait Foo = Bar;`. fn parse_item_trait(&mut self, unsafety: Unsafety) -> PResult<'a, ItemInfo> { // Parse optional `auto` prefix. @@ -957,13 +882,7 @@ impl<'a> Parser<'a> { // trait item macro. (Ident::invalid(), TraitItemKind::Macro(mac), Generics::default()) } else { - // This is somewhat dubious; We don't want to allow - // argument names to be left off if there is a definition... - // - // We don't allow argument names to be left off in edition 2018. - let (ident, sig, generics) = self.parse_method_sig(|t| t.span.rust_2018())?; - let body = self.parse_trait_method_body(at_end, &mut attrs)?; - (ident, TraitItemKind::Method(sig, body), generics) + self.parse_trait_item_method(at_end, &mut attrs)? }; Ok(TraitItem { @@ -991,43 +910,6 @@ impl<'a> Parser<'a> { Ok((ident, TraitItemKind::Const(ty, default), Generics::default())) } - /// Parse the "body" of a method in a trait item definition. - /// This can either be `;` when there's no body, - /// or e.g. a block when the method is a provided one. - fn parse_trait_method_body( - &mut self, - at_end: &mut bool, - attrs: &mut Vec, - ) -> PResult<'a, Option>> { - Ok(match self.token.kind { - token::Semi => { - debug!("parse_trait_method_body(): parsing required method"); - self.bump(); - *at_end = true; - None - } - token::OpenDelim(token::Brace) => { - debug!("parse_trait_method_body(): parsing provided method"); - *at_end = true; - let (inner_attrs, body) = self.parse_inner_attrs_and_block()?; - attrs.extend(inner_attrs.iter().cloned()); - Some(body) - } - token::Interpolated(ref nt) => { - match **nt { - token::NtBlock(..) => { - *at_end = true; - let (inner_attrs, body) = self.parse_inner_attrs_and_block()?; - attrs.extend(inner_attrs.iter().cloned()); - Some(body) - } - _ => return self.expected_semi_or_open_brace(), - } - } - _ => return self.expected_semi_or_open_brace(), - }) - } - /// Parses the following grammar: /// /// TraitItemAssocTy = Ident ["<"...">"] [":" [GenericBounds]] ["where" ...] ["=" Ty] @@ -1194,45 +1076,6 @@ impl<'a> Parser<'a> { Ok(ident) } - /// Parses an item-position function declaration. - fn parse_item_fn( - &mut self, - lo: Span, - vis: Visibility, - attrs: Vec, - header: FnHeader, - ) -> PResult<'a, Option>> { - let (ident, decl, generics) = self.parse_fn_sig(ParamCfg { - is_self_allowed: false, - allow_c_variadic: header.abi == Abi::C && header.unsafety == Unsafety::Unsafe, - is_name_required: |_| true, - })?; - let (inner_attrs, body) = self.parse_inner_attrs_and_block()?; - let kind = ItemKind::Fn(decl, header, generics, body); - self.mk_item_with_info(attrs, lo, vis, (ident, kind, Some(inner_attrs))) - } - - /// Parse the "signature", including the identifier, parameters, and generics of a function. - fn parse_fn_sig(&mut self, cfg: ParamCfg) -> PResult<'a, (Ident, P, Generics)> { - let ident = self.parse_ident()?; - let mut generics = self.parse_generics()?; - let decl = self.parse_fn_decl(cfg, true)?; - generics.where_clause = self.parse_where_clause()?; - Ok((ident, decl, generics)) - } - - /// Parses the parameter list and result type of a function declaration. - pub(super) fn parse_fn_decl( - &mut self, - cfg: ParamCfg, - ret_allow_plus: bool, - ) -> PResult<'a, P> { - Ok(P(FnDecl { - inputs: self.parse_fn_params(cfg)?, - output: self.parse_ret_ty(ret_allow_plus)?, - })) - } - /// Parses `extern` for foreign ABIs modules. /// /// `extern` is expected to have been @@ -1344,32 +1187,6 @@ impl<'a> Parser<'a> { } } - /// Parses a function declaration from a foreign module. - fn parse_item_foreign_fn( - &mut self, - vis: ast::Visibility, - lo: Span, - attrs: Vec, - extern_sp: Span, - ) -> PResult<'a, ForeignItem> { - self.expect_keyword(kw::Fn)?; - let (ident, decl, generics) = self.parse_fn_sig(super::ParamCfg { - is_self_allowed: false, - allow_c_variadic: true, - is_name_required: |_| true, - })?; - let span = lo.to(self.token.span); - self.parse_semi_or_incorrect_foreign_fn_body(&ident, extern_sp)?; - Ok(ast::ForeignItem { - ident, - attrs, - kind: ForeignItemKind::Fn(decl, generics), - id: DUMMY_NODE_ID, - span, - vis, - }) - } - /// Parses a static item from a foreign module. /// Assumes that the `static` keyword is already parsed. fn parse_item_foreign_static(&mut self, vis: ast::Visibility, lo: Span, attrs: Vec) @@ -1910,3 +1727,466 @@ impl<'a> Parser<'a> { }) } } + +/// The parsing configuration used to parse a parameter list (see `parse_fn_params`). +pub(super) struct ParamCfg { + /// Is `self` is allowed as the first parameter? + pub is_self_allowed: bool, + /// Is `...` allowed as the tail of the parameter list? + pub allow_c_variadic: bool, + /// `is_name_required` decides if, per-parameter, + /// the parameter must have a pattern or just a type. + pub is_name_required: fn(&token::Token) -> bool, +} + +/// Parsing of functions and methods. +impl<'a> Parser<'a> { + /// Parses an item-position function declaration. + fn parse_item_fn( + &mut self, + lo: Span, + vis: Visibility, + attrs: Vec, + header: FnHeader, + ) -> PResult<'a, Option>> { + let (ident, decl, generics) = self.parse_fn_sig(ParamCfg { + is_self_allowed: false, + allow_c_variadic: header.abi == Abi::C && header.unsafety == Unsafety::Unsafe, + is_name_required: |_| true, + })?; + let (inner_attrs, body) = self.parse_inner_attrs_and_block()?; + let kind = ItemKind::Fn(decl, header, generics, body); + self.mk_item_with_info(attrs, lo, vis, (ident, kind, Some(inner_attrs))) + } + + /// Parses a function declaration from a foreign module. + fn parse_item_foreign_fn( + &mut self, + vis: ast::Visibility, + lo: Span, + attrs: Vec, + extern_sp: Span, + ) -> PResult<'a, ForeignItem> { + self.expect_keyword(kw::Fn)?; + let (ident, decl, generics) = self.parse_fn_sig(ParamCfg { + is_self_allowed: false, + allow_c_variadic: true, + is_name_required: |_| true, + })?; + let span = lo.to(self.token.span); + self.parse_semi_or_incorrect_foreign_fn_body(&ident, extern_sp)?; + Ok(ast::ForeignItem { + ident, + attrs, + kind: ForeignItemKind::Fn(decl, generics), + id: DUMMY_NODE_ID, + span, + vis, + }) + } + + /// Parses a method or a macro invocation in a trait impl. + fn parse_impl_method( + &mut self, + at_end: &mut bool, + ) -> PResult<'a, (Ident, Vec, Generics, ImplItemKind)> { + let (ident, sig, generics) = self.parse_method_sig(|_| true)?; + *at_end = true; + let (inner_attrs, body) = self.parse_inner_attrs_and_block()?; + Ok((ident, inner_attrs, generics, ast::ImplItemKind::Method(sig, body))) + } + + fn parse_trait_item_method( + &mut self, + at_end: &mut bool, + attrs: &mut Vec, + ) -> PResult<'a, (Ident, TraitItemKind, Generics)> { + // This is somewhat dubious; We don't want to allow + // argument names to be left off if there is a definition... + // + // We don't allow argument names to be left off in edition 2018. + let (ident, sig, generics) = self.parse_method_sig(|t| t.span.rust_2018())?; + let body = self.parse_trait_method_body(at_end, attrs)?; + Ok((ident, TraitItemKind::Method(sig, body), generics)) + } + + /// Parse the "body" of a method in a trait item definition. + /// This can either be `;` when there's no body, + /// or e.g. a block when the method is a provided one. + fn parse_trait_method_body( + &mut self, + at_end: &mut bool, + attrs: &mut Vec, + ) -> PResult<'a, Option>> { + Ok(match self.token.kind { + token::Semi => { + debug!("parse_trait_method_body(): parsing required method"); + self.bump(); + *at_end = true; + None + } + token::OpenDelim(token::Brace) => { + debug!("parse_trait_method_body(): parsing provided method"); + *at_end = true; + let (inner_attrs, body) = self.parse_inner_attrs_and_block()?; + attrs.extend(inner_attrs.iter().cloned()); + Some(body) + } + token::Interpolated(ref nt) => { + match **nt { + token::NtBlock(..) => { + *at_end = true; + let (inner_attrs, body) = self.parse_inner_attrs_and_block()?; + attrs.extend(inner_attrs.iter().cloned()); + Some(body) + } + _ => return self.expected_semi_or_open_brace(), + } + } + _ => return self.expected_semi_or_open_brace(), + }) + } + + /// Parse the "signature", including the identifier, parameters, and generics + /// of a method. The body is not parsed as that differs between `trait`s and `impl`s. + fn parse_method_sig( + &mut self, + is_name_required: fn(&token::Token) -> bool, + ) -> PResult<'a, (Ident, MethodSig, Generics)> { + let header = self.parse_fn_front_matter()?; + let (ident, decl, generics) = self.parse_fn_sig(ParamCfg { + is_self_allowed: true, + allow_c_variadic: false, + is_name_required, + })?; + Ok((ident, MethodSig { header, decl }, generics)) + } + + /// Parses all the "front matter" for a `fn` declaration, up to + /// and including the `fn` keyword: + /// + /// - `const fn` + /// - `unsafe fn` + /// - `const unsafe fn` + /// - `extern fn` + /// - etc. + fn parse_fn_front_matter(&mut self) -> PResult<'a, FnHeader> { + let is_const_fn = self.eat_keyword(kw::Const); + let const_span = self.prev_span; + let asyncness = self.parse_asyncness(); + if let IsAsync::Async { .. } = asyncness { + self.ban_async_in_2015(self.prev_span); + } + let asyncness = respan(self.prev_span, asyncness); + let unsafety = self.parse_unsafety(); + let (constness, unsafety, abi) = if is_const_fn { + (respan(const_span, Constness::Const), unsafety, Abi::Rust) + } else { + let abi = self.parse_extern_abi()?; + (respan(self.prev_span, Constness::NotConst), unsafety, abi) + }; + if !self.eat_keyword(kw::Fn) { + // It is possible for `expect_one_of` to recover given the contents of + // `self.expected_tokens`, therefore, do not use `self.unexpected()` which doesn't + // account for this. + if !self.expect_one_of(&[], &[])? { unreachable!() } + } + Ok(FnHeader { constness, unsafety, asyncness, abi }) + } + + /// Parse the "signature", including the identifier, parameters, and generics of a function. + fn parse_fn_sig(&mut self, cfg: ParamCfg) -> PResult<'a, (Ident, P, Generics)> { + let ident = self.parse_ident()?; + let mut generics = self.parse_generics()?; + let decl = self.parse_fn_decl(cfg, true)?; + generics.where_clause = self.parse_where_clause()?; + Ok((ident, decl, generics)) + } + + /// Parses the parameter list and result type of a function declaration. + pub(super) fn parse_fn_decl( + &mut self, + cfg: ParamCfg, + ret_allow_plus: bool, + ) -> PResult<'a, P> { + Ok(P(FnDecl { + inputs: self.parse_fn_params(cfg)?, + output: self.parse_ret_ty(ret_allow_plus)?, + })) + } + + /// Parses the parameter list of a function, including the `(` and `)` delimiters. + fn parse_fn_params(&mut self, mut cfg: ParamCfg) -> PResult<'a, Vec> { + let sp = self.token.span; + let is_trait_item = cfg.is_self_allowed; + let mut c_variadic = false; + // Parse the arguments, starting out with `self` being possibly allowed... + let (params, _) = self.parse_paren_comma_seq(|p| { + let param = p.parse_param_general(&cfg, is_trait_item); + // ...now that we've parsed the first argument, `self` is no longer allowed. + cfg.is_self_allowed = false; + + match param { + Ok(param) => Ok( + if let TyKind::CVarArgs = param.ty.kind { + c_variadic = true; + if p.token != token::CloseDelim(token::Paren) { + p.span_err( + p.token.span, + "`...` must be the last argument of a C-variadic function", + ); + // FIXME(eddyb) this should probably still push `CVarArgs`. + // Maybe AST validation/HIR lowering should emit the above error? + None + } else { + Some(param) + } + } else { + Some(param) + } + ), + Err(mut e) => { + e.emit(); + let lo = p.prev_span; + // Skip every token until next possible arg or end. + p.eat_to_tokens(&[&token::Comma, &token::CloseDelim(token::Paren)]); + // Create a placeholder argument for proper arg count (issue #34264). + let span = lo.to(p.prev_span); + Ok(Some(dummy_arg(Ident::new(kw::Invalid, span)))) + } + } + })?; + + let mut params: Vec<_> = params.into_iter().filter_map(|x| x).collect(); + + // Replace duplicated recovered params with `_` pattern to avoid unecessary errors. + self.deduplicate_recovered_params_names(&mut params); + + if c_variadic && params.len() <= 1 { + self.span_err( + sp, + "C-variadic function must be declared with at least one named argument", + ); + } + + Ok(params) + } + + /// Skips unexpected attributes and doc comments in this position and emits an appropriate + /// error. + /// This version of parse param doesn't necessarily require identifier names. + fn parse_param_general(&mut self, cfg: &ParamCfg, is_trait_item: bool) -> PResult<'a, Param> { + let lo = self.token.span; + let attrs = self.parse_outer_attributes()?; + + // Possibly parse `self`. Recover if we parsed it and it wasn't allowed here. + if let Some(mut param) = self.parse_self_param()? { + param.attrs = attrs.into(); + return if cfg.is_self_allowed { + Ok(param) + } else { + self.recover_bad_self_param(param, is_trait_item) + }; + } + + let is_name_required = match self.token.kind { + token::DotDotDot => false, + _ => (cfg.is_name_required)(&self.token), + }; + let (pat, ty) = if is_name_required || self.is_named_param() { + debug!("parse_param_general parse_pat (is_name_required:{})", is_name_required); + + let pat = self.parse_fn_param_pat()?; + if let Err(mut err) = self.expect(&token::Colon) { + return if let Some(ident) = self.parameter_without_type( + &mut err, + pat, + is_name_required, + cfg.is_self_allowed, + is_trait_item, + ) { + err.emit(); + Ok(dummy_arg(ident)) + } else { + Err(err) + }; + } + + self.eat_incorrect_doc_comment_for_param_type(); + (pat, self.parse_ty_common(true, true, cfg.allow_c_variadic)?) + } else { + debug!("parse_param_general ident_to_pat"); + let parser_snapshot_before_ty = self.clone(); + self.eat_incorrect_doc_comment_for_param_type(); + let mut ty = self.parse_ty_common(true, true, cfg.allow_c_variadic); + if ty.is_ok() && self.token != token::Comma && + self.token != token::CloseDelim(token::Paren) { + // This wasn't actually a type, but a pattern looking like a type, + // so we are going to rollback and re-parse for recovery. + ty = self.unexpected(); + } + match ty { + Ok(ty) => { + let ident = Ident::new(kw::Invalid, self.prev_span); + let bm = BindingMode::ByValue(Mutability::Immutable); + let pat = self.mk_pat_ident(ty.span, bm, ident); + (pat, ty) + } + // If this is a C-variadic argument and we hit an error, return the error. + Err(err) if self.token == token::DotDotDot => return Err(err), + // Recover from attempting to parse the argument as a type without pattern. + Err(mut err) => { + err.cancel(); + mem::replace(self, parser_snapshot_before_ty); + self.recover_arg_parse()? + } + } + }; + + let span = lo.to(self.token.span); + + Ok(Param { + attrs: attrs.into(), + id: ast::DUMMY_NODE_ID, + is_placeholder: false, + pat, + span, + ty, + }) + } + + /// Returns the parsed optional self parameter and whether a self shortcut was used. + /// + /// See `parse_self_param_with_attrs` to collect attributes. + fn parse_self_param(&mut self) -> PResult<'a, Option> { + // Extract an identifier *after* having confirmed that the token is one. + let expect_self_ident = |this: &mut Self| { + match this.token.kind { + // Preserve hygienic context. + token::Ident(name, _) => { + let span = this.token.span; + this.bump(); + Ident::new(name, span) + } + _ => unreachable!(), + } + }; + // Is `self` `n` tokens ahead? + let is_isolated_self = |this: &Self, n| { + this.is_keyword_ahead(n, &[kw::SelfLower]) + && this.look_ahead(n + 1, |t| t != &token::ModSep) + }; + // Is `mut self` `n` tokens ahead? + let is_isolated_mut_self = |this: &Self, n| { + this.is_keyword_ahead(n, &[kw::Mut]) + && is_isolated_self(this, n + 1) + }; + // Parse `self` or `self: TYPE`. We already know the current token is `self`. + let parse_self_possibly_typed = |this: &mut Self, m| { + let eself_ident = expect_self_ident(this); + let eself_hi = this.prev_span; + let eself = if this.eat(&token::Colon) { + SelfKind::Explicit(this.parse_ty()?, m) + } else { + SelfKind::Value(m) + }; + Ok((eself, eself_ident, eself_hi)) + }; + // Recover for the grammar `*self`, `*const self`, and `*mut self`. + let recover_self_ptr = |this: &mut Self| { + let msg = "cannot pass `self` by raw pointer"; + let span = this.token.span; + this.struct_span_err(span, msg) + .span_label(span, msg) + .emit(); + + Ok((SelfKind::Value(Mutability::Immutable), expect_self_ident(this), this.prev_span)) + }; + + // Parse optional `self` parameter of a method. + // Only a limited set of initial token sequences is considered `self` parameters; anything + // else is parsed as a normal function parameter list, so some lookahead is required. + let eself_lo = self.token.span; + let (eself, eself_ident, eself_hi) = match self.token.kind { + token::BinOp(token::And) => { + let eself = if is_isolated_self(self, 1) { + // `&self` + self.bump(); + SelfKind::Region(None, Mutability::Immutable) + } else if is_isolated_mut_self(self, 1) { + // `&mut self` + self.bump(); + self.bump(); + SelfKind::Region(None, Mutability::Mutable) + } else if self.look_ahead(1, |t| t.is_lifetime()) && is_isolated_self(self, 2) { + // `&'lt self` + self.bump(); + let lt = self.expect_lifetime(); + SelfKind::Region(Some(lt), Mutability::Immutable) + } else if self.look_ahead(1, |t| t.is_lifetime()) && is_isolated_mut_self(self, 2) { + // `&'lt mut self` + self.bump(); + let lt = self.expect_lifetime(); + self.bump(); + SelfKind::Region(Some(lt), Mutability::Mutable) + } else { + // `¬_self` + return Ok(None); + }; + (eself, expect_self_ident(self), self.prev_span) + } + // `*self` + token::BinOp(token::Star) if is_isolated_self(self, 1) => { + self.bump(); + recover_self_ptr(self)? + } + // `*mut self` and `*const self` + token::BinOp(token::Star) if + self.look_ahead(1, |t| t.is_mutability()) + && is_isolated_self(self, 2) => + { + self.bump(); + self.bump(); + recover_self_ptr(self)? + } + // `self` and `self: TYPE` + token::Ident(..) if is_isolated_self(self, 0) => { + parse_self_possibly_typed(self, Mutability::Immutable)? + } + // `mut self` and `mut self: TYPE` + token::Ident(..) if is_isolated_mut_self(self, 0) => { + self.bump(); + parse_self_possibly_typed(self, Mutability::Mutable)? + } + _ => return Ok(None), + }; + + let eself = source_map::respan(eself_lo.to(eself_hi), eself); + Ok(Some(Param::from_self(ThinVec::default(), eself, eself_ident))) + } + + fn is_named_param(&self) -> bool { + let offset = match self.token.kind { + token::Interpolated(ref nt) => match **nt { + token::NtPat(..) => return self.look_ahead(1, |t| t == &token::Colon), + _ => 0, + } + token::BinOp(token::And) | token::AndAnd => 1, + _ if self.token.is_keyword(kw::Mut) => 1, + _ => 0, + }; + + self.look_ahead(offset, |t| t.is_ident()) && + self.look_ahead(offset + 1, |t| t == &token::Colon) + } + + fn recover_first_param(&mut self) -> &'static str { + match self.parse_outer_attributes() + .and_then(|_| self.parse_self_param()) + .map_err(|mut e| e.cancel()) + { + Ok(Some(_)) => "method", + _ => "function", + } + } +} diff --git a/src/libsyntax/parse/parser/ty.rs b/src/libsyntax/parse/parser/ty.rs index 018b5951e6e..e696ab0804d 100644 --- a/src/libsyntax/parse/parser/ty.rs +++ b/src/libsyntax/parse/parser/ty.rs @@ -1,4 +1,5 @@ use super::{Parser, PResult, PathStyle, PrevTokenKind, TokenType}; +use super::item::ParamCfg; use crate::{maybe_whole, maybe_recover_from_interpolated_ty_qpath}; use crate::ptr::P; @@ -281,7 +282,7 @@ impl<'a> Parser<'a> { let unsafety = self.parse_unsafety(); let abi = self.parse_extern_abi()?; self.expect_keyword(kw::Fn)?; - let cfg = super::ParamCfg { + let cfg = ParamCfg { is_self_allowed: false, allow_c_variadic: true, is_name_required: |_| false, -- cgit 1.4.1-3-g733a5 From 42f32f06d63b9947db63f0c739483e0294f588b2 Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Wed, 9 Oct 2019 02:34:22 +0200 Subject: token: extract Nonterminal::to_tokenstream to parser. --- src/librustc/hir/lowering.rs | 6 +- src/libsyntax/ext/proc_macro_server.rs | 2 +- src/libsyntax/parse/mod.rs | 138 ++++++++++++++++++++++++++++++++- src/libsyntax/parse/token.rs | 137 +------------------------------- 4 files changed, 143 insertions(+), 140 deletions(-) (limited to 'src/libsyntax') diff --git a/src/librustc/hir/lowering.rs b/src/librustc/hir/lowering.rs index 72fd054ee8a..513512b9f6b 100644 --- a/src/librustc/hir/lowering.rs +++ b/src/librustc/hir/lowering.rs @@ -70,6 +70,7 @@ use syntax::print::pprust; use syntax::source_map::{respan, ExpnData, ExpnKind, DesugaringKind, Spanned}; use syntax::symbol::{kw, sym, Symbol}; use syntax::tokenstream::{TokenStream, TokenTree}; +use syntax::parse; use syntax::parse::token::{self, Token}; use syntax::visit::{self, Visitor}; use syntax_pos::Span; @@ -1022,7 +1023,10 @@ impl<'a> LoweringContext<'a> { fn lower_token(&mut self, token: Token) -> TokenStream { match token.kind { token::Interpolated(nt) => { - let tts = nt.to_tokenstream(&self.sess.parse_sess, token.span); + // FIXME(Centril): Consider indirection `(parse_sess.nt_to_tokenstream)(...)` + // to hack around the current hack that requires `nt_to_tokenstream` to live + // in the parser. + let tts = parse::nt_to_tokenstream(&nt, &self.sess.parse_sess, token.span); self.lower_token_stream(tts) } _ => TokenTree::Token(token).into(), diff --git a/src/libsyntax/ext/proc_macro_server.rs b/src/libsyntax/ext/proc_macro_server.rs index 021ec46d987..0647c2fa313 100644 --- a/src/libsyntax/ext/proc_macro_server.rs +++ b/src/libsyntax/ext/proc_macro_server.rs @@ -174,7 +174,7 @@ impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec)> } Interpolated(nt) => { - let stream = nt.to_tokenstream(sess, span); + let stream = parse::nt_to_tokenstream(&nt, sess, span); TokenTree::Group(Group { delimiter: Delimiter::None, stream, diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index 1518da23b09..0df695d37b9 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -4,10 +4,9 @@ use crate::ast::{self, CrateConfig, NodeId}; use crate::early_buffered_lints::{BufferedEarlyLint, BufferedEarlyLintId}; use crate::source_map::{SourceMap, FilePathMapping}; use crate::feature_gate::UnstableFeatures; -use crate::parse::parser::Parser; -use crate::parse::parser::emit_unclosed_delims; -use crate::parse::token::TokenKind; -use crate::tokenstream::{TokenStream, TokenTree}; +use crate::parse::parser::{Parser, emit_unclosed_delims}; +use crate::parse::token::{Nonterminal, TokenKind}; +use crate::tokenstream::{self, TokenStream, TokenTree}; use crate::print::pprust; use crate::symbol::Symbol; @@ -24,6 +23,8 @@ use std::borrow::Cow; use std::path::{Path, PathBuf}; use std::str; +use log::info; + #[cfg(test)] mod tests; @@ -407,3 +408,132 @@ impl SeqSep { } } } + +// NOTE(Centril): The following probably shouldn't be here but it acknowledges the +// fact that architecturally, we are using parsing (read on below to understand why). + +pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> TokenStream { + // A `Nonterminal` is often a parsed AST item. At this point we now + // need to convert the parsed AST to an actual token stream, e.g. + // un-parse it basically. + // + // Unfortunately there's not really a great way to do that in a + // guaranteed lossless fashion right now. The fallback here is to just + // stringify the AST node and reparse it, but this loses all span + // information. + // + // As a result, some AST nodes are annotated with the token stream they + // came from. Here we attempt to extract these lossless token streams + // before we fall back to the stringification. + let tokens = match *nt { + Nonterminal::NtItem(ref item) => { + prepend_attrs(sess, &item.attrs, item.tokens.as_ref(), span) + } + Nonterminal::NtTraitItem(ref item) => { + prepend_attrs(sess, &item.attrs, item.tokens.as_ref(), span) + } + Nonterminal::NtImplItem(ref item) => { + prepend_attrs(sess, &item.attrs, item.tokens.as_ref(), span) + } + Nonterminal::NtIdent(ident, is_raw) => { + Some(tokenstream::TokenTree::token(token::Ident(ident.name, is_raw), ident.span).into()) + } + Nonterminal::NtLifetime(ident) => { + Some(tokenstream::TokenTree::token(token::Lifetime(ident.name), ident.span).into()) + } + Nonterminal::NtTT(ref tt) => { + Some(tt.clone().into()) + } + _ => None, + }; + + // FIXME(#43081): Avoid this pretty-print + reparse hack + let source = pprust::nonterminal_to_string(nt); + let filename = FileName::macro_expansion_source_code(&source); + let tokens_for_real = parse_stream_from_source_str(filename, source, sess, Some(span)); + + // During early phases of the compiler the AST could get modified + // directly (e.g., attributes added or removed) and the internal cache + // of tokens my not be invalidated or updated. Consequently if the + // "lossless" token stream disagrees with our actual stringification + // (which has historically been much more battle-tested) then we go + // with the lossy stream anyway (losing span information). + // + // Note that the comparison isn't `==` here to avoid comparing spans, + // but it *also* is a "probable" equality which is a pretty weird + // definition. We mostly want to catch actual changes to the AST + // like a `#[cfg]` being processed or some weird `macro_rules!` + // expansion. + // + // What we *don't* want to catch is the fact that a user-defined + // literal like `0xf` is stringified as `15`, causing the cached token + // stream to not be literal `==` token-wise (ignoring spans) to the + // token stream we got from stringification. + // + // Instead the "probably equal" check here is "does each token + // recursively have the same discriminant?" We basically don't look at + // the token values here and assume that such fine grained token stream + // modifications, including adding/removing typically non-semantic + // tokens such as extra braces and commas, don't happen. + if let Some(tokens) = tokens { + if tokens.probably_equal_for_proc_macro(&tokens_for_real) { + return tokens + } + info!("cached tokens found, but they're not \"probably equal\", \ + going with stringified version"); + } + return tokens_for_real +} + +fn prepend_attrs( + sess: &ParseSess, + attrs: &[ast::Attribute], + tokens: Option<&tokenstream::TokenStream>, + span: syntax_pos::Span +) -> Option { + let tokens = tokens?; + if attrs.len() == 0 { + return Some(tokens.clone()) + } + let mut builder = tokenstream::TokenStreamBuilder::new(); + for attr in attrs { + assert_eq!(attr.style, ast::AttrStyle::Outer, + "inner attributes should prevent cached tokens from existing"); + + let source = pprust::attribute_to_string(attr); + let macro_filename = FileName::macro_expansion_source_code(&source); + if attr.is_sugared_doc { + let stream = parse_stream_from_source_str(macro_filename, source, sess, Some(span)); + builder.push(stream); + continue + } + + // synthesize # [ $path $tokens ] manually here + let mut brackets = tokenstream::TokenStreamBuilder::new(); + + // For simple paths, push the identifier directly + if attr.path.segments.len() == 1 && attr.path.segments[0].args.is_none() { + let ident = attr.path.segments[0].ident; + let token = token::Ident(ident.name, ident.as_str().starts_with("r#")); + brackets.push(tokenstream::TokenTree::token(token, ident.span)); + + // ... and for more complicated paths, fall back to a reparse hack that + // should eventually be removed. + } else { + let stream = parse_stream_from_source_str(macro_filename, source, sess, Some(span)); + brackets.push(stream); + } + + brackets.push(attr.tokens.clone()); + + // The span we list here for `#` and for `[ ... ]` are both wrong in + // that it encompasses more than each token, but it hopefully is "good + // enough" for now at least. + builder.push(tokenstream::TokenTree::token(token::Pound, attr.span)); + let delim_span = tokenstream::DelimSpan::from_single(attr.span); + builder.push(tokenstream::TokenTree::Delimited( + delim_span, token::DelimToken::Bracket, brackets.build().into())); + } + builder.push(tokens.clone()); + Some(builder.build()) +} diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index fd78a2bd534..eb74ab2b919 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -4,16 +4,13 @@ pub use DelimToken::*; pub use LitKind::*; pub use TokenKind::*; -use crate::ast::{self}; -use crate::parse::{parse_stream_from_source_str, ParseSess}; -use crate::print::pprust; +use crate::ast; use crate::ptr::P; use crate::symbol::kw; -use crate::tokenstream::{self, DelimSpan, TokenStream, TokenTree}; +use crate::tokenstream::TokenTree; use syntax_pos::symbol::Symbol; -use syntax_pos::{self, Span, FileName, DUMMY_SP}; -use log::info; +use syntax_pos::{self, Span, DUMMY_SP}; use std::fmt; use std::mem; @@ -737,131 +734,3 @@ impl fmt::Debug for Nonterminal { } } } - -impl Nonterminal { - pub fn to_tokenstream(&self, sess: &ParseSess, span: Span) -> TokenStream { - // A `Nonterminal` is often a parsed AST item. At this point we now - // need to convert the parsed AST to an actual token stream, e.g. - // un-parse it basically. - // - // Unfortunately there's not really a great way to do that in a - // guaranteed lossless fashion right now. The fallback here is to just - // stringify the AST node and reparse it, but this loses all span - // information. - // - // As a result, some AST nodes are annotated with the token stream they - // came from. Here we attempt to extract these lossless token streams - // before we fall back to the stringification. - let tokens = match *self { - Nonterminal::NtItem(ref item) => { - prepend_attrs(sess, &item.attrs, item.tokens.as_ref(), span) - } - Nonterminal::NtTraitItem(ref item) => { - prepend_attrs(sess, &item.attrs, item.tokens.as_ref(), span) - } - Nonterminal::NtImplItem(ref item) => { - prepend_attrs(sess, &item.attrs, item.tokens.as_ref(), span) - } - Nonterminal::NtIdent(ident, is_raw) => { - Some(TokenTree::token(Ident(ident.name, is_raw), ident.span).into()) - } - Nonterminal::NtLifetime(ident) => { - Some(TokenTree::token(Lifetime(ident.name), ident.span).into()) - } - Nonterminal::NtTT(ref tt) => { - Some(tt.clone().into()) - } - _ => None, - }; - - // FIXME(#43081): Avoid this pretty-print + reparse hack - let source = pprust::nonterminal_to_string(self); - let filename = FileName::macro_expansion_source_code(&source); - let tokens_for_real = parse_stream_from_source_str(filename, source, sess, Some(span)); - - // During early phases of the compiler the AST could get modified - // directly (e.g., attributes added or removed) and the internal cache - // of tokens my not be invalidated or updated. Consequently if the - // "lossless" token stream disagrees with our actual stringification - // (which has historically been much more battle-tested) then we go - // with the lossy stream anyway (losing span information). - // - // Note that the comparison isn't `==` here to avoid comparing spans, - // but it *also* is a "probable" equality which is a pretty weird - // definition. We mostly want to catch actual changes to the AST - // like a `#[cfg]` being processed or some weird `macro_rules!` - // expansion. - // - // What we *don't* want to catch is the fact that a user-defined - // literal like `0xf` is stringified as `15`, causing the cached token - // stream to not be literal `==` token-wise (ignoring spans) to the - // token stream we got from stringification. - // - // Instead the "probably equal" check here is "does each token - // recursively have the same discriminant?" We basically don't look at - // the token values here and assume that such fine grained token stream - // modifications, including adding/removing typically non-semantic - // tokens such as extra braces and commas, don't happen. - if let Some(tokens) = tokens { - if tokens.probably_equal_for_proc_macro(&tokens_for_real) { - return tokens - } - info!("cached tokens found, but they're not \"probably equal\", \ - going with stringified version"); - } - return tokens_for_real - } -} - -fn prepend_attrs(sess: &ParseSess, - attrs: &[ast::Attribute], - tokens: Option<&tokenstream::TokenStream>, - span: syntax_pos::Span) - -> Option -{ - let tokens = tokens?; - if attrs.len() == 0 { - return Some(tokens.clone()) - } - let mut builder = tokenstream::TokenStreamBuilder::new(); - for attr in attrs { - assert_eq!(attr.style, ast::AttrStyle::Outer, - "inner attributes should prevent cached tokens from existing"); - - let source = pprust::attribute_to_string(attr); - let macro_filename = FileName::macro_expansion_source_code(&source); - if attr.is_sugared_doc { - let stream = parse_stream_from_source_str(macro_filename, source, sess, Some(span)); - builder.push(stream); - continue - } - - // synthesize # [ $path $tokens ] manually here - let mut brackets = tokenstream::TokenStreamBuilder::new(); - - // For simple paths, push the identifier directly - if attr.path.segments.len() == 1 && attr.path.segments[0].args.is_none() { - let ident = attr.path.segments[0].ident; - let token = Ident(ident.name, ident.as_str().starts_with("r#")); - brackets.push(tokenstream::TokenTree::token(token, ident.span)); - - // ... and for more complicated paths, fall back to a reparse hack that - // should eventually be removed. - } else { - let stream = parse_stream_from_source_str(macro_filename, source, sess, Some(span)); - brackets.push(stream); - } - - brackets.push(attr.tokens.clone()); - - // The span we list here for `#` and for `[ ... ]` are both wrong in - // that it encompasses more than each token, but it hopefully is "good - // enough" for now at least. - builder.push(tokenstream::TokenTree::token(Pound, attr.span)); - let delim_span = DelimSpan::from_single(attr.span); - builder.push(tokenstream::TokenTree::Delimited( - delim_span, DelimToken::Bracket, brackets.build().into())); - } - builder.push(tokens.clone()); - Some(builder.build()) -} -- cgit 1.4.1-3-g733a5