From f2d14077439a3037941781b7b53ea4e9ef19f52c Mon Sep 17 00:00:00 2001 From: Jeffrey Seyfried Date: Tue, 17 Jan 2017 03:14:42 +0000 Subject: Remove field `tokens_consumed` of `Parser`. --- src/libsyntax/parse/parser.rs | 3 --- 1 file changed, 3 deletions(-) (limited to 'src/libsyntax') diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index 167fa78d7e0..9ba6d4d17f7 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -185,7 +185,6 @@ pub struct Parser<'a> { /// the previous token kind prev_token_kind: PrevTokenKind, lookahead_buffer: LookaheadBuffer, - pub tokens_consumed: usize, pub restrictions: Restrictions, pub quote_depth: usize, // not (yet) related to the quasiquoter parsing_token_tree: bool, @@ -282,7 +281,6 @@ impl<'a> Parser<'a> { prev_span: syntax_pos::DUMMY_SP, prev_token_kind: PrevTokenKind::Other, lookahead_buffer: Default::default(), - tokens_consumed: 0, restrictions: Restrictions::empty(), quote_depth: 0, parsing_token_tree: false, @@ -902,7 +900,6 @@ impl<'a> Parser<'a> { }; self.span = next.sp; self.token = next.tok; - self.tokens_consumed += 1; self.expected_tokens.clear(); // check after each token self.check_unknown_macro_variable(); -- cgit 1.4.1-3-g733a5 From 375cbd20cfcc9dbf15682bcfc0081ce5ce95567b Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 9 Jan 2017 01:31:14 -0800 Subject: Implement `#[proc_macro_attribute]` * Add support for `#[proc_macro]` * Reactivate `proc_macro` feature and gate `#[proc_macro_attribute]` under it * Have `#![feature(proc_macro)]` imply `#![feature(use_extern_macros)]`, error on legacy import of proc macros via `#[macro_use]` --- src/librustc_driver/driver.rs | 1 + src/librustc_metadata/creader.rs | 10 + src/librustc_resolve/lib.rs | 46 ++++- src/librustc_resolve/macros.rs | 43 ++++- src/libsyntax/ext/expand.rs | 30 ++- src/libsyntax/feature_gate.rs | 60 +++++- src/libsyntax_ext/lib.rs | 2 + src/libsyntax_ext/proc_macro_impl.rs | 58 ++++++ src/libsyntax_ext/proc_macro_registrar.rs | 204 ++++++++++++++------- .../proc-macro/auxiliary/attr_proc_macro.rs | 23 +++ .../proc-macro/feature-gate-proc_macro.rs | 24 +++ .../proc-macro/macro-use-attr.rs | 22 +++ .../proc-macro/proc-macro-custom-attr-mutex.rs | 24 +++ src/test/run-pass-fulldeps/proc-macro/attr-args.rs | 24 +++ .../proc-macro/auxiliary/attr-args.rs | 32 ++++ src/tools/tidy/src/features.rs | 2 +- 16 files changed, 526 insertions(+), 79 deletions(-) create mode 100644 src/libsyntax_ext/proc_macro_impl.rs create mode 100644 src/test/compile-fail-fulldeps/proc-macro/auxiliary/attr_proc_macro.rs create mode 100644 src/test/compile-fail-fulldeps/proc-macro/feature-gate-proc_macro.rs create mode 100644 src/test/compile-fail-fulldeps/proc-macro/macro-use-attr.rs create mode 100644 src/test/compile-fail-fulldeps/proc-macro/proc-macro-custom-attr-mutex.rs create mode 100644 src/test/run-pass-fulldeps/proc-macro/attr-args.rs create mode 100644 src/test/run-pass-fulldeps/proc-macro/auxiliary/attr-args.rs (limited to 'src/libsyntax') diff --git a/src/librustc_driver/driver.rs b/src/librustc_driver/driver.rs index 442c139f14c..f34d1203a3d 100644 --- a/src/librustc_driver/driver.rs +++ b/src/librustc_driver/driver.rs @@ -677,6 +677,7 @@ pub fn phase_2_configure_and_expand(sess: &Session, should_test: sess.opts.test, ..syntax::ext::expand::ExpansionConfig::default(crate_name.to_string()) }; + let mut ecx = ExtCtxt::new(&sess.parse_sess, cfg, &mut resolver); let err_count = ecx.parse_sess.span_diagnostic.err_count(); diff --git a/src/librustc_metadata/creader.rs b/src/librustc_metadata/creader.rs index 8f7b9c24cbf..161331b1728 100644 --- a/src/librustc_metadata/creader.rs +++ b/src/librustc_metadata/creader.rs @@ -578,6 +578,7 @@ impl<'a> CrateLoader<'a> { use proc_macro::__internal::Registry; use rustc_back::dynamic_lib::DynamicLibrary; use syntax_ext::deriving::custom::CustomDerive; + use syntax_ext::proc_macro_impl::AttrProcMacro; let path = match dylib { Some(dylib) => dylib, @@ -613,6 +614,15 @@ impl<'a> CrateLoader<'a> { ); self.0.push((Symbol::intern(trait_name), Rc::new(derive))); } + + fn register_attr_proc_macro(&mut self, + name: &str, + expand: fn(TokenStream, TokenStream) -> TokenStream) { + let expand = SyntaxExtension::AttrProcMacro( + Box::new(AttrProcMacro { inner: expand }) + ); + self.0.push((Symbol::intern(name), Rc::new(expand))); + } } let mut my_registrar = MyRegistrar(Vec::new()); diff --git a/src/librustc_resolve/lib.rs b/src/librustc_resolve/lib.rs index 56e8c75b859..2c5e3385638 100644 --- a/src/librustc_resolve/lib.rs +++ b/src/librustc_resolve/lib.rs @@ -61,7 +61,7 @@ use syntax::ast::{FnDecl, ForeignItem, ForeignItemKind, Generics}; use syntax::ast::{Item, ItemKind, ImplItem, ImplItemKind}; use syntax::ast::{Local, Mutability, Pat, PatKind, Path}; use syntax::ast::{QSelf, TraitItemKind, TraitRef, Ty, TyKind}; -use syntax::feature_gate::{emit_feature_err, GateIssue}; +use syntax::feature_gate::{feature_err, emit_feature_err, GateIssue}; use syntax_pos::{Span, DUMMY_SP, MultiSpan}; use errors::DiagnosticBuilder; @@ -1123,6 +1123,12 @@ pub struct Resolver<'a> { // Avoid duplicated errors for "name already defined". name_already_seen: FxHashMap, + + // If `#![feature(proc_macro)]` is set + proc_macro_enabled: bool, + + // A set of procedural macros imported by `#[macro_use]` that have already been warned about + warned_proc_macros: FxHashSet, } pub struct ResolverArenas<'a> { @@ -1227,6 +1233,8 @@ impl<'a> Resolver<'a> { invocations.insert(Mark::root(), arenas.alloc_invocation_data(InvocationData::root(graph_root))); + let features = session.features.borrow(); + Resolver { session: session, @@ -1284,7 +1292,9 @@ impl<'a> Resolver<'a> { span: DUMMY_SP, vis: ty::Visibility::Public, }), - use_extern_macros: session.features.borrow().use_extern_macros, + + // `#![feature(proc_macro)]` implies `#[feature(extern_macros)]` + use_extern_macros: features.use_extern_macros || features.proc_macro, exported_macros: Vec::new(), crate_loader: crate_loader, @@ -1296,6 +1306,8 @@ impl<'a> Resolver<'a> { invocations: invocations, name_already_seen: FxHashMap(), whitelisted_legacy_custom_derives: Vec::new(), + proc_macro_enabled: features.proc_macro, + warned_proc_macros: FxHashSet(), } } @@ -1525,6 +1537,8 @@ impl<'a> Resolver<'a> { debug!("(resolving item) resolving {}", name); + self.check_proc_macro_attrs(&item.attrs); + match item.node { ItemKind::Enum(_, ref generics) | ItemKind::Ty(_, ref generics) | @@ -1554,6 +1568,8 @@ impl<'a> Resolver<'a> { walk_list!(this, visit_ty_param_bound, bounds); for trait_item in trait_items { + this.check_proc_macro_attrs(&trait_item.attrs); + match trait_item.node { TraitItemKind::Const(_, ref default) => { // Only impose the restrictions of @@ -1738,6 +1754,7 @@ impl<'a> Resolver<'a> { this.with_self_rib(Def::SelfTy(trait_id, Some(item_def_id)), |this| { this.with_current_self_type(self_type, |this| { for impl_item in impl_items { + this.check_proc_macro_attrs(&impl_item.attrs); this.resolve_visibility(&impl_item.vis); match impl_item.node { ImplItemKind::Const(..) => { @@ -3184,6 +3201,31 @@ impl<'a> Resolver<'a> { let msg = "`self` no longer imports values".to_string(); self.session.add_lint(lint::builtin::LEGACY_IMPORTS, id, span, msg); } + + fn check_proc_macro_attrs(&mut self, attrs: &[ast::Attribute]) { + if self.proc_macro_enabled { return; } + + for attr in attrs { + let maybe_binding = self.builtin_macros.get(&attr.name()).cloned().or_else(|| { + let ident = Ident::with_empty_ctxt(attr.name()); + self.resolve_lexical_macro_path_segment(ident, MacroNS, None).ok() + }); + + if let Some(binding) = maybe_binding { + if let SyntaxExtension::AttrProcMacro(..) = *binding.get_macro(self) { + attr::mark_known(attr); + + let msg = "attribute procedural macros are experimental"; + let feature = "proc_macro"; + + feature_err(&self.session.parse_sess, feature, + attr.span, GateIssue::Language, msg) + .span_note(binding.span, "procedural macro imported here") + .emit(); + } + } + } + } } fn is_struct_like(def: Def) -> bool { diff --git a/src/librustc_resolve/macros.rs b/src/librustc_resolve/macros.rs index 403797b6d31..9b7d6f33a7f 100644 --- a/src/librustc_resolve/macros.rs +++ b/src/librustc_resolve/macros.rs @@ -27,7 +27,7 @@ use syntax::ext::base::{NormalTT, Resolver as SyntaxResolver, SyntaxExtension}; use syntax::ext::expand::{Expansion, mark_tts}; use syntax::ext::hygiene::Mark; use syntax::ext::tt::macro_rules; -use syntax::feature_gate::{emit_feature_err, GateIssue}; +use syntax::feature_gate::{emit_feature_err, GateIssue, is_builtin_attr}; use syntax::fold::{self, Folder}; use syntax::ptr::P; use syntax::symbol::keywords; @@ -183,6 +183,10 @@ impl<'a> base::Resolver for Resolver<'a> { }, None => {} } + + if self.proc_macro_enabled && !is_builtin_attr(&attrs[i]) { + return Some(attrs.remove(i)); + } } None } @@ -373,6 +377,10 @@ impl<'a> Resolver<'a> { let resolution = self.resolve_lexical_macro_path_segment(ident, MacroNS, Some(span)); let (legacy_resolution, resolution) = match (legacy_resolution, resolution) { (Some(legacy_resolution), Ok(resolution)) => (legacy_resolution, resolution), + (Some(MacroBinding::Modern(binding)), Err(_)) => { + self.err_if_macro_use_proc_macro(ident.name, span, binding); + continue + }, _ => continue, }; let (legacy_span, participle) = match legacy_resolution { @@ -469,4 +477,37 @@ impl<'a> Resolver<'a> { self.exported_macros.push(def); } } + + /// Error if `ext` is a Macros 1.1 procedural macro being imported by `#[macro_use]` + fn err_if_macro_use_proc_macro(&mut self, name: Name, use_span: Span, + binding: &NameBinding<'a>) { + use self::SyntaxExtension::*; + + let krate = binding.def().def_id().krate; + + // Plugin-based syntax extensions are exempt from this check + if krate == BUILTIN_MACROS_CRATE { return; } + + let ext = binding.get_macro(self); + + match *ext { + // If `ext` is a procedural macro, check if we've already warned about it + AttrProcMacro(_) | ProcMacro(_) => if !self.warned_proc_macros.insert(name) { return; }, + _ => return, + } + + let warn_msg = match *ext { + AttrProcMacro(_) => "attribute procedural macros cannot be \ + imported with `#[macro_use]`", + ProcMacro(_) => "procedural macros cannot be imported with `#[macro_use]`", + _ => return, + }; + + let crate_name = self.session.cstore.crate_name(krate); + + self.session.struct_span_err(use_span, warn_msg) + .help(&format!("instead, import the procedural macro like any other item: \ + `use {}::{};`", crate_name, name)) + .emit(); + } } diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index 201e8d69494..1f7874274f7 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -364,7 +364,9 @@ impl<'a, 'b> MacroExpander<'a, 'b> { kind.expect_from_annotatables(items) } SyntaxExtension::AttrProcMacro(ref mac) => { - let attr_toks = TokenStream::from_tts(tts_for_attr(&attr, &self.cx.parse_sess)); + let attr_toks = TokenStream::from_tts(tts_for_attr_args(&attr, + &self.cx.parse_sess)); + let item_toks = TokenStream::from_tts(tts_for_item(&item, &self.cx.parse_sess)); let tok_result = mac.expand(self.cx, attr.span, attr_toks, item_toks); @@ -640,8 +642,30 @@ fn tts_for_item(item: &Annotatable, parse_sess: &ParseSess) -> Vec { string_to_tts(text, parse_sess) } -fn tts_for_attr(attr: &ast::Attribute, parse_sess: &ParseSess) -> Vec { - string_to_tts(pprust::attr_to_string(attr), parse_sess) +fn tts_for_attr_args(attr: &ast::Attribute, parse_sess: &ParseSess) -> Vec { + use ast::MetaItemKind::*; + use print::pp::Breaks; + use print::pprust::PrintState; + + let token_string = match attr.value.node { + // For `#[foo]`, an empty token + Word => return vec![], + // For `#[foo(bar, baz)]`, returns `(bar, baz)` + List(ref items) => pprust::to_string(|s| { + s.popen()?; + s.commasep(Breaks::Consistent, + &items[..], + |s, i| s.print_meta_list_item(&i))?; + s.pclose() + }), + // For `#[foo = "bar"]`, returns `= "bar"` + NameValue(ref lit) => pprust::to_string(|s| { + s.word_space("=")?; + s.print_literal(lit) + }), + }; + + string_to_tts(token_string, parse_sess) } fn string_to_tts(text: String, parse_sess: &ParseSess) -> Vec { diff --git a/src/libsyntax/feature_gate.rs b/src/libsyntax/feature_gate.rs index 90cca3129dc..2478ed169cd 100644 --- a/src/libsyntax/feature_gate.rs +++ b/src/libsyntax/feature_gate.rs @@ -30,7 +30,7 @@ use ast::{self, NodeId, PatKind}; use attr; use codemap::{CodeMap, Spanned}; use syntax_pos::Span; -use errors::{DiagnosticBuilder, Handler}; +use errors::{DiagnosticBuilder, Handler, FatalError}; use visit::{self, FnKind, Visitor}; use parse::ParseSess; use symbol::Symbol; @@ -325,6 +325,9 @@ declare_features! ( // The `unadjusted` ABI. Perma unstable. (active, abi_unadjusted, "1.16.0", None), + // Macros 1.1 + (active, proc_macro, "1.16.0", Some(35900)), + // Allows attributes on struct literal fields. (active, struct_field_attributes, "1.16.0", Some(38814)), ); @@ -377,8 +380,6 @@ declare_features! ( // Allows `..` in tuple (struct) patterns (accepted, dotdot_in_tuple_patterns, "1.14.0", Some(33627)), (accepted, item_like_imports, "1.14.0", Some(35120)), - // Macros 1.1 - (accepted, proc_macro, "1.15.0", Some(35900)), ); // (changing above list without updating src/doc/reference.md makes @cmr sad) @@ -446,6 +447,10 @@ pub fn deprecated_attributes() -> Vec<&'static (&'static str, AttributeType, Att BUILTIN_ATTRIBUTES.iter().filter(|a| a.2.is_deprecated()).collect() } +pub fn is_builtin_attr(attr: &ast::Attribute) -> bool { + BUILTIN_ATTRIBUTES.iter().any(|&(builtin_name, _, _)| attr.check_name(builtin_name)) +} + // Attributes that have a special meaning to rustc or rustdoc pub const BUILTIN_ATTRIBUTES: &'static [(&'static str, AttributeType, AttributeGate)] = &[ // Normal attributes @@ -739,6 +744,16 @@ pub const BUILTIN_ATTRIBUTES: &'static [(&'static str, AttributeType, AttributeG is currently unstable", cfg_fn!(windows_subsystem))), + ("proc_macro_attribute", Normal, Gated(Stability::Unstable, + "proc_macro", + "attribute proc macros are currently unstable", + cfg_fn!(proc_macro))), + + ("rustc_derive_registrar", Normal, Gated(Stability::Unstable, + "rustc_derive_registrar", + "used internally by rustc", + cfg_fn!(rustc_attrs))), + // Crate level attributes ("crate_name", CrateLevel, Ungated), ("crate_type", CrateLevel, Ungated), @@ -1380,6 +1395,8 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> { pub fn get_features(span_handler: &Handler, krate_attrs: &[ast::Attribute]) -> Features { let mut features = Features::new(); + let mut feature_checker = MutexFeatureChecker::default(); + for attr in krate_attrs { if !attr.check_name("feature") { continue @@ -1403,6 +1420,7 @@ pub fn get_features(span_handler: &Handler, krate_attrs: &[ast::Attribute]) -> F if let Some(&(_, _, _, setter)) = ACTIVE_FEATURES.iter() .find(|& &(n, _, _, _)| name == n) { *(setter(&mut features)) = true; + feature_checker.collect(&features, mi.span); } else if let Some(&(_, _, _)) = REMOVED_FEATURES.iter() .find(|& &(n, _, _)| name == n) { @@ -1419,9 +1437,45 @@ pub fn get_features(span_handler: &Handler, krate_attrs: &[ast::Attribute]) -> F } } + feature_checker.check(span_handler); + features } +// A collector for mutually-exclusive features and their flag spans +#[derive(Default)] +struct MutexFeatureChecker { + proc_macro: Option, + custom_attribute: Option, +} + +impl MutexFeatureChecker { + // If this method turns out to be a hotspot due to branching, + // the branching can be eliminated by modifying `setter!()` to set these spans + // only for the features that need to be checked for mutual exclusion. + fn collect(&mut self, features: &Features, span: Span) { + if features.proc_macro { + // If self.proc_macro is None, set to Some(span) + self.proc_macro = self.proc_macro.or(Some(span)); + } + + if features.custom_attribute { + self.custom_attribute = self.custom_attribute.or(Some(span)); + } + } + + fn check(self, handler: &Handler) { + if let (Some(pm_span), Some(ca_span)) = (self.proc_macro, self.custom_attribute) { + handler.struct_span_err(pm_span, "Cannot use `#![feature(proc_macro)]` and \ + `#![feature(custom_attribute)] at the same time") + .span_note(ca_span, "`#![feature(custom_attribute)]` declared here") + .emit(); + + panic!(FatalError); + } + } +} + pub fn check_crate(krate: &ast::Crate, sess: &ParseSess, features: &Features, diff --git a/src/libsyntax_ext/lib.rs b/src/libsyntax_ext/lib.rs index bdec86158a4..ebec23d0901 100644 --- a/src/libsyntax_ext/lib.rs +++ b/src/libsyntax_ext/lib.rs @@ -47,6 +47,8 @@ pub mod proc_macro_registrar; // for custom_derive pub mod deriving; +pub mod proc_macro_impl; + use std::rc::Rc; use syntax::ast; use syntax::ext::base::{MacroExpanderFn, NormalTT, MultiModifier, NamedSyntaxExtension}; diff --git a/src/libsyntax_ext/proc_macro_impl.rs b/src/libsyntax_ext/proc_macro_impl.rs new file mode 100644 index 00000000000..b454628acb1 --- /dev/null +++ b/src/libsyntax_ext/proc_macro_impl.rs @@ -0,0 +1,58 @@ +// Copyright 2016 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use std::panic; + +use errors::FatalError; + +use syntax::codemap::Span; +use syntax::ext::base::*; +use syntax::tokenstream::TokenStream; +use syntax::ext::base; + +use proc_macro::TokenStream as TsShim; +use proc_macro::__internal; + +pub struct AttrProcMacro { + pub inner: fn(TsShim, TsShim) -> TsShim, +} + +impl base::AttrProcMacro for AttrProcMacro { + fn expand<'cx>(&self, + ecx: &'cx mut ExtCtxt, + span: Span, + annotation: TokenStream, + annotated: TokenStream) + -> TokenStream { + let annotation = __internal::token_stream_wrap(annotation); + let annotated = __internal::token_stream_wrap(annotated); + + let res = __internal::set_parse_sess(&ecx.parse_sess, || { + panic::catch_unwind(panic::AssertUnwindSafe(|| (self.inner)(annotation, annotated))) + }); + + match res { + Ok(stream) => __internal::token_stream_inner(stream), + Err(e) => { + let msg = "custom attribute panicked"; + let mut err = ecx.struct_span_fatal(span, msg); + if let Some(s) = e.downcast_ref::() { + err.help(&format!("message: {}", s)); + } + if let Some(s) = e.downcast_ref::<&'static str>() { + err.help(&format!("message: {}", s)); + } + + err.emit(); + panic!(FatalError); + } + } + } +} diff --git a/src/libsyntax_ext/proc_macro_registrar.rs b/src/libsyntax_ext/proc_macro_registrar.rs index c93e2c054d2..c8af16e9242 100644 --- a/src/libsyntax_ext/proc_macro_registrar.rs +++ b/src/libsyntax_ext/proc_macro_registrar.rs @@ -11,18 +11,20 @@ use std::mem; use errors; + use syntax::ast::{self, Ident, NodeId}; use syntax::codemap::{ExpnInfo, NameAndSpan, MacroAttribute}; use syntax::ext::base::ExtCtxt; use syntax::ext::build::AstBuilder; use syntax::ext::expand::ExpansionConfig; -use syntax::parse::ParseSess; use syntax::fold::Folder; +use syntax::parse::ParseSess; use syntax::ptr::P; use syntax::symbol::Symbol; -use syntax_pos::{Span, DUMMY_SP}; use syntax::visit::{self, Visitor}; +use syntax_pos::{Span, DUMMY_SP}; + use deriving; struct CustomDerive { @@ -32,8 +34,14 @@ struct CustomDerive { attrs: Vec, } -struct CollectCustomDerives<'a> { +struct AttrProcMacro { + function_name: Ident, + span: Span, +} + +struct CollectProcMacros<'a> { derives: Vec, + attr_macros: Vec, in_root: bool, handler: &'a errors::Handler, is_proc_macro_crate: bool, @@ -50,16 +58,17 @@ pub fn modify(sess: &ParseSess, let ecfg = ExpansionConfig::default("proc_macro".to_string()); let mut cx = ExtCtxt::new(sess, ecfg, resolver); - let derives = { - let mut collect = CollectCustomDerives { + let (derives, attr_macros) = { + let mut collect = CollectProcMacros { derives: Vec::new(), + attr_macros: Vec::new(), in_root: true, handler: handler, is_proc_macro_crate: is_proc_macro_crate, is_test_crate: is_test_crate, }; visit::walk_crate(&mut collect, &krate); - collect.derives + (collect.derives, collect.attr_macros) }; if !is_proc_macro_crate { @@ -74,7 +83,7 @@ pub fn modify(sess: &ParseSess, return krate; } - krate.module.items.push(mk_registrar(&mut cx, &derives)); + krate.module.items.push(mk_registrar(&mut cx, &derives, &attr_macros)); if krate.exported_macros.len() > 0 { handler.err("cannot export macro_rules! macros from a `proc-macro` \ @@ -84,7 +93,7 @@ pub fn modify(sess: &ParseSess, return krate } -impl<'a> CollectCustomDerives<'a> { +impl<'a> CollectProcMacros<'a> { fn check_not_pub_in_root(&self, vis: &ast::Visibility, sp: Span) { if self.is_proc_macro_crate && self.in_root && @@ -92,61 +101,11 @@ impl<'a> CollectCustomDerives<'a> { self.handler.span_err(sp, "`proc-macro` crate types cannot \ export any items other than functions \ - tagged with `#[proc_macro_derive]` \ - currently"); + tagged with `#[proc_macro_derive]` currently"); } } -} - -impl<'a> Visitor<'a> for CollectCustomDerives<'a> { - fn visit_item(&mut self, item: &'a ast::Item) { - let mut attrs = item.attrs.iter().filter(|a| a.check_name("proc_macro_derive")); - - // First up, make sure we're checking a bare function. If we're not then - // we're just not interested in this item. - // - // If we find one, try to locate a `#[proc_macro_derive]` attribute on - // it. - match item.node { - ast::ItemKind::Fn(..) => {} - _ => { - // Check for invalid use of proc_macro_derive - if let Some(attr) = attrs.next() { - self.handler.span_err(attr.span(), - "the `#[proc_macro_derive]` \ - attribute may only be used \ - on bare functions"); - return; - } - self.check_not_pub_in_root(&item.vis, item.span); - return visit::walk_item(self, item) - } - } - - let attr = match attrs.next() { - Some(attr) => attr, - None => { - self.check_not_pub_in_root(&item.vis, item.span); - return visit::walk_item(self, item) - } - }; - - if let Some(a) = attrs.next() { - self.handler.span_err(a.span(), "multiple `#[proc_macro_derive]` \ - attributes found"); - } - - if self.is_test_crate { - return; - } - - if !self.is_proc_macro_crate { - self.handler.span_err(attr.span(), - "the `#[proc_macro_derive]` attribute is \ - only usable with crates of the `proc-macro` \ - crate type"); - } + fn collect_custom_derive(&mut self, item: &'a ast::Item, attr: &'a ast::Attribute) { // Once we've located the `#[proc_macro_derive]` attribute, verify // that it's of the form `#[proc_macro_derive(Foo)]` or // `#[proc_macro_derive(Foo, attributes(A, ..))]` @@ -232,6 +191,101 @@ impl<'a> Visitor<'a> for CollectCustomDerives<'a> { }; self.handler.span_err(item.span, msg); } + } + + fn collect_attr_proc_macro(&mut self, item: &'a ast::Item, attr: &'a ast::Attribute) { + if let Some(_) = attr.meta_item_list() { + self.handler.span_err(attr.span, "`#[proc_macro_attribute]` attribute + cannot contain any meta items"); + return; + } + + if self.in_root && item.vis == ast::Visibility::Public { + self.attr_macros.push(AttrProcMacro { + span: item.span, + function_name: item.ident, + }); + } else { + let msg = if !self.in_root { + "functions tagged with `#[proc_macro_attribute]` must \ + currently reside in the root of the crate" + } else { + "functions tagged with `#[proc_macro_attribute]` must be `pub`" + }; + self.handler.span_err(item.span, msg); + } + } +} + +impl<'a> Visitor<'a> for CollectProcMacros<'a> { + fn visit_item(&mut self, item: &'a ast::Item) { + // First up, make sure we're checking a bare function. If we're not then + // we're just not interested in this item. + // + // If we find one, try to locate a `#[proc_macro_derive]` attribute on + // it. + let is_fn = match item.node { + ast::ItemKind::Fn(..) => true, + _ => false, + }; + + let mut found_attr: Option<&'a ast::Attribute> = None; + + for attr in &item.attrs { + if attr.check_name("proc_macro_derive") || attr.check_name("proc_macro_attribute") { + if let Some(prev_attr) = found_attr { + let msg = if attr.name() == prev_attr.name() { + format!("Only one `#[{}]` attribute is allowed on any given function", + attr.name()) + } else { + format!("`#[{}]` and `#[{}]` attributes cannot both be applied \ + to the same function", attr.name(), prev_attr.name()) + }; + + self.handler.struct_span_err(attr.span(), &msg) + .span_note(prev_attr.span(), "Previous attribute here") + .emit(); + + return; + } + + found_attr = Some(attr); + } + } + + let attr = match found_attr { + None => { + self.check_not_pub_in_root(&item.vis, item.span); + return visit::walk_item(self, item); + }, + Some(attr) => attr, + }; + + if !is_fn { + let msg = format!("the `#[{}]` attribute may only be used on bare functions", + attr.name()); + + self.handler.span_err(attr.span(), &msg); + return; + } + + if self.is_test_crate { + return; + } + + if !self.is_proc_macro_crate { + let msg = format!("the `#[{}]` attribute is only usable with crates of the \ + `proc-macro` crate type", attr.name()); + + self.handler.span_err(attr.span(), &msg); + return; + } + + if attr.check_name("proc_macro_derive") { + self.collect_custom_derive(item, attr); + } else if attr.check_name("proc_macro_attribute") { + self.collect_attr_proc_macro(item, attr); + }; visit::walk_item(self, item); } @@ -265,7 +319,8 @@ impl<'a> Visitor<'a> for CollectCustomDerives<'a> { // } // } fn mk_registrar(cx: &mut ExtCtxt, - custom_derives: &[CustomDerive]) -> P { + custom_derives: &[CustomDerive], + custom_attrs: &[AttrProcMacro]) -> P { let eid = cx.codemap().record_expansion(ExpnInfo { call_site: DUMMY_SP, callee: NameAndSpan { @@ -286,25 +341,36 @@ fn mk_registrar(cx: &mut ExtCtxt, let registry = Ident::from_str("Registry"); let registrar = Ident::from_str("registrar"); let register_custom_derive = Ident::from_str("register_custom_derive"); - let stmts = custom_derives.iter().map(|cd| { + let register_attr_proc_macro = Ident::from_str("register_attr_proc_macro"); + + let mut stmts = custom_derives.iter().map(|cd| { let path = cx.path_global(cd.span, vec![cd.function_name]); let trait_name = cx.expr_str(cd.span, cd.trait_name); let attrs = cx.expr_vec_slice( span, cd.attrs.iter().map(|&s| cx.expr_str(cd.span, s)).collect::>() ); - (path, trait_name, attrs) - }).map(|(path, trait_name, attrs)| { let registrar = cx.expr_ident(span, registrar); let ufcs_path = cx.path(span, vec![proc_macro, __internal, registry, register_custom_derive]); - cx.expr_call(span, - cx.expr_path(ufcs_path), - vec![registrar, trait_name, cx.expr_path(path), attrs]) - }).map(|expr| { - cx.stmt_expr(expr) + + cx.stmt_expr(cx.expr_call(span, cx.expr_path(ufcs_path), + vec![registrar, trait_name, cx.expr_path(path), attrs])) + }).collect::>(); + stmts.extend(custom_attrs.iter().map(|ca| { + let name = cx.expr_str(ca.span, ca.function_name.name); + let path = cx.path_global(ca.span, vec![ca.function_name]); + let registrar = cx.expr_ident(ca.span, registrar); + + let ufcs_path = cx.path(span, + vec![proc_macro, __internal, registry, register_attr_proc_macro]); + + cx.stmt_expr(cx.expr_call(span, cx.expr_path(ufcs_path), + vec![registrar, name, cx.expr_path(path)])) + })); + let path = cx.path(span, vec![proc_macro, __internal, registry]); let registrar_path = cx.ty_path(path); let arg_ty = cx.ty_rptr(span, registrar_path, None, ast::Mutability::Mutable); diff --git a/src/test/compile-fail-fulldeps/proc-macro/auxiliary/attr_proc_macro.rs b/src/test/compile-fail-fulldeps/proc-macro/auxiliary/attr_proc_macro.rs new file mode 100644 index 00000000000..db0c19e96f8 --- /dev/null +++ b/src/test/compile-fail-fulldeps/proc-macro/auxiliary/attr_proc_macro.rs @@ -0,0 +1,23 @@ +// Copyright 2016 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// force-host +// no-prefer-dynamic +#![feature(proc_macro)] +#![crate_type = "proc-macro"] + +extern crate proc_macro; + +use proc_macro::TokenStream; + +#[proc_macro_attribute] +pub fn attr_proc_macro(_: TokenStream, input: TokenStream) -> TokenStream { + input +} diff --git a/src/test/compile-fail-fulldeps/proc-macro/feature-gate-proc_macro.rs b/src/test/compile-fail-fulldeps/proc-macro/feature-gate-proc_macro.rs new file mode 100644 index 00000000000..7e32800e0f9 --- /dev/null +++ b/src/test/compile-fail-fulldeps/proc-macro/feature-gate-proc_macro.rs @@ -0,0 +1,24 @@ +// Copyright 2016 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// aux-build:attr_proc_macro.rs +// gate-test-proc_macro +#![feature(use_extern_macros)] + +extern crate attr_proc_macro; +use attr_proc_macro::attr_proc_macro; + +#[attr_proc_macro] +//~^ ERROR: attribute procedural macros are experimental +struct Foo; + +fn main() { + let _ = Foo; +} \ No newline at end of file diff --git a/src/test/compile-fail-fulldeps/proc-macro/macro-use-attr.rs b/src/test/compile-fail-fulldeps/proc-macro/macro-use-attr.rs new file mode 100644 index 00000000000..76253487b51 --- /dev/null +++ b/src/test/compile-fail-fulldeps/proc-macro/macro-use-attr.rs @@ -0,0 +1,22 @@ +// Copyright 2016 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// aux-build:attr_proc_macro.rs +#![feature(proc_macro)] + +#[macro_use] extern crate attr_proc_macro; + +#[attr_proc_macro] +//~^ ERROR: attribute procedural macros cannot be imported with `#[macro_use]` +struct Foo; + +fn main() { + let _ = Foo; +} diff --git a/src/test/compile-fail-fulldeps/proc-macro/proc-macro-custom-attr-mutex.rs b/src/test/compile-fail-fulldeps/proc-macro/proc-macro-custom-attr-mutex.rs new file mode 100644 index 00000000000..288cab71ff4 --- /dev/null +++ b/src/test/compile-fail-fulldeps/proc-macro/proc-macro-custom-attr-mutex.rs @@ -0,0 +1,24 @@ +// Copyright 2016 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// aux-build:attr_proc_macro.rs + +#![feature(proc_macro, custom_attribute)] +//~^ ERROR Cannot use `#![feature(proc_macro)]` and `#![feature(custom_attribute)] at the same time + +extern crate attr_proc_macro; +use attr_proc_macro::attr_proc_macro; + +#[attr_proc_macro] +fn foo() {} + +fn main() { + foo(); +} diff --git a/src/test/run-pass-fulldeps/proc-macro/attr-args.rs b/src/test/run-pass-fulldeps/proc-macro/attr-args.rs new file mode 100644 index 00000000000..d28d75d81a2 --- /dev/null +++ b/src/test/run-pass-fulldeps/proc-macro/attr-args.rs @@ -0,0 +1,24 @@ +// Copyright 2016 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// aux-build:attr-args.rs + +#![allow(warnings)] +#![feature(proc_macro)] + +extern crate attr_args; +use attr_args::attr_with_args; + +#[attr_with_args(text = "Hello, world!")] +fn foo() {} + +fn main() { + assert_eq!(foo(), "Hello, world!"); +} diff --git a/src/test/run-pass-fulldeps/proc-macro/auxiliary/attr-args.rs b/src/test/run-pass-fulldeps/proc-macro/auxiliary/attr-args.rs new file mode 100644 index 00000000000..6e1eb395a0a --- /dev/null +++ b/src/test/run-pass-fulldeps/proc-macro/auxiliary/attr-args.rs @@ -0,0 +1,32 @@ +// Copyright 2016 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// no-prefer-dynamic +#![feature(proc_macro)] +#![crate_type = "proc-macro"] + +extern crate proc_macro; + +use proc_macro::TokenStream; + +#[proc_macro_attribute] +pub fn attr_with_args(args: TokenStream, input: TokenStream) -> TokenStream { + let args = args.to_string(); + + assert_eq!(args, r#"( text = "Hello, world!" )"#); + + let input = input.to_string(); + + assert_eq!(input, "fn foo ( ) { }"); + + r#" + fn foo() -> &'static str { "Hello, world!" } + "#.parse().unwrap() +} diff --git a/src/tools/tidy/src/features.rs b/src/tools/tidy/src/features.rs index b0da6647eb9..9df3e254943 100644 --- a/src/tools/tidy/src/features.rs +++ b/src/tools/tidy/src/features.rs @@ -172,7 +172,7 @@ pub fn check(path: &Path, bad: &mut bool) { "use_extern_macros", "staged_api", "const_indexing", "unboxed_closures", "stmt_expr_attributes", "cfg_target_thread_local", "unwind_attributes", - "inclusive_range_syntax" + "inclusive_range_syntax", "proc_macro" ]; // Only check the number of lang features. -- cgit 1.4.1-3-g733a5 From 68439614bad206570f0958fc672a3e11cccebf53 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 16 Jan 2017 17:59:11 -0800 Subject: Fix `feature_gate::find_lang_feature_issue()` to not use `unwrap()` --- src/libsyntax/feature_gate.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'src/libsyntax') diff --git a/src/libsyntax/feature_gate.rs b/src/libsyntax/feature_gate.rs index 2478ed169cd..66c66ad8940 100644 --- a/src/libsyntax/feature_gate.rs +++ b/src/libsyntax/feature_gate.rs @@ -896,9 +896,10 @@ fn find_lang_feature_issue(feature: &str) -> Option { issue } else { // search in Accepted or Removed features - ACCEPTED_FEATURES.iter().chain(REMOVED_FEATURES.iter()) - .find(|t| t.0 == feature) - .unwrap().2 + match ACCEPTED_FEATURES.iter().chain(REMOVED_FEATURES).find(|t| t.0 == feature) { + Some(&(_, _, issue)) => issue, + None => panic!("Feature `{}` is not declared anywhere", feature), + } } } -- cgit 1.4.1-3-g733a5 From 6466f55ebca18e3795800d8d606622d36f6ee763 Mon Sep 17 00:00:00 2001 From: Jeffrey Seyfried Date: Tue, 17 Jan 2017 01:14:53 +0000 Subject: Give the `StringReader` a `sess: &ParseSess`. --- src/librustc/hir/print.rs | 13 ++--- src/librustc_driver/pretty.rs | 8 +-- src/librustc_save_analysis/span_utils.rs | 3 +- src/libsyntax/ext/expand.rs | 2 +- src/libsyntax/parse/lexer/comments.rs | 9 +-- src/libsyntax/parse/lexer/mod.rs | 92 +++++++++++++++--------------- src/libsyntax/parse/lexer/unicode_chars.rs | 6 +- src/libsyntax/parse/mod.rs | 2 +- src/libsyntax/print/pprust.rs | 21 ++----- 9 files changed, 68 insertions(+), 88 deletions(-) (limited to 'src/libsyntax') diff --git a/src/librustc/hir/print.rs b/src/librustc/hir/print.rs index c06c53810d7..c0818f05286 100644 --- a/src/librustc/hir/print.rs +++ b/src/librustc/hir/print.rs @@ -13,6 +13,7 @@ pub use self::AnnNode::*; use syntax::abi::Abi; use syntax::ast; use syntax::codemap::{CodeMap, Spanned}; +use syntax::parse::ParseSess; use syntax::parse::lexer::comments; use syntax::print::pp::{self, break_offset, word, space, hardbreak}; use syntax::print::pp::{Breaks, eof}; @@ -21,7 +22,6 @@ use syntax::print::pprust::{self as ast_pp, PrintState}; use syntax::ptr::P; use syntax::symbol::keywords; use syntax_pos::{self, BytePos}; -use errors; use hir; use hir::{PatKind, RegionTyParamBound, TraitTyParamBound, TraitBoundModifier}; @@ -116,7 +116,7 @@ pub const default_columns: usize = 78; /// it can scan the input text for comments and literals to /// copy forward. pub fn print_crate<'a>(cm: &'a CodeMap, - span_diagnostic: &errors::Handler, + sess: &ParseSess, krate: &hir::Crate, filename: String, input: &mut Read, @@ -124,8 +124,7 @@ pub fn print_crate<'a>(cm: &'a CodeMap, ann: &'a PpAnn, is_expanded: bool) -> io::Result<()> { - let mut s = State::new_from_input(cm, span_diagnostic, filename, input, - out, ann, is_expanded); + let mut s = State::new_from_input(cm, sess, filename, input, out, ann, is_expanded); // When printing the AST, we sometimes need to inject `#[no_std]` here. // Since you can't compile the HIR, it's not necessary. @@ -137,16 +136,14 @@ pub fn print_crate<'a>(cm: &'a CodeMap, impl<'a> State<'a> { pub fn new_from_input(cm: &'a CodeMap, - span_diagnostic: &errors::Handler, + sess: &ParseSess, filename: String, input: &mut Read, out: Box, ann: &'a PpAnn, is_expanded: bool) -> State<'a> { - let (cmnts, lits) = comments::gather_comments_and_literals(span_diagnostic, - filename, - input); + let (cmnts, lits) = comments::gather_comments_and_literals(sess, filename, input); State::new(cm, out, diff --git a/src/librustc_driver/pretty.rs b/src/librustc_driver/pretty.rs index afacfb6e3f9..3c8a529bdae 100644 --- a/src/librustc_driver/pretty.rs +++ b/src/librustc_driver/pretty.rs @@ -838,7 +838,7 @@ pub fn print_after_parsing(sess: &Session, debug!("pretty printing source code {:?}", s); let sess = annotation.sess(); pprust::print_crate(sess.codemap(), - sess.diagnostic(), + &sess.parse_sess, krate, src_name.to_string(), &mut rdr, @@ -896,7 +896,7 @@ pub fn print_after_hir_lowering<'tcx, 'a: 'tcx>(sess: &'a Session, debug!("pretty printing source code {:?}", s); let sess = annotation.sess(); pprust::print_crate(sess.codemap(), - sess.diagnostic(), + &sess.parse_sess, krate, src_name.to_string(), &mut rdr, @@ -920,7 +920,7 @@ pub fn print_after_hir_lowering<'tcx, 'a: 'tcx>(sess: &'a Session, debug!("pretty printing source code {:?}", s); let sess = annotation.sess(); pprust_hir::print_crate(sess.codemap(), - sess.diagnostic(), + &sess.parse_sess, krate, src_name.to_string(), &mut rdr, @@ -945,7 +945,7 @@ pub fn print_after_hir_lowering<'tcx, 'a: 'tcx>(sess: &'a Session, let sess = annotation.sess(); let ast_map = annotation.ast_map().expect("--unpretty missing HIR map"); let mut pp_state = pprust_hir::State::new_from_input(sess.codemap(), - sess.diagnostic(), + &sess.parse_sess, src_name.to_string(), &mut rdr, box out, diff --git a/src/librustc_save_analysis/span_utils.rs b/src/librustc_save_analysis/span_utils.rs index 448bb2e7617..ebfea90527c 100644 --- a/src/librustc_save_analysis/span_utils.rs +++ b/src/librustc_save_analysis/span_utils.rs @@ -85,8 +85,7 @@ impl<'a> SpanUtils<'a> { let filemap = self.sess .codemap() .new_filemap(String::from(""), None, self.snippet(span)); - let s = self.sess; - lexer::StringReader::new(s.diagnostic(), filemap) + lexer::StringReader::new(&self.sess.parse_sess, filemap) } fn span_to_tts(&self, span: Span) -> Vec { diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index 201e8d69494..4b9b6518b48 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -648,7 +648,7 @@ fn string_to_tts(text: String, parse_sess: &ParseSess) -> Vec { let filemap = parse_sess.codemap() .new_filemap(String::from(""), None, text); - let lexer = lexer::StringReader::new(&parse_sess.span_diagnostic, filemap); + let lexer = lexer::StringReader::new(parse_sess, filemap); let mut parser = Parser::new(parse_sess, Box::new(lexer), None, false); panictry!(parser.parse_all_token_trees()) } diff --git a/src/libsyntax/parse/lexer/comments.rs b/src/libsyntax/parse/lexer/comments.rs index ba83a55ea79..8c94cf67bf6 100644 --- a/src/libsyntax/parse/lexer/comments.rs +++ b/src/libsyntax/parse/lexer/comments.rs @@ -13,11 +13,10 @@ pub use self::CommentStyle::*; use ast; use codemap::CodeMap; use syntax_pos::{BytePos, CharPos, Pos}; -use errors; use parse::lexer::is_block_doc_comment; use parse::lexer::{StringReader, TokenAndSpan}; use parse::lexer::{is_pattern_whitespace, Reader}; -use parse::lexer; +use parse::{lexer, ParseSess}; use print::pprust; use str::char_at; @@ -346,16 +345,14 @@ pub struct Literal { // it appears this function is called only from pprust... that's // probably not a good thing. -pub fn gather_comments_and_literals(span_diagnostic: &errors::Handler, - path: String, - srdr: &mut Read) +pub fn gather_comments_and_literals(sess: &ParseSess, path: String, srdr: &mut Read) -> (Vec, Vec) { let mut src = Vec::new(); srdr.read_to_end(&mut src).unwrap(); let src = String::from_utf8(src).unwrap(); let cm = CodeMap::new(); let filemap = cm.new_filemap(path, None, src); - let mut rdr = lexer::StringReader::new_raw(span_diagnostic, filemap); + let mut rdr = lexer::StringReader::new_raw(sess, filemap); let mut comments: Vec = Vec::new(); let mut literals: Vec = Vec::new(); diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs index 818742e4492..f1cb81a4c7d 100644 --- a/src/libsyntax/parse/lexer/mod.rs +++ b/src/libsyntax/parse/lexer/mod.rs @@ -11,9 +11,9 @@ use ast::{self, Ident}; use syntax_pos::{self, BytePos, CharPos, Pos, Span}; use codemap::CodeMap; -use errors::{FatalError, Handler, DiagnosticBuilder}; +use errors::{FatalError, DiagnosticBuilder}; use ext::tt::transcribe::tt_next_token; -use parse::token; +use parse::{token, ParseSess}; use str::char_at; use symbol::{Symbol, keywords}; use std_unicode::property::Pattern_White_Space; @@ -82,7 +82,7 @@ impl Default for TokenAndSpan { } pub struct StringReader<'a> { - pub span_diagnostic: &'a Handler, + pub sess: &'a ParseSess, /// The absolute offset within the codemap of the next character to read pub next_pos: BytePos, /// The absolute offset within the codemap of the current character @@ -181,27 +181,22 @@ impl<'a> Reader for TtReader<'a> { impl<'a> StringReader<'a> { /// For comments.rs, which hackily pokes into next_pos and ch - pub fn new_raw<'b>(span_diagnostic: &'b Handler, - filemap: Rc) - -> StringReader<'b> { - let mut sr = StringReader::new_raw_internal(span_diagnostic, filemap); + pub fn new_raw<'b>(sess: &'a ParseSess, filemap: Rc) -> Self { + let mut sr = StringReader::new_raw_internal(sess, filemap); sr.bump(); sr } - fn new_raw_internal<'b>(span_diagnostic: &'b Handler, - filemap: Rc) - -> StringReader<'b> { + fn new_raw_internal(sess: &'a ParseSess, filemap: Rc) -> Self { if filemap.src.is_none() { - span_diagnostic.bug(&format!("Cannot lex filemap \ - without source: {}", - filemap.name)[..]); + sess.span_diagnostic.bug(&format!("Cannot lex filemap without source: {}", + filemap.name)); } let source_text = (*filemap.src.as_ref().unwrap()).clone(); StringReader { - span_diagnostic: span_diagnostic, + sess: sess, next_pos: filemap.start_pos, pos: filemap.start_pos, col: CharPos(0), @@ -217,10 +212,8 @@ impl<'a> StringReader<'a> { } } - pub fn new<'b>(span_diagnostic: &'b Handler, - filemap: Rc) - -> StringReader<'b> { - let mut sr = StringReader::new_raw(span_diagnostic, filemap); + pub fn new(sess: &'a ParseSess, filemap: Rc) -> Self { + let mut sr = StringReader::new_raw(sess, filemap); if let Err(_) = sr.advance_token() { sr.emit_fatal_errors(); panic!(FatalError); @@ -234,12 +227,12 @@ impl<'a> StringReader<'a> { /// Report a fatal lexical error with a given span. pub fn fatal_span(&self, sp: Span, m: &str) -> FatalError { - self.span_diagnostic.span_fatal(sp, m) + self.sess.span_diagnostic.span_fatal(sp, m) } /// Report a lexical error with a given span. pub fn err_span(&self, sp: Span, m: &str) { - self.span_diagnostic.span_err(sp, m) + self.sess.span_diagnostic.span_err(sp, m) } @@ -274,7 +267,7 @@ impl<'a> StringReader<'a> { for c in c.escape_default() { m.push(c) } - self.span_diagnostic.struct_span_fatal(syntax_pos::mk_sp(from_pos, to_pos), &m[..]) + self.sess.span_diagnostic.struct_span_fatal(syntax_pos::mk_sp(from_pos, to_pos), &m[..]) } /// Report a lexical error spanning [`from_pos`, `to_pos`), appending an @@ -298,7 +291,7 @@ impl<'a> StringReader<'a> { for c in c.escape_default() { m.push(c) } - self.span_diagnostic.struct_span_err(syntax_pos::mk_sp(from_pos, to_pos), &m[..]) + self.sess.span_diagnostic.struct_span_err(syntax_pos::mk_sp(from_pos, to_pos), &m[..]) } /// Report a lexical error spanning [`from_pos`, `to_pos`), appending the @@ -503,9 +496,8 @@ impl<'a> StringReader<'a> { fn scan_comment(&mut self) -> Option { if let Some(c) = self.ch { if c.is_whitespace() { - self.span_diagnostic.span_err(syntax_pos::mk_sp(self.pos, self.pos), - "called consume_any_line_comment, but there \ - was whitespace"); + let msg = "called consume_any_line_comment, but there was whitespace"; + self.sess.span_diagnostic.span_err(syntax_pos::mk_sp(self.pos, self.pos), msg); } } @@ -875,7 +867,7 @@ impl<'a> StringReader<'a> { self.scan_unicode_escape(delim) && !ascii_only } else { let span = syntax_pos::mk_sp(start, self.pos); - self.span_diagnostic + self.sess.span_diagnostic .struct_span_err(span, "incorrect unicode escape sequence") .span_help(span, "format of unicode escape sequences is \ @@ -1701,35 +1693,41 @@ fn ident_continue(c: Option) -> bool { mod tests { use super::*; - use ast::Ident; + use ast::{Ident, CrateConfig}; use symbol::Symbol; use syntax_pos::{BytePos, Span, NO_EXPANSION}; use codemap::CodeMap; use errors; + use feature_gate::UnstableFeatures; use parse::token; + use std::cell::RefCell; use std::io; use std::rc::Rc; - fn mk_sh(cm: Rc) -> errors::Handler { - // FIXME (#22405): Replace `Box::new` with `box` here when/if possible. - let emitter = errors::emitter::EmitterWriter::new(Box::new(io::sink()), - Some(cm)); - errors::Handler::with_emitter(true, false, Box::new(emitter)) + fn mk_sess(cm: Rc) -> ParseSess { + let emitter = errors::emitter::EmitterWriter::new(Box::new(io::sink()), Some(cm.clone())); + ParseSess { + span_diagnostic: errors::Handler::with_emitter(true, false, Box::new(emitter)), + unstable_features: UnstableFeatures::from_environment(), + config: CrateConfig::new(), + included_mod_stack: RefCell::new(Vec::new()), + code_map: cm, + } } // open a string reader for the given string fn setup<'a>(cm: &CodeMap, - span_handler: &'a errors::Handler, + sess: &'a ParseSess, teststr: String) -> StringReader<'a> { let fm = cm.new_filemap("zebra.rs".to_string(), None, teststr); - StringReader::new(span_handler, fm) + StringReader::new(sess, fm) } #[test] fn t1() { let cm = Rc::new(CodeMap::new()); - let sh = mk_sh(cm.clone()); + let sh = mk_sess(cm.clone()); let mut string_reader = setup(&cm, &sh, "/* my source file */ fn main() { println!(\"zebra\"); }\n" @@ -1781,7 +1779,7 @@ mod tests { #[test] fn doublecolonparsing() { let cm = Rc::new(CodeMap::new()); - let sh = mk_sh(cm.clone()); + let sh = mk_sess(cm.clone()); check_tokenization(setup(&cm, &sh, "a b".to_string()), vec![mk_ident("a"), token::Whitespace, mk_ident("b")]); } @@ -1789,7 +1787,7 @@ mod tests { #[test] fn dcparsing_2() { let cm = Rc::new(CodeMap::new()); - let sh = mk_sh(cm.clone()); + let sh = mk_sess(cm.clone()); check_tokenization(setup(&cm, &sh, "a::b".to_string()), vec![mk_ident("a"), token::ModSep, mk_ident("b")]); } @@ -1797,7 +1795,7 @@ mod tests { #[test] fn dcparsing_3() { let cm = Rc::new(CodeMap::new()); - let sh = mk_sh(cm.clone()); + let sh = mk_sess(cm.clone()); check_tokenization(setup(&cm, &sh, "a ::b".to_string()), vec![mk_ident("a"), token::Whitespace, token::ModSep, mk_ident("b")]); } @@ -1805,7 +1803,7 @@ mod tests { #[test] fn dcparsing_4() { let cm = Rc::new(CodeMap::new()); - let sh = mk_sh(cm.clone()); + let sh = mk_sess(cm.clone()); check_tokenization(setup(&cm, &sh, "a:: b".to_string()), vec![mk_ident("a"), token::ModSep, token::Whitespace, mk_ident("b")]); } @@ -1813,7 +1811,7 @@ mod tests { #[test] fn character_a() { let cm = Rc::new(CodeMap::new()); - let sh = mk_sh(cm.clone()); + let sh = mk_sess(cm.clone()); assert_eq!(setup(&cm, &sh, "'a'".to_string()).next_token().tok, token::Literal(token::Char(Symbol::intern("a")), None)); } @@ -1821,7 +1819,7 @@ mod tests { #[test] fn character_space() { let cm = Rc::new(CodeMap::new()); - let sh = mk_sh(cm.clone()); + let sh = mk_sess(cm.clone()); assert_eq!(setup(&cm, &sh, "' '".to_string()).next_token().tok, token::Literal(token::Char(Symbol::intern(" ")), None)); } @@ -1829,7 +1827,7 @@ mod tests { #[test] fn character_escaped() { let cm = Rc::new(CodeMap::new()); - let sh = mk_sh(cm.clone()); + let sh = mk_sess(cm.clone()); assert_eq!(setup(&cm, &sh, "'\\n'".to_string()).next_token().tok, token::Literal(token::Char(Symbol::intern("\\n")), None)); } @@ -1837,7 +1835,7 @@ mod tests { #[test] fn lifetime_name() { let cm = Rc::new(CodeMap::new()); - let sh = mk_sh(cm.clone()); + let sh = mk_sess(cm.clone()); assert_eq!(setup(&cm, &sh, "'abc".to_string()).next_token().tok, token::Lifetime(Ident::from_str("'abc"))); } @@ -1845,7 +1843,7 @@ mod tests { #[test] fn raw_string() { let cm = Rc::new(CodeMap::new()); - let sh = mk_sh(cm.clone()); + let sh = mk_sess(cm.clone()); assert_eq!(setup(&cm, &sh, "r###\"\"#a\\b\x00c\"\"###".to_string()) .next_token() .tok, @@ -1855,7 +1853,7 @@ mod tests { #[test] fn literal_suffixes() { let cm = Rc::new(CodeMap::new()); - let sh = mk_sh(cm.clone()); + let sh = mk_sess(cm.clone()); macro_rules! test { ($input: expr, $tok_type: ident, $tok_contents: expr) => {{ assert_eq!(setup(&cm, &sh, format!("{}suffix", $input)).next_token().tok, @@ -1899,7 +1897,7 @@ mod tests { #[test] fn nested_block_comments() { let cm = Rc::new(CodeMap::new()); - let sh = mk_sh(cm.clone()); + let sh = mk_sess(cm.clone()); let mut lexer = setup(&cm, &sh, "/* /* */ */'a'".to_string()); match lexer.next_token().tok { token::Comment => {} @@ -1912,7 +1910,7 @@ mod tests { #[test] fn crlf_comments() { let cm = Rc::new(CodeMap::new()); - let sh = mk_sh(cm.clone()); + let sh = mk_sess(cm.clone()); let mut lexer = setup(&cm, &sh, "// test\r\n/// test\r\n".to_string()); let comment = lexer.next_token(); assert_eq!(comment.tok, token::Comment); diff --git a/src/libsyntax/parse/lexer/unicode_chars.rs b/src/libsyntax/parse/lexer/unicode_chars.rs index 1e08b20b7e1..6da3e5de75c 100644 --- a/src/libsyntax/parse/lexer/unicode_chars.rs +++ b/src/libsyntax/parse/lexer/unicode_chars.rs @@ -243,10 +243,8 @@ pub fn check_for_substitution<'a>(reader: &StringReader<'a>, err.span_help(span, &msg); }, None => { - reader - .span_diagnostic - .span_bug_no_panic(span, - &format!("substitution character not found for '{}'", ch)); + let msg = format!("substitution character not found for '{}'", ch); + reader.sess.span_diagnostic.span_bug_no_panic(span, &msg); } } }); diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index 32b61a88ac1..74b313ba395 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -223,7 +223,7 @@ pub fn filemap_to_tts(sess: &ParseSess, filemap: Rc) -> Vec { // it appears to me that the cfg doesn't matter here... indeed, // parsing tt's probably shouldn't require a parser at all. - let srdr = lexer::StringReader::new(&sess.span_diagnostic, filemap); + let srdr = lexer::StringReader::new(sess, filemap); let mut p1 = Parser::new(sess, Box::new(srdr), None, false); panictry!(p1.parse_all_token_trees()) } diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs index ff77732f535..33b4636ce08 100644 --- a/src/libsyntax/print/pprust.rs +++ b/src/libsyntax/print/pprust.rs @@ -18,10 +18,9 @@ use util::parser::AssocOp; use attr; use codemap::{self, CodeMap}; use syntax_pos::{self, BytePos}; -use errors; use parse::token::{self, BinOpToken, Token}; use parse::lexer::comments; -use parse; +use parse::{self, ParseSess}; use print::pp::{self, break_offset, word, space, zerobreak, hardbreak}; use print::pp::{Breaks, eof}; use print::pp::Breaks::{Consistent, Inconsistent}; @@ -101,20 +100,15 @@ pub const DEFAULT_COLUMNS: usize = 78; /// it can scan the input text for comments and literals to /// copy forward. pub fn print_crate<'a>(cm: &'a CodeMap, - span_diagnostic: &errors::Handler, + sess: &ParseSess, krate: &ast::Crate, filename: String, input: &mut Read, out: Box, ann: &'a PpAnn, is_expanded: bool) -> io::Result<()> { - let mut s = State::new_from_input(cm, - span_diagnostic, - filename, - input, - out, - ann, - is_expanded); + let mut s = State::new_from_input(cm, sess, filename, input, out, ann, is_expanded); + if is_expanded && !std_inject::injected_crate_name(krate).is_none() { // We need to print `#![no_std]` (and its feature gate) so that // compiling pretty-printed source won't inject libstd again. @@ -140,16 +134,13 @@ pub fn print_crate<'a>(cm: &'a CodeMap, impl<'a> State<'a> { pub fn new_from_input(cm: &'a CodeMap, - span_diagnostic: &errors::Handler, + sess: &ParseSess, filename: String, input: &mut Read, out: Box, ann: &'a PpAnn, is_expanded: bool) -> State<'a> { - let (cmnts, lits) = comments::gather_comments_and_literals( - span_diagnostic, - filename, - input); + let (cmnts, lits) = comments::gather_comments_and_literals(sess, filename, input); State::new( cm, -- cgit 1.4.1-3-g733a5 From de46b247585999ae70674f1fa0543d62f2889c7f Mon Sep 17 00:00:00 2001 From: Jeffrey Seyfried Date: Thu, 12 Jan 2017 23:32:00 +0000 Subject: Introduce `string_reader.parse_all_token_trees()`. --- src/librustc_save_analysis/span_utils.rs | 10 +-- src/libsyntax/ext/expand.rs | 10 +-- src/libsyntax/parse/lexer/comments.rs | 6 +- src/libsyntax/parse/lexer/mod.rs | 44 +++++++++- src/libsyntax/parse/lexer/tokentrees.rs | 138 +++++++++++++++++++++++++++++++ src/libsyntax/parse/mod.rs | 11 +-- 6 files changed, 192 insertions(+), 27 deletions(-) create mode 100644 src/libsyntax/parse/lexer/tokentrees.rs (limited to 'src/libsyntax') diff --git a/src/librustc_save_analysis/span_utils.rs b/src/librustc_save_analysis/span_utils.rs index ebfea90527c..89525b27ed3 100644 --- a/src/librustc_save_analysis/span_utils.rs +++ b/src/librustc_save_analysis/span_utils.rs @@ -17,9 +17,9 @@ use std::env; use std::path::Path; use syntax::ast; -use syntax::parse::lexer::{self, Reader, StringReader}; +use syntax::parse::filemap_to_tts; +use syntax::parse::lexer::{self, StringReader}; use syntax::parse::token::{self, Token}; -use syntax::parse::parser::Parser; use syntax::symbol::keywords; use syntax::tokenstream::TokenTree; use syntax_pos::*; @@ -89,9 +89,9 @@ impl<'a> SpanUtils<'a> { } fn span_to_tts(&self, span: Span) -> Vec { - let srdr = self.retokenise_span(span); - let mut p = Parser::new(&self.sess.parse_sess, Box::new(srdr), None, false); - p.parse_all_token_trees().expect("Couldn't re-parse span") + let filename = String::from(""); + let filemap = self.sess.codemap().new_filemap(filename, None, self.snippet(span)); + filemap_to_tts(&self.sess.parse_sess, filemap) } // Re-parses a path and returns the span for the last identifier in the path diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index 4b9b6518b48..ca4b2caaf55 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -21,7 +21,7 @@ use ext::base::*; use feature_gate::{self, Features}; use fold; use fold::*; -use parse::{ParseSess, DirectoryOwnership, PResult, lexer}; +use parse::{ParseSess, DirectoryOwnership, PResult, filemap_to_tts}; use parse::parser::Parser; use parse::token; use print::pprust; @@ -645,12 +645,8 @@ fn tts_for_attr(attr: &ast::Attribute, parse_sess: &ParseSess) -> Vec } fn string_to_tts(text: String, parse_sess: &ParseSess) -> Vec { - let filemap = parse_sess.codemap() - .new_filemap(String::from(""), None, text); - - let lexer = lexer::StringReader::new(parse_sess, filemap); - let mut parser = Parser::new(parse_sess, Box::new(lexer), None, false); - panictry!(parser.parse_all_token_trees()) + let filename = String::from(""); + filemap_to_tts(parse_sess, parse_sess.codemap().new_filemap(filename, None, text)) } impl<'a, 'b> Folder for InvocationCollector<'a, 'b> { diff --git a/src/libsyntax/parse/lexer/comments.rs b/src/libsyntax/parse/lexer/comments.rs index 8c94cf67bf6..c97b8ddf919 100644 --- a/src/libsyntax/parse/lexer/comments.rs +++ b/src/libsyntax/parse/lexer/comments.rs @@ -13,10 +13,8 @@ pub use self::CommentStyle::*; use ast; use codemap::CodeMap; use syntax_pos::{BytePos, CharPos, Pos}; -use parse::lexer::is_block_doc_comment; -use parse::lexer::{StringReader, TokenAndSpan}; -use parse::lexer::{is_pattern_whitespace, Reader}; -use parse::{lexer, ParseSess}; +use parse::lexer::{is_block_doc_comment, is_pattern_whitespace}; +use parse::lexer::{self, ParseSess, StringReader, TokenAndSpan}; use print::pprust; use str::char_at; diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs index f1cb81a4c7d..6c6161998d7 100644 --- a/src/libsyntax/parse/lexer/mod.rs +++ b/src/libsyntax/parse/lexer/mod.rs @@ -26,6 +26,7 @@ use std::rc::Rc; pub use ext::tt::transcribe::{TtReader, new_tt_reader}; pub mod comments; +mod tokentrees; mod unicode_chars; pub trait Reader { @@ -105,9 +106,44 @@ pub struct StringReader<'a> { // cache a direct reference to the source text, so that we don't have to // retrieve it via `self.filemap.src.as_ref().unwrap()` all the time. source_text: Rc, + /// Stack of open delimiters and their spans. Used for error message. + token: token::Token, + span: Span, + open_braces: Vec<(token::DelimToken, Span)>, } -impl<'a> Reader for StringReader<'a> { +impl<'a> StringReader<'a> { + fn next_token(&mut self) -> TokenAndSpan where Self: Sized { + let res = self.try_next_token(); + self.unwrap_or_abort(res) + } + fn unwrap_or_abort(&mut self, res: Result) -> TokenAndSpan { + match res { + Ok(tok) => tok, + Err(_) => { + self.emit_fatal_errors(); + panic!(FatalError); + } + } + } + fn try_real_token(&mut self) -> Result { + let mut t = self.try_next_token()?; + loop { + match t.tok { + token::Whitespace | token::Comment | token::Shebang(_) => { + t = self.try_next_token()?; + } + _ => break, + } + } + self.token = t.tok.clone(); + self.span = t.sp; + Ok(t) + } + pub fn real_token(&mut self) -> TokenAndSpan { + let res = self.try_real_token(); + self.unwrap_or_abort(res) + } fn is_eof(&self) -> bool { if self.ch.is_none() { return true; @@ -131,9 +167,6 @@ impl<'a> Reader for StringReader<'a> { fn fatal(&self, m: &str) -> FatalError { self.fatal_span(self.peek_span, m) } - fn err(&self, m: &str) { - self.err_span(self.peek_span, m) - } fn emit_fatal_errors(&mut self) { for err in &mut self.fatal_errs { err.emit(); @@ -209,6 +242,9 @@ impl<'a> StringReader<'a> { peek_span: syntax_pos::DUMMY_SP, source_text: source_text, fatal_errs: Vec::new(), + token: token::Eof, + span: syntax_pos::DUMMY_SP, + open_braces: Vec::new(), } } diff --git a/src/libsyntax/parse/lexer/tokentrees.rs b/src/libsyntax/parse/lexer/tokentrees.rs new file mode 100644 index 00000000000..7b6f00e0e82 --- /dev/null +++ b/src/libsyntax/parse/lexer/tokentrees.rs @@ -0,0 +1,138 @@ +// Copyright 2016 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use print::pprust::token_to_string; +use parse::lexer::StringReader; +use parse::{token, PResult}; +use syntax_pos::Span; +use tokenstream::{Delimited, TokenTree}; + +use std::rc::Rc; + +impl<'a> StringReader<'a> { + // Parse a stream of tokens into a list of `TokenTree`s, up to an `Eof`. + pub fn parse_all_token_trees(&mut self) -> PResult<'a, Vec> { + let mut tts = Vec::new(); + while self.token != token::Eof { + tts.push(self.parse_token_tree()?); + } + Ok(tts) + } + + // Parse a stream of tokens into a list of `TokenTree`s, up to a `CloseDelim`. + fn parse_token_trees_until_close_delim(&mut self) -> Vec { + let mut tts = vec![]; + loop { + if let token::CloseDelim(..) = self.token { + return tts; + } + match self.parse_token_tree() { + Ok(tt) => tts.push(tt), + Err(mut e) => { + e.emit(); + return tts; + } + } + } + } + + fn parse_token_tree(&mut self) -> PResult<'a, TokenTree> { + match self.token { + token::Eof => { + let msg = "this file contains an un-closed delimiter"; + let mut err = self.sess.span_diagnostic.struct_span_err(self.span, msg); + for &(_, sp) in &self.open_braces { + err.span_help(sp, "did you mean to close this delimiter?"); + } + Err(err) + }, + token::OpenDelim(delim) => { + // The span for beginning of the delimited section + let pre_span = self.span; + + // Parse the open delimiter. + self.open_braces.push((delim, self.span)); + let open_span = self.span; + self.real_token(); + + // Parse the token trees within the delimiters. + // We stop at any delimiter so we can try to recover if the user + // uses an incorrect delimiter. + let tts = self.parse_token_trees_until_close_delim(); + + let close_span = self.span; + // Expand to cover the entire delimited token tree + let span = Span { hi: close_span.hi, ..pre_span }; + + match self.token { + // Correct delimiter. + token::CloseDelim(d) if d == delim => { + self.open_braces.pop().unwrap(); + + // Parse the close delimiter. + self.real_token(); + } + // Incorrect delimiter. + token::CloseDelim(other) => { + let token_str = token_to_string(&self.token); + let msg = format!("incorrect close delimiter: `{}`", token_str); + let mut err = self.sess.span_diagnostic.struct_span_err(self.span, &msg); + // This is a conservative error: only report the last unclosed delimiter. + // The previous unclosed delimiters could actually be closed! The parser + // just hasn't gotten to them yet. + if let Some(&(_, sp)) = self.open_braces.last() { + err.span_note(sp, "unclosed delimiter"); + }; + err.emit(); + + self.open_braces.pop().unwrap(); + + // If the incorrect delimiter matches an earlier opening + // delimiter, then don't consume it (it can be used to + // close the earlier one). Otherwise, consume it. + // E.g., we try to recover from: + // fn foo() { + // bar(baz( + // } // Incorrect delimiter but matches the earlier `{` + if !self.open_braces.iter().any(|&(b, _)| b == other) { + self.real_token(); + } + } + token::Eof => { + // Silently recover, the EOF token will be seen again + // and an error emitted then. Thus we don't pop from + // self.open_braces here. + }, + _ => {} + } + + Ok(TokenTree::Delimited(span, Rc::new(Delimited { + delim: delim, + open_span: open_span, + tts: tts, + close_span: close_span, + }))) + }, + token::CloseDelim(_) => { + // An unexpected closing delimiter (i.e., there is no + // matching opening delimiter). + let token_str = token_to_string(&self.token); + let msg = format!("unexpected close delimiter: `{}`", token_str); + let err = self.sess.span_diagnostic.struct_span_err(self.span, &msg); + Err(err) + }, + _ => { + let tt = TokenTree::Token(self.span, self.token.clone()); + self.real_token(); + Ok(tt) + } + } + } +} diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index 74b313ba395..500e8285b4c 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -219,13 +219,10 @@ fn file_to_filemap(sess: &ParseSess, path: &Path, spanopt: Option) } /// Given a filemap, produce a sequence of token-trees -pub fn filemap_to_tts(sess: &ParseSess, filemap: Rc) - -> Vec { - // it appears to me that the cfg doesn't matter here... indeed, - // parsing tt's probably shouldn't require a parser at all. - let srdr = lexer::StringReader::new(sess, filemap); - let mut p1 = Parser::new(sess, Box::new(srdr), None, false); - panictry!(p1.parse_all_token_trees()) +pub fn filemap_to_tts(sess: &ParseSess, filemap: Rc) -> Vec { + let mut srdr = lexer::StringReader::new(sess, filemap); + srdr.real_token(); + panictry!(srdr.parse_all_token_trees()) } /// Given tts and the ParseSess, produce a parser -- cgit 1.4.1-3-g733a5 From debcbf0b8e8fcf6f1d44e8f79cc06c0866d8d1dd Mon Sep 17 00:00:00 2001 From: Jeffrey Seyfried Date: Fri, 13 Jan 2017 04:49:20 +0000 Subject: Refactor the parser to consume token trees. --- src/librustc/session/config.rs | 3 +- src/librustc_metadata/cstore_impl.rs | 18 ++---- src/libsyntax/ext/base.rs | 4 +- src/libsyntax/ext/tt/macro_parser.rs | 7 +-- src/libsyntax/ext/tt/macro_rules.rs | 20 +++--- src/libsyntax/ext/tt/transcribe.rs | 11 ++-- src/libsyntax/parse/lexer/mod.rs | 74 ---------------------- src/libsyntax/parse/mod.rs | 5 +- src/libsyntax/parse/parser.rs | 115 ++++++----------------------------- src/libsyntax/tokenstream.rs | 13 ++-- src/test/parse-fail/issue-33569.rs | 2 +- 11 files changed, 59 insertions(+), 213 deletions(-) (limited to 'src/libsyntax') diff --git a/src/librustc/session/config.rs b/src/librustc/session/config.rs index 104c851e057..7d8f7fcefe6 100644 --- a/src/librustc/session/config.rs +++ b/src/librustc/session/config.rs @@ -25,6 +25,7 @@ use lint; use middle::cstore; use syntax::ast::{self, IntTy, UintTy}; +use syntax::parse::token; use syntax::parse; use syntax::symbol::Symbol; use syntax::feature_gate::UnstableFeatures; @@ -1259,7 +1260,7 @@ pub fn parse_cfgspecs(cfgspecs: Vec ) -> ast::CrateConfig { let meta_item = panictry!(parser.parse_meta_item()); - if !parser.reader.is_eof() { + if parser.token != token::Eof { early_error(ErrorOutputType::default(), &format!("invalid --cfg argument: {}", s)) } else if meta_item.is_meta_item_list() { let msg = diff --git a/src/librustc_metadata/cstore_impl.rs b/src/librustc_metadata/cstore_impl.rs index 3d025e984b0..d962d117552 100644 --- a/src/librustc_metadata/cstore_impl.rs +++ b/src/librustc_metadata/cstore_impl.rs @@ -29,7 +29,7 @@ use rustc_back::PanicStrategy; use syntax::ast; use syntax::attr; -use syntax::parse::new_parser_from_source_str; +use syntax::parse::filemap_to_tts; use syntax::symbol::Symbol; use syntax_pos::{mk_sp, Span}; use rustc::hir::svh::Svh; @@ -395,19 +395,9 @@ impl<'tcx> CrateStore<'tcx> for cstore::CStore { let (name, def) = data.get_macro(id.index); let source_name = format!("<{} macros>", name); - // NB: Don't use parse_tts_from_source_str because it parses with quote_depth > 0. - let mut parser = new_parser_from_source_str(&sess.parse_sess, source_name, def.body); - - let lo = parser.span.lo; - let body = match parser.parse_all_token_trees() { - Ok(body) => body, - Err(mut err) => { - err.emit(); - sess.abort_if_errors(); - unreachable!(); - } - }; - let local_span = mk_sp(lo, parser.prev_span.hi); + let filemap = sess.parse_sess.codemap().new_filemap(source_name, None, def.body); + let local_span = mk_sp(filemap.start_pos, filemap.end_pos); + let body = filemap_to_tts(&sess.parse_sess, filemap); // Mark the attrs as used let attrs = data.get_item_attrs(id.index); diff --git a/src/libsyntax/ext/base.rs b/src/libsyntax/ext/base.rs index 68d261c64f8..edf74e1fe19 100644 --- a/src/libsyntax/ext/base.rs +++ b/src/libsyntax/ext/base.rs @@ -615,9 +615,7 @@ impl<'a> ExtCtxt<'a> { pub fn new_parser_from_tts(&self, tts: &[tokenstream::TokenTree]) -> parser::Parser<'a> { - let mut parser = parse::tts_to_parser(self.parse_sess, tts.to_vec()); - parser.allow_interpolated_tts = false; // FIXME(jseyfried) `quote!` can't handle these yet - parser + parse::tts_to_parser(self.parse_sess, tts.to_vec()) } pub fn codemap(&self) -> &'a CodeMap { self.parse_sess.codemap() } pub fn parse_sess(&self) -> &'a parse::ParseSess { self.parse_sess } diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs index 2de31166070..46ffc93d2ee 100644 --- a/src/libsyntax/ext/tt/macro_parser.rs +++ b/src/libsyntax/ext/tt/macro_parser.rs @@ -82,7 +82,6 @@ use ast::Ident; use syntax_pos::{self, BytePos, mk_sp, Span}; use codemap::Spanned; use errors::FatalError; -use parse::lexer::*; //resolve bug? use parse::{Directory, ParseSess}; use parse::parser::{PathStyle, Parser}; use parse::token::{DocComment, MatchNt, SubstNt}; @@ -407,9 +406,9 @@ fn inner_parse_loop(cur_eis: &mut SmallVector>, Success(()) } -pub fn parse(sess: &ParseSess, rdr: TtReader, ms: &[TokenTree], directory: Option) +pub fn parse(sess: &ParseSess, tts: Vec, ms: &[TokenTree], directory: Option) -> NamedParseResult { - let mut parser = Parser::new(sess, Box::new(rdr), directory, true); + let mut parser = Parser::new(sess, tts, directory, true); let mut cur_eis = SmallVector::one(initial_matcher_pos(ms.to_owned(), parser.span.lo)); let mut next_eis = Vec::new(); // or proceed normally @@ -527,7 +526,7 @@ fn parse_nt<'a>(p: &mut Parser<'a>, sp: Span, name: &str) -> Nonterminal { "ident" => match p.token { token::Ident(sn) => { p.bump(); - token::NtIdent(Spanned::{node: sn, span: p.span}) + token::NtIdent(Spanned::{node: sn, span: p.prev_span}) } _ => { let token_str = pprust::token_to_string(&p.token); diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs index 3abd24b50ba..585232c5462 100644 --- a/src/libsyntax/ext/tt/macro_rules.rs +++ b/src/libsyntax/ext/tt/macro_rules.rs @@ -16,8 +16,8 @@ use ext::expand::{Expansion, ExpansionKind}; use ext::tt::macro_parser::{Success, Error, Failure}; use ext::tt::macro_parser::{MatchedSeq, MatchedNonterminal}; use ext::tt::macro_parser::{parse, parse_failure_msg}; +use ext::tt::transcribe::new_tt_reader; use parse::{Directory, ParseSess}; -use parse::lexer::new_tt_reader; use parse::parser::Parser; use parse::token::{self, NtTT, Token}; use parse::token::Token::*; @@ -113,13 +113,21 @@ fn generic_extension<'cx>(cx: &'cx ExtCtxt, _ => cx.span_bug(sp, "malformed macro rhs"), }; // rhs has holes ( `$id` and `$(...)` that need filled) - let trncbr = + let mut trncbr = new_tt_reader(&cx.parse_sess.span_diagnostic, Some(named_matches), rhs); + let mut tts = Vec::new(); + loop { + let tok = trncbr.real_token(); + if tok.tok == token::Eof { + break + } + tts.push(TokenTree::Token(tok.sp, tok.tok)); + } let directory = Directory { path: cx.current_expansion.module.directory.clone(), ownership: cx.current_expansion.directory_ownership, }; - let mut p = Parser::new(cx.parse_sess(), Box::new(trncbr), Some(directory), false); + let mut p = Parser::new(cx.parse_sess(), tts, Some(directory), false); p.root_module_name = cx.current_expansion.module.mod_path.last() .map(|id| (*id.name.as_str()).to_owned()); @@ -187,10 +195,8 @@ pub fn compile(sess: &ParseSess, def: &ast::MacroDef) -> SyntaxExtension { })), ]; - // Parse the macro_rules! invocation (`none` is for no interpolations): - let arg_reader = new_tt_reader(&sess.span_diagnostic, None, def.body.clone()); - - let argument_map = match parse(sess, arg_reader, &argument_gram, None) { + // Parse the macro_rules! invocation + let argument_map = match parse(sess, def.body.clone(), &argument_gram, None) { Success(m) => m, Failure(sp, tok) => { let s = parse_failure_msg(tok); diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/tt/transcribe.rs index 37e329e5d3b..82f1e183895 100644 --- a/src/libsyntax/ext/tt/transcribe.rs +++ b/src/libsyntax/ext/tt/transcribe.rs @@ -10,7 +10,7 @@ use self::LockstepIterSize::*; use ast::Ident; -use errors::{Handler, DiagnosticBuilder}; +use errors::Handler; use ext::tt::macro_parser::{NamedMatch, MatchedSeq, MatchedNonterminal}; use parse::token::{self, MatchNt, SubstNt, Token, NtIdent}; use parse::lexer::TokenAndSpan; @@ -44,8 +44,12 @@ pub struct TtReader<'a> { /* cached: */ pub cur_tok: Token, pub cur_span: Span, - /// Transform doc comments. Only useful in macro invocations - pub fatal_errs: Vec>, +} + +impl<'a> TtReader<'a> { + pub fn real_token(&mut self) -> TokenAndSpan { + tt_next_token(self) + } } /// This can do Macro-By-Example transcription. On the other hand, if @@ -76,7 +80,6 @@ pub fn new_tt_reader(sp_diag: &Handler, /* dummy values, never read: */ cur_tok: token::Eof, cur_span: DUMMY_SP, - fatal_errs: Vec::new(), }; tt_next_token(&mut r); /* get cur_tok and cur_span set up */ r diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs index 6c6161998d7..12b9130c474 100644 --- a/src/libsyntax/parse/lexer/mod.rs +++ b/src/libsyntax/parse/lexer/mod.rs @@ -12,7 +12,6 @@ use ast::{self, Ident}; use syntax_pos::{self, BytePos, CharPos, Pos, Span}; use codemap::CodeMap; use errors::{FatalError, DiagnosticBuilder}; -use ext::tt::transcribe::tt_next_token; use parse::{token, ParseSess}; use str::char_at; use symbol::{Symbol, keywords}; @@ -23,53 +22,10 @@ use std::char; use std::mem::replace; use std::rc::Rc; -pub use ext::tt::transcribe::{TtReader, new_tt_reader}; - pub mod comments; mod tokentrees; mod unicode_chars; -pub trait Reader { - fn is_eof(&self) -> bool; - fn try_next_token(&mut self) -> Result; - fn next_token(&mut self) -> TokenAndSpan where Self: Sized { - let res = self.try_next_token(); - self.unwrap_or_abort(res) - } - /// Report a fatal error with the current span. - fn fatal(&self, &str) -> FatalError; - /// Report a non-fatal error with the current span. - fn err(&self, &str); - fn emit_fatal_errors(&mut self); - fn unwrap_or_abort(&mut self, res: Result) -> TokenAndSpan { - match res { - Ok(tok) => tok, - Err(_) => { - self.emit_fatal_errors(); - panic!(FatalError); - } - } - } - fn peek(&self) -> TokenAndSpan; - /// Get a token the parser cares about. - fn try_real_token(&mut self) -> Result { - let mut t = self.try_next_token()?; - loop { - match t.tok { - token::Whitespace | token::Comment | token::Shebang(_) => { - t = self.try_next_token()?; - } - _ => break, - } - } - Ok(t) - } - fn real_token(&mut self) -> TokenAndSpan { - let res = self.try_real_token(); - self.unwrap_or_abort(res) - } -} - #[derive(Clone, PartialEq, Eq, Debug)] pub struct TokenAndSpan { pub tok: token::Token, @@ -182,36 +138,6 @@ impl<'a> StringReader<'a> { } } -impl<'a> Reader for TtReader<'a> { - fn is_eof(&self) -> bool { - self.peek().tok == token::Eof - } - fn try_next_token(&mut self) -> Result { - assert!(self.fatal_errs.is_empty()); - let r = tt_next_token(self); - debug!("TtReader: r={:?}", r); - Ok(r) - } - fn fatal(&self, m: &str) -> FatalError { - self.sp_diag.span_fatal(self.cur_span, m) - } - fn err(&self, m: &str) { - self.sp_diag.span_err(self.cur_span, m); - } - fn emit_fatal_errors(&mut self) { - for err in &mut self.fatal_errs { - err.emit(); - } - self.fatal_errs.clear(); - } - fn peek(&self) -> TokenAndSpan { - TokenAndSpan { - tok: self.cur_tok.clone(), - sp: self.cur_span, - } - } -} - impl<'a> StringReader<'a> { /// For comments.rs, which hackily pokes into next_pos and ch pub fn new_raw<'b>(sess: &'a ParseSess, filemap: Rc) -> Self { diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index 500e8285b4c..0937ef15b4d 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -45,7 +45,7 @@ pub mod obsolete; /// Info about a parsing session. pub struct ParseSess { - pub span_diagnostic: Handler, // better be the same as the one in the reader! + pub span_diagnostic: Handler, pub unstable_features: UnstableFeatures, pub config: CrateConfig, /// Used to determine and report recursive mod inclusions @@ -227,8 +227,7 @@ pub fn filemap_to_tts(sess: &ParseSess, filemap: Rc) -> Vec(sess: &'a ParseSess, tts: Vec) -> Parser<'a> { - let trdr = lexer::new_tt_reader(&sess.span_diagnostic, None, tts); - let mut p = Parser::new(sess, Box::new(trdr), None, false); + let mut p = Parser::new(sess, tts, None, false); p.check_unknown_macro_variable(); p } diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index 9ba6d4d17f7..608f8688e88 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -46,7 +46,7 @@ use ext::tt::macro_parser; use parse; use parse::classify; use parse::common::SeqSep; -use parse::lexer::{Reader, TokenAndSpan}; +use parse::lexer::TokenAndSpan; use parse::obsolete::ObsoleteSyntax; use parse::token::{self, MatchNt, SubstNt}; use parse::{new_sub_parser_from_file, ParseSess, Directory, DirectoryOwnership}; @@ -188,14 +188,11 @@ pub struct Parser<'a> { pub restrictions: Restrictions, pub quote_depth: usize, // not (yet) related to the quasiquoter parsing_token_tree: bool, - pub reader: Box, /// The set of seen errors about obsolete syntax. Used to suppress /// extra detail when the same error is seen twice pub obsolete_set: HashSet, /// Used to determine the path to externally loaded source files pub directory: Directory, - /// Stack of open delimiters and their spans. Used for error message. - pub open_braces: Vec<(token::DelimToken, Span)>, /// Name of the root module this parser originated from. If `None`, then the /// name is not known. This does not change while the parser is descending /// into modules, and sub-parsers have new values for this name. @@ -203,7 +200,6 @@ pub struct Parser<'a> { pub expected_tokens: Vec, pub tts: Vec<(TokenTree, usize)>, pub desugar_doc_comments: bool, - pub allow_interpolated_tts: bool, } #[derive(PartialEq, Eq, Clone)] @@ -269,12 +265,17 @@ impl From> for LhsExpr { impl<'a> Parser<'a> { pub fn new(sess: &'a ParseSess, - rdr: Box, + tokens: Vec, directory: Option, desugar_doc_comments: bool) -> Self { + let tt = TokenTree::Delimited(syntax_pos::DUMMY_SP, Rc::new(Delimited { + delim: token::NoDelim, + open_span: syntax_pos::DUMMY_SP, + tts: tokens, + close_span: syntax_pos::DUMMY_SP, + })); let mut parser = Parser { - reader: rdr, sess: sess, token: token::Underscore, span: syntax_pos::DUMMY_SP, @@ -286,12 +287,10 @@ impl<'a> Parser<'a> { parsing_token_tree: false, obsolete_set: HashSet::new(), directory: Directory { path: PathBuf::new(), ownership: DirectoryOwnership::Owned }, - open_braces: Vec::new(), root_module_name: None, expected_tokens: Vec::new(), - tts: Vec::new(), + tts: if tt.len() > 0 { vec![(tt, 0)] } else { Vec::new() }, desugar_doc_comments: desugar_doc_comments, - allow_interpolated_tts: true, }; let tok = parser.next_tok(); @@ -320,7 +319,7 @@ impl<'a> Parser<'a> { continue } } else { - self.reader.real_token() + TokenAndSpan { tok: token::Eof, sp: self.span } }; loop { @@ -2688,94 +2687,28 @@ impl<'a> Parser<'a> { // whether something will be a nonterminal or a seq // yet. match self.token { - token::Eof => { - let mut err: DiagnosticBuilder<'a> = - self.diagnostic().struct_span_err(self.span, - "this file contains an un-closed delimiter"); - for &(_, sp) in &self.open_braces { - err.span_help(sp, "did you mean to close this delimiter?"); - } - - Err(err) - }, token::OpenDelim(delim) => { - if self.tts.last().map(|&(_, i)| i == 1).unwrap_or(false) { + if self.quote_depth == 0 && self.tts.last().map(|&(_, i)| i == 1).unwrap_or(false) { let tt = self.tts.pop().unwrap().0; self.bump(); - return Ok(if self.allow_interpolated_tts { - // avoid needlessly reparsing token trees in recursive macro expansions - TokenTree::Token(tt.span(), token::Interpolated(Rc::new(token::NtTT(tt)))) - } else { - tt - }); + return Ok(tt); } let parsing_token_tree = ::std::mem::replace(&mut self.parsing_token_tree, true); - // The span for beginning of the delimited section - let pre_span = self.span; - - // Parse the open delimiter. - self.open_braces.push((delim, self.span)); let open_span = self.span; self.bump(); - - // Parse the token trees within the delimiters. - // We stop at any delimiter so we can try to recover if the user - // uses an incorrect delimiter. let tts = self.parse_seq_to_before_tokens(&[&token::CloseDelim(token::Brace), &token::CloseDelim(token::Paren), &token::CloseDelim(token::Bracket)], SeqSep::none(), |p| p.parse_token_tree(), |mut e| e.emit()); + self.parsing_token_tree = parsing_token_tree; let close_span = self.span; - // Expand to cover the entire delimited token tree - let span = Span { hi: close_span.hi, ..pre_span }; - - match self.token { - // Correct delimiter. - token::CloseDelim(d) if d == delim => { - self.open_braces.pop().unwrap(); - - // Parse the close delimiter. - self.bump(); - } - // Incorrect delimiter. - token::CloseDelim(other) => { - let token_str = self.this_token_to_string(); - let mut err = self.diagnostic().struct_span_err(self.span, - &format!("incorrect close delimiter: `{}`", token_str)); - // This is a conservative error: only report the last unclosed delimiter. - // The previous unclosed delimiters could actually be closed! The parser - // just hasn't gotten to them yet. - if let Some(&(_, sp)) = self.open_braces.last() { - err.span_note(sp, "unclosed delimiter"); - }; - err.emit(); - - self.open_braces.pop().unwrap(); - - // If the incorrect delimiter matches an earlier opening - // delimiter, then don't consume it (it can be used to - // close the earlier one). Otherwise, consume it. - // E.g., we try to recover from: - // fn foo() { - // bar(baz( - // } // Incorrect delimiter but matches the earlier `{` - if !self.open_braces.iter().any(|&(b, _)| b == other) { - self.bump(); - } - } - token::Eof => { - // Silently recover, the EOF token will be seen again - // and an error emitted then. Thus we don't pop from - // self.open_braces here. - }, - _ => {} - } + self.bump(); - self.parsing_token_tree = parsing_token_tree; + let span = Span { lo: open_span.lo, ..close_span }; Ok(TokenTree::Delimited(span, Rc::new(Delimited { delim: delim, open_span: open_span, @@ -2783,21 +2716,9 @@ impl<'a> Parser<'a> { close_span: close_span, }))) }, - token::CloseDelim(_) => { - // An unexpected closing delimiter (i.e., there is no - // matching opening delimiter). - let token_str = self.this_token_to_string(); - let err = self.diagnostic().struct_span_err(self.span, - &format!("unexpected close delimiter: `{}`", token_str)); - Err(err) - }, - /* we ought to allow different depths of unquotation */ - token::Dollar | token::SubstNt(..) if self.quote_depth > 0 => { - self.parse_unquoted() - } - _ => { - Ok(TokenTree::Token(self.span, self.bump_and_get())) - } + token::CloseDelim(_) | token::Eof => unreachable!(), + token::Dollar | token::SubstNt(..) if self.quote_depth > 0 => self.parse_unquoted(), + _ => Ok(TokenTree::Token(self.span, self.bump_and_get())), } } diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index e352e7853c7..ab5dc8181e0 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -30,7 +30,6 @@ use codemap::{Spanned, combine_spans}; use ext::base; use ext::tt::macro_parser; use parse::lexer::comments::{doc_comment_style, strip_doc_comment_decoration}; -use parse::lexer; use parse::{self, Directory}; use parse::token::{self, Token, Lit, Nonterminal}; use print::pprust; @@ -139,7 +138,10 @@ impl TokenTree { if let Nonterminal::NtTT(..) = **nt { 1 } else { 0 } }, TokenTree::Token(_, token::MatchNt(..)) => 3, - TokenTree::Delimited(_, ref delimed) => delimed.tts.len() + 2, + TokenTree::Delimited(_, ref delimed) => match delimed.delim { + token::NoDelim => delimed.tts.len(), + _ => delimed.tts.len() + 2, + }, TokenTree::Sequence(_, ref seq) => seq.tts.len(), TokenTree::Token(..) => 0, } @@ -181,6 +183,9 @@ impl TokenTree { close_span: sp, })) } + (&TokenTree::Delimited(_, ref delimed), _) if delimed.delim == token::NoDelim => { + delimed.tts[index].clone() + } (&TokenTree::Delimited(_, ref delimed), _) => { if index == 0 { return delimed.open_tt(); @@ -215,14 +220,12 @@ impl TokenTree { mtch: &[TokenTree], tts: &[TokenTree]) -> macro_parser::NamedParseResult { - let diag = &cx.parse_sess().span_diagnostic; // `None` is because we're not interpolating - let arg_rdr = lexer::new_tt_reader(diag, None, tts.iter().cloned().collect()); let directory = Directory { path: cx.current_expansion.module.directory.clone(), ownership: cx.current_expansion.directory_ownership, }; - macro_parser::parse(cx.parse_sess(), arg_rdr, mtch, Some(directory)) + macro_parser::parse(cx.parse_sess(), tts.iter().cloned().collect(), mtch, Some(directory)) } /// Check if this TokenTree is equal to the other, regardless of span information. diff --git a/src/test/parse-fail/issue-33569.rs b/src/test/parse-fail/issue-33569.rs index 130278d778a..e3c17af82aa 100644 --- a/src/test/parse-fail/issue-33569.rs +++ b/src/test/parse-fail/issue-33569.rs @@ -13,6 +13,6 @@ macro_rules! foo { { $+ } => { //~ ERROR expected identifier, found `+` $(x)(y) //~ ERROR expected `*` or `+` - //~^ ERROR no rules expected the token `y` + //~^ ERROR no rules expected the token `)` } } -- cgit 1.4.1-3-g733a5 From 6a9248fc1525e619d4ffb2b895a8d15c4bf90de8 Mon Sep 17 00:00:00 2001 From: Jeffrey Seyfried Date: Sat, 14 Jan 2017 12:15:26 +0000 Subject: Clean up `ext::tt::transcribe`. --- src/libsyntax/ext/tt/macro_rules.rs | 13 +---- src/libsyntax/ext/tt/transcribe.rs | 96 +++++++++++++------------------------ 2 files changed, 35 insertions(+), 74 deletions(-) (limited to 'src/libsyntax') diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs index 585232c5462..f6a25d4acee 100644 --- a/src/libsyntax/ext/tt/macro_rules.rs +++ b/src/libsyntax/ext/tt/macro_rules.rs @@ -16,7 +16,7 @@ use ext::expand::{Expansion, ExpansionKind}; use ext::tt::macro_parser::{Success, Error, Failure}; use ext::tt::macro_parser::{MatchedSeq, MatchedNonterminal}; use ext::tt::macro_parser::{parse, parse_failure_msg}; -use ext::tt::transcribe::new_tt_reader; +use ext::tt::transcribe::transcribe; use parse::{Directory, ParseSess}; use parse::parser::Parser; use parse::token::{self, NtTT, Token}; @@ -113,16 +113,7 @@ fn generic_extension<'cx>(cx: &'cx ExtCtxt, _ => cx.span_bug(sp, "malformed macro rhs"), }; // rhs has holes ( `$id` and `$(...)` that need filled) - let mut trncbr = - new_tt_reader(&cx.parse_sess.span_diagnostic, Some(named_matches), rhs); - let mut tts = Vec::new(); - loop { - let tok = trncbr.real_token(); - if tok.tok == token::Eof { - break - } - tts.push(TokenTree::Token(tok.sp, tok.tok)); - } + let tts = transcribe(&cx.parse_sess.span_diagnostic, Some(named_matches), rhs); let directory = Directory { path: cx.current_expansion.module.directory.clone(), ownership: cx.current_expansion.directory_ownership, diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/tt/transcribe.rs index 82f1e183895..bf6851ec1dc 100644 --- a/src/libsyntax/ext/tt/transcribe.rs +++ b/src/libsyntax/ext/tt/transcribe.rs @@ -13,7 +13,6 @@ use ast::Ident; use errors::Handler; use ext::tt::macro_parser::{NamedMatch, MatchedSeq, MatchedNonterminal}; use parse::token::{self, MatchNt, SubstNt, Token, NtIdent}; -use parse::lexer::TokenAndSpan; use syntax_pos::{Span, DUMMY_SP}; use tokenstream::{self, TokenTree}; use util::small_vector::SmallVector; @@ -32,8 +31,8 @@ struct TtFrame { } #[derive(Clone)] -pub struct TtReader<'a> { - pub sp_diag: &'a Handler, +struct TtReader<'a> { + sp_diag: &'a Handler, /// the unzipped tree: stack: SmallVector, /* for MBE-style macro transcription */ @@ -41,24 +40,15 @@ pub struct TtReader<'a> { repeat_idx: Vec, repeat_len: Vec, - /* cached: */ - pub cur_tok: Token, - pub cur_span: Span, -} - -impl<'a> TtReader<'a> { - pub fn real_token(&mut self) -> TokenAndSpan { - tt_next_token(self) - } } /// This can do Macro-By-Example transcription. On the other hand, if /// `src` contains no `TokenTree::Sequence`s, `MatchNt`s or `SubstNt`s, `interp` can /// (and should) be None. -pub fn new_tt_reader(sp_diag: &Handler, - interp: Option>>, - src: Vec) - -> TtReader { +pub fn transcribe(sp_diag: &Handler, + interp: Option>>, + src: Vec) + -> Vec { let mut r = TtReader { sp_diag: sp_diag, stack: SmallVector::one(TtFrame { @@ -77,12 +67,15 @@ pub fn new_tt_reader(sp_diag: &Handler, }, repeat_idx: Vec::new(), repeat_len: Vec::new(), - /* dummy values, never read: */ - cur_tok: token::Eof, - cur_span: DUMMY_SP, }; - tt_next_token(&mut r); /* get cur_tok and cur_span set up */ - r + + let mut tts = Vec::new(); + let mut prev_span = DUMMY_SP; + while let Some(tt) = tt_next_token(&mut r, prev_span) { + prev_span = tt.span(); + tts.push(tt); + } + tts } fn lookup_cur_matched_by_matched(r: &TtReader, start: Rc) -> Rc { @@ -156,38 +149,24 @@ fn lockstep_iter_size(t: &TokenTree, r: &TtReader) -> LockstepIterSize { /// Return the next token from the TtReader. /// EFFECT: advances the reader's token field -pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan { - // FIXME(pcwalton): Bad copy? - let ret_val = TokenAndSpan { - tok: r.cur_tok.clone(), - sp: r.cur_span.clone(), - }; +fn tt_next_token(r: &mut TtReader, prev_span: Span) -> Option { loop { - let should_pop = match r.stack.last() { - None => { - assert_eq!(ret_val.tok, token::Eof); - return ret_val; - } - Some(frame) => { - if frame.idx < frame.forest.len() { - break; - } - !frame.dotdotdoted || - *r.repeat_idx.last().unwrap() == *r.repeat_len.last().unwrap() - 1 + let should_pop = if let Some(frame) = r.stack.last() { + if frame.idx < frame.forest.len() { + break; } + !frame.dotdotdoted || *r.repeat_idx.last().unwrap() == *r.repeat_len.last().unwrap() - 1 + } else { + return None; }; /* done with this set; pop or repeat? */ if should_pop { let prev = r.stack.pop().unwrap(); - match r.stack.last_mut() { - None => { - r.cur_tok = token::Eof; - return ret_val; - } - Some(frame) => { - frame.idx += 1; - } + if let Some(frame) = r.stack.last_mut() { + frame.idx += 1; + } else { + return None; } if prev.dotdotdoted { r.repeat_idx.pop(); @@ -197,8 +176,7 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan { *r.repeat_idx.last_mut().unwrap() += 1; r.stack.last_mut().unwrap().idx = 0; if let Some(tk) = r.stack.last().unwrap().sep.clone() { - r.cur_tok = tk; // repeat same span, I guess - return ret_val; + return Some(TokenTree::Token(prev_span, tk)); // repeat same span, I guess } } } @@ -234,7 +212,7 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan { } r.stack.last_mut().unwrap().idx += 1; - return tt_next_token(r); + return tt_next_token(r, prev_span); } r.repeat_len.push(len); r.repeat_idx.push(0); @@ -252,9 +230,7 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan { r.stack.last_mut().unwrap().idx += 1; match lookup_cur_matched(r, ident) { None => { - r.cur_span = sp; - r.cur_tok = SubstNt(ident); - return ret_val; + return Some(TokenTree::Token(sp, SubstNt(ident))); // this can't be 0 length, just like TokenTree::Delimited } Some(cur_matched) => if let MatchedNonterminal(ref nt) = *cur_matched { @@ -263,15 +239,11 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan { // (a) idents can be in lots of places, so it'd be a pain // (b) we actually can, since it's a token. NtIdent(ref sn) => { - r.cur_span = sn.span; - r.cur_tok = token::Ident(sn.node); - return ret_val; + return Some(TokenTree::Token(sn.span, token::Ident(sn.node))); } _ => { - // FIXME(pcwalton): Bad copy. - r.cur_span = sp; - r.cur_tok = token::Interpolated(nt.clone()); - return ret_val; + // FIXME(pcwalton): Bad copy + return Some(TokenTree::Token(sp, token::Interpolated(nt.clone()))); } } } else { @@ -292,11 +264,9 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan { }); // if this could be 0-length, we'd need to potentially recur here } - TokenTree::Token(sp, tok) => { - r.cur_span = sp; - r.cur_tok = tok; + tt @ TokenTree::Token(..) => { r.stack.last_mut().unwrap().idx += 1; - return ret_val; + return Some(tt); } } } -- cgit 1.4.1-3-g733a5 From 57c0ed097ce150fa1d684b5b3b5479a5dedd2b7b Mon Sep 17 00:00:00 2001 From: Jeffrey Seyfried Date: Sat, 14 Jan 2017 12:42:00 +0000 Subject: Avoid interpolated token trees. --- src/libsyntax/ext/tt/macro_parser.rs | 17 +---------------- src/libsyntax/ext/tt/transcribe.rs | 3 ++- src/libsyntax/parse/parser.rs | 26 ++++++-------------------- 3 files changed, 9 insertions(+), 37 deletions(-) (limited to 'src/libsyntax') diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs index 46ffc93d2ee..834ece97af5 100644 --- a/src/libsyntax/ext/tt/macro_parser.rs +++ b/src/libsyntax/ext/tt/macro_parser.rs @@ -480,23 +480,8 @@ fn parse_nt<'a>(p: &mut Parser<'a>, sp: Span, name: &str) -> Nonterminal { match name { "tt" => { p.quote_depth += 1; //but in theory, non-quoted tts might be useful - let mut tt = panictry!(p.parse_token_tree()); + let tt = panictry!(p.parse_token_tree()); p.quote_depth -= 1; - while let TokenTree::Token(sp, token::Interpolated(nt)) = tt { - if let token::NtTT(..) = *nt { - match Rc::try_unwrap(nt) { - Ok(token::NtTT(sub_tt)) => tt = sub_tt, - Ok(_) => unreachable!(), - Err(nt_rc) => match *nt_rc { - token::NtTT(ref sub_tt) => tt = sub_tt.clone(), - _ => unreachable!(), - }, - } - } else { - tt = TokenTree::Token(sp, token::Interpolated(nt.clone())); - break - } - } return token::NtTT(tt); } _ => {} diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/tt/transcribe.rs index bf6851ec1dc..38becbe7b1d 100644 --- a/src/libsyntax/ext/tt/transcribe.rs +++ b/src/libsyntax/ext/tt/transcribe.rs @@ -12,7 +12,7 @@ use self::LockstepIterSize::*; use ast::Ident; use errors::Handler; use ext::tt::macro_parser::{NamedMatch, MatchedSeq, MatchedNonterminal}; -use parse::token::{self, MatchNt, SubstNt, Token, NtIdent}; +use parse::token::{self, MatchNt, SubstNt, Token, NtIdent, NtTT}; use syntax_pos::{Span, DUMMY_SP}; use tokenstream::{self, TokenTree}; use util::small_vector::SmallVector; @@ -241,6 +241,7 @@ fn tt_next_token(r: &mut TtReader, prev_span: Span) -> Option { NtIdent(ref sn) => { return Some(TokenTree::Token(sn.span, token::Ident(sn.node))); } + NtTT(ref tt) => return Some(tt.clone()), _ => { // FIXME(pcwalton): Bad copy return Some(TokenTree::Token(sp, token::Interpolated(nt.clone()))); diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index 608f8688e88..f958cedd286 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -306,8 +306,8 @@ impl<'a> Parser<'a> { } fn next_tok(&mut self) -> TokenAndSpan { - 'outer: loop { - let mut tok = if let Some((tts, i)) = self.tts.pop() { + loop { + let tok = if let Some((tts, i)) = self.tts.pop() { let tt = tts.get_tt(i); if i + 1 < tts.len() { self.tts.push((tts, i + 1)); @@ -322,25 +322,11 @@ impl<'a> Parser<'a> { TokenAndSpan { tok: token::Eof, sp: self.span } }; - loop { - let nt = match tok.tok { - token::Interpolated(ref nt) => nt.clone(), - token::DocComment(name) if self.desugar_doc_comments => { - self.tts.push((TokenTree::Token(tok.sp, token::DocComment(name)), 0)); - continue 'outer - } - _ => return tok, - }; - match *nt { - token::NtTT(TokenTree::Token(sp, ref t)) => { - tok = TokenAndSpan { tok: t.clone(), sp: sp }; - } - token::NtTT(ref tt) => { - self.tts.push((tt.clone(), 0)); - continue 'outer - } - _ => return tok, + match tok.tok { + token::DocComment(name) if self.desugar_doc_comments => { + self.tts.push((TokenTree::Token(tok.sp, token::DocComment(name)), 0)); } + _ => return tok, } } } -- cgit 1.4.1-3-g733a5 From 4c98e1bc592841c03b5228787c97c2edf67ccc6c Mon Sep 17 00:00:00 2001 From: Jeffrey Seyfried Date: Sat, 14 Jan 2017 11:13:45 +0000 Subject: Remove the lookahead buffer. --- src/libsyntax/parse/parser.rs | 51 ++++++++++++------------------------------- 1 file changed, 14 insertions(+), 37 deletions(-) (limited to 'src/libsyntax') diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index f958cedd286..aa5331e4c7d 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -156,22 +156,6 @@ enum PrevTokenKind { Other, } -// Simple circular buffer used for keeping few next tokens. -#[derive(Default)] -struct LookaheadBuffer { - buffer: [TokenAndSpan; LOOKAHEAD_BUFFER_CAPACITY], - start: usize, - end: usize, -} - -const LOOKAHEAD_BUFFER_CAPACITY: usize = 8; - -impl LookaheadBuffer { - fn len(&self) -> usize { - (LOOKAHEAD_BUFFER_CAPACITY + self.end - self.start) % LOOKAHEAD_BUFFER_CAPACITY - } -} - /* ident is handled by common.rs */ pub struct Parser<'a> { @@ -184,7 +168,6 @@ pub struct Parser<'a> { pub prev_span: Span, /// the previous token kind prev_token_kind: PrevTokenKind, - lookahead_buffer: LookaheadBuffer, pub restrictions: Restrictions, pub quote_depth: usize, // not (yet) related to the quasiquoter parsing_token_tree: bool, @@ -281,7 +264,6 @@ impl<'a> Parser<'a> { span: syntax_pos::DUMMY_SP, prev_span: syntax_pos::DUMMY_SP, prev_token_kind: PrevTokenKind::Other, - lookahead_buffer: Default::default(), restrictions: Restrictions::empty(), quote_depth: 0, parsing_token_tree: false, @@ -875,14 +857,7 @@ impl<'a> Parser<'a> { _ => PrevTokenKind::Other, }; - let next = if self.lookahead_buffer.start == self.lookahead_buffer.end { - self.next_tok() - } else { - // Avoid token copies with `replace`. - let old_start = self.lookahead_buffer.start; - self.lookahead_buffer.start = (old_start + 1) % LOOKAHEAD_BUFFER_CAPACITY; - mem::replace(&mut self.lookahead_buffer.buffer[old_start], Default::default()) - }; + let next = self.next_tok(); self.span = next.sp; self.token = next.tok; self.expected_tokens.clear(); @@ -917,18 +892,20 @@ impl<'a> Parser<'a> { F: FnOnce(&token::Token) -> R, { if dist == 0 { - f(&self.token) - } else if dist < LOOKAHEAD_BUFFER_CAPACITY { - while self.lookahead_buffer.len() < dist { - self.lookahead_buffer.buffer[self.lookahead_buffer.end] = self.next_tok(); - self.lookahead_buffer.end = - (self.lookahead_buffer.end + 1) % LOOKAHEAD_BUFFER_CAPACITY; - } - let index = (self.lookahead_buffer.start + dist - 1) % LOOKAHEAD_BUFFER_CAPACITY; - f(&self.lookahead_buffer.buffer[index].tok) - } else { - self.bug("lookahead distance is too large"); + return f(&self.token); + } + let mut tok = token::Eof; + if let Some(&(ref tts, mut i)) = self.tts.last() { + i += dist - 1; + if i < tts.len() { + tok = match tts.get_tt(i) { + TokenTree::Token(_, tok) => tok, + TokenTree::Delimited(_, delimited) => token::OpenDelim(delimited.delim), + TokenTree::Sequence(..) => token::Dollar, + }; + } } + f(&tok) } pub fn fatal(&self, m: &str) -> DiagnosticBuilder<'a> { self.sess.span_diagnostic.struct_span_fatal(self.span, m) -- cgit 1.4.1-3-g733a5 From 0b9e26f390403aa95620d3b813f046732b371fb1 Mon Sep 17 00:00:00 2001 From: Jeffrey Seyfried Date: Tue, 17 Jan 2017 04:50:46 +0000 Subject: Fix fallout in `rustdoc`. --- src/librustdoc/html/highlight.rs | 16 +++++++--------- src/libsyntax/parse/lexer/mod.rs | 6 +++--- 2 files changed, 10 insertions(+), 12 deletions(-) (limited to 'src/libsyntax') diff --git a/src/librustdoc/html/highlight.rs b/src/librustdoc/html/highlight.rs index a031be8b3c2..0629e93e7ef 100644 --- a/src/librustdoc/html/highlight.rs +++ b/src/librustdoc/html/highlight.rs @@ -27,7 +27,7 @@ use std::io; use std::io::prelude::*; use syntax::codemap::CodeMap; -use syntax::parse::lexer::{self, Reader, TokenAndSpan}; +use syntax::parse::lexer::{self, TokenAndSpan}; use syntax::parse::token; use syntax::parse; use syntax_pos::Span; @@ -42,8 +42,7 @@ pub fn render_with_highlighting(src: &str, class: Option<&str>, id: Option<&str> let mut out = Vec::new(); write_header(class, id, &mut out).unwrap(); - let mut classifier = Classifier::new(lexer::StringReader::new(&sess.span_diagnostic, fm), - sess.codemap()); + let mut classifier = Classifier::new(lexer::StringReader::new(&sess, fm), sess.codemap()); if let Err(_) = classifier.write_source(&mut out) { return format!("
{}
", src); } @@ -63,8 +62,7 @@ pub fn render_inner_with_highlighting(src: &str) -> io::Result { let fm = sess.codemap().new_filemap("".to_string(), None, src.to_string()); let mut out = Vec::new(); - let mut classifier = Classifier::new(lexer::StringReader::new(&sess.span_diagnostic, fm), - sess.codemap()); + let mut classifier = Classifier::new(lexer::StringReader::new(&sess, fm), sess.codemap()); classifier.write_source(&mut out)?; Ok(String::from_utf8_lossy(&out).into_owned()) @@ -185,10 +183,10 @@ impl<'a> Classifier<'a> { Ok(tas) => tas, Err(_) => { self.lexer.emit_fatal_errors(); - self.lexer.span_diagnostic.struct_warn("Backing out of syntax highlighting") - .note("You probably did not intend to render this \ - as a rust code-block") - .emit(); + self.lexer.sess.span_diagnostic + .struct_warn("Backing out of syntax highlighting") + .note("You probably did not intend to render this as a rust code-block") + .emit(); return Err(io::Error::new(io::ErrorKind::Other, "")); } }; diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs index 12b9130c474..6bc15115b09 100644 --- a/src/libsyntax/parse/lexer/mod.rs +++ b/src/libsyntax/parse/lexer/mod.rs @@ -111,7 +111,7 @@ impl<'a> StringReader<'a> { } } /// Return the next token. EFFECT: advances the string_reader. - fn try_next_token(&mut self) -> Result { + pub fn try_next_token(&mut self) -> Result { assert!(self.fatal_errs.is_empty()); let ret_val = TokenAndSpan { tok: replace(&mut self.peek_tok, token::Underscore), @@ -123,13 +123,13 @@ impl<'a> StringReader<'a> { fn fatal(&self, m: &str) -> FatalError { self.fatal_span(self.peek_span, m) } - fn emit_fatal_errors(&mut self) { + pub fn emit_fatal_errors(&mut self) { for err in &mut self.fatal_errs { err.emit(); } self.fatal_errs.clear(); } - fn peek(&self) -> TokenAndSpan { + pub fn peek(&self) -> TokenAndSpan { // FIXME(pcwalton): Bad copy! TokenAndSpan { tok: self.peek_tok.clone(), -- cgit 1.4.1-3-g733a5 From 853f6974767243e2e5defb43f5f7195ce1fb1cc7 Mon Sep 17 00:00:00 2001 From: Vadim Petrochenkov Date: Thu, 19 Jan 2017 13:28:45 +0300 Subject: Fix regression in parsing of trait object types --- src/libsyntax/parse/parser.rs | 11 +++++++++-- src/test/parse-fail/bounds-obj-parens.rs | 15 +++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 src/test/parse-fail/bounds-obj-parens.rs (limited to 'src/libsyntax') diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index d1a683b0bd5..b5913daaa5b 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -1277,10 +1277,17 @@ impl<'a> Parser<'a> { "at least one type parameter bound \ must be specified"); } - if let TyKind::Path(None, ref path) = lhs.node { + + let mut lhs = lhs.unwrap(); + if let TyKind::Paren(ty) = lhs.node { + // We have to accept the first bound in parens for backward compatibility. + // Example: `(Bound) + Bound + Bound` + lhs = ty.unwrap(); + } + if let TyKind::Path(None, path) = lhs.node { let poly_trait_ref = PolyTraitRef { bound_lifetimes: Vec::new(), - trait_ref: TraitRef { path: path.clone(), ref_id: lhs.id }, + trait_ref: TraitRef { path: path, ref_id: lhs.id }, span: lhs.span, }; let poly_trait_ref = TraitTyParamBound(poly_trait_ref, TraitBoundModifier::None); diff --git a/src/test/parse-fail/bounds-obj-parens.rs b/src/test/parse-fail/bounds-obj-parens.rs new file mode 100644 index 00000000000..cbdffb4a255 --- /dev/null +++ b/src/test/parse-fail/bounds-obj-parens.rs @@ -0,0 +1,15 @@ +// Copyright 2017 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// compile-flags: -Z parse-only + +type A = Box<(Fn(D::Error) -> E) + 'static + Send + Sync>; // OK + +FAIL //~ ERROR -- cgit 1.4.1-3-g733a5