From 8f044fae36b73ec4593c127ec2a7c28716208591 Mon Sep 17 00:00:00 2001 From: Jonathan Turner Date: Tue, 5 Jul 2016 15:24:23 -0400 Subject: Remove BasicEmitter --- src/libsyntax/parse/lexer/mod.rs | 2 +- src/libsyntax/parse/mod.rs | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) (limited to 'src/libsyntax/parse') diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs index 77b5c10899a..5ea1d6be9fe 100644 --- a/src/libsyntax/parse/lexer/mod.rs +++ b/src/libsyntax/parse/lexer/mod.rs @@ -1686,7 +1686,7 @@ mod tests { // FIXME (#22405): Replace `Box::new` with `box` here when/if possible. let emitter = errors::emitter::EmitterWriter::new(Box::new(io::sink()), None, - cm, + Some(cm), errors::snippet::FormatMode::EnvironmentSelected); errors::Handler::with_emitter(true, false, Box::new(emitter)) } diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index 9502bc48a3e..6af4d95e888 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -50,7 +50,11 @@ pub struct ParseSess { impl ParseSess { pub fn new() -> ParseSess { let cm = Rc::new(CodeMap::new()); - let handler = Handler::with_tty_emitter(ColorConfig::Auto, None, true, false, cm.clone()); + let handler = Handler::with_tty_emitter(ColorConfig::Auto, + None, + true, + false, + Some(cm.clone())); ParseSess::with_span_handler(handler, cm) } -- cgit 1.4.1-3-g733a5 From fecf3b6a2c192d52ae7f69551f76b1ac4b01a85c Mon Sep 17 00:00:00 2001 From: cgswords Date: Thu, 14 Jul 2016 09:52:43 -0700 Subject: Added tokenstream parser procedure --- src/libsyntax/parse/mod.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'src/libsyntax/parse') diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index 9502bc48a3e..80ffd877518 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -224,10 +224,18 @@ pub fn filemap_to_parser<'a>(sess: &'a ParseSess, // compiler expands into it pub fn new_parser_from_tts<'a>(sess: &'a ParseSess, cfg: ast::CrateConfig, - tts: Vec) -> Parser<'a> { + tts: Vec) + -> Parser<'a> { tts_to_parser(sess, tts, cfg) } +pub fn new_parser_from_ts<'a>(sess: &'a ParseSess, + cfg: ast::CrateConfig, + ts: tokenstream::TokenStream) + -> Parser<'a> { + tts_to_parser(sess, ts.tts, cfg) +} + // base abstractions -- cgit 1.4.1-3-g733a5 From ff95ba3a8c1cc2cf9b1901a18ba73434f3380c0d Mon Sep 17 00:00:00 2001 From: Aravind Gollakota Date: Tue, 5 Jul 2016 21:35:12 -0700 Subject: syntax: Better error message for inner attr following doc comment --- src/libsyntax/parse/attr.rs | 70 +++++++++++++++++----- .../parse-fail/inner-attr-after-doc-comment.rs | 20 +++++++ src/test/parse-fail/inner-attr.rs | 16 +++++ 3 files changed, 92 insertions(+), 14 deletions(-) create mode 100644 src/test/parse-fail/inner-attr-after-doc-comment.rs create mode 100644 src/test/parse-fail/inner-attr.rs (limited to 'src/libsyntax/parse') diff --git a/src/libsyntax/parse/attr.rs b/src/libsyntax/parse/attr.rs index 15344cef1db..2ae3236cd5a 100644 --- a/src/libsyntax/parse/attr.rs +++ b/src/libsyntax/parse/attr.rs @@ -18,23 +18,43 @@ use parse::token; use parse::parser::{Parser, TokenType}; use ptr::P; +#[derive(PartialEq, Eq, Debug)] +enum InnerAttributeParsePolicy<'a> { + Permitted, + NotPermitted { reason: &'a str }, +} + +const DEFAULT_UNEXPECTED_INNER_ATTR_ERR_MSG: &'static str = "an inner attribute is not \ + permitted in this context"; + impl<'a> Parser<'a> { /// Parse attributes that appear before an item pub fn parse_outer_attributes(&mut self) -> PResult<'a, Vec> { let mut attrs: Vec = Vec::new(); + let mut just_parsed_doc_comment = false; loop { debug!("parse_outer_attributes: self.token={:?}", self.token); match self.token { token::Pound => { - attrs.push(self.parse_attribute(false)?); + let inner_error_reason = if just_parsed_doc_comment { + "an inner attribute is not permitted following an outer doc comment" + } else if !attrs.is_empty() { + "an inner attribute is not permitted following an outer attribute" + } else { + DEFAULT_UNEXPECTED_INNER_ATTR_ERR_MSG + }; + let inner_parse_policy = + InnerAttributeParsePolicy::NotPermitted { reason: inner_error_reason }; + attrs.push(self.parse_attribute_with_inner_parse_policy(inner_parse_policy)?); + just_parsed_doc_comment = false; } token::DocComment(s) => { let attr = ::attr::mk_sugared_doc_attr( - attr::mk_attr_id(), - self.id_to_interned_str(ast::Ident::with_empty_ctxt(s)), - self.span.lo, - self.span.hi - ); + attr::mk_attr_id(), + self.id_to_interned_str(ast::Ident::with_empty_ctxt(s)), + self.span.lo, + self.span.hi + ); if attr.node.style != ast::AttrStyle::Outer { let mut err = self.fatal("expected outer doc comment"); err.note("inner doc comments like this (starting with \ @@ -43,6 +63,7 @@ impl<'a> Parser<'a> { } attrs.push(attr); self.bump(); + just_parsed_doc_comment = true; } _ => break, } @@ -55,26 +76,46 @@ impl<'a> Parser<'a> { /// If permit_inner is true, then a leading `!` indicates an inner /// attribute pub fn parse_attribute(&mut self, permit_inner: bool) -> PResult<'a, ast::Attribute> { - debug!("parse_attributes: permit_inner={:?} self.token={:?}", + debug!("parse_attribute: permit_inner={:?} self.token={:?}", permit_inner, self.token); + let inner_parse_policy = if permit_inner { + InnerAttributeParsePolicy::Permitted + } else { + InnerAttributeParsePolicy::NotPermitted + { reason: DEFAULT_UNEXPECTED_INNER_ATTR_ERR_MSG } + }; + self.parse_attribute_with_inner_parse_policy(inner_parse_policy) + } + + /// The same as `parse_attribute`, except it takes in an `InnerAttributeParsePolicy` + /// that prescribes how to handle inner attributes. + fn parse_attribute_with_inner_parse_policy(&mut self, + inner_parse_policy: InnerAttributeParsePolicy) + -> PResult<'a, ast::Attribute> { + debug!("parse_attribute_with_inner_parse_policy: inner_parse_policy={:?} self.token={:?}", + inner_parse_policy, + self.token); let (span, value, mut style) = match self.token { token::Pound => { let lo = self.span.lo; self.bump(); - if permit_inner { + if inner_parse_policy == InnerAttributeParsePolicy::Permitted { self.expected_tokens.push(TokenType::Token(token::Not)); } let style = if self.token == token::Not { self.bump(); - if !permit_inner { + if let InnerAttributeParsePolicy::NotPermitted { reason } = inner_parse_policy + { let span = self.span; self.diagnostic() - .struct_span_err(span, - "an inner attribute is not permitted in this context") - .help("place inner attribute at the top of the module or \ - block") + .struct_span_err(span, reason) + .note("inner attributes and doc comments, like `#![no_std]` or \ + `//! My crate`, annotate the item enclosing them, and are \ + usually found at the beginning of source files. Outer \ + attributes and doc comments, like `#[test]` and + `/// My function`, annotate the item following them.") .emit() } ast::AttrStyle::Inner @@ -95,7 +136,8 @@ impl<'a> Parser<'a> { } }; - if permit_inner && self.token == token::Semi { + if inner_parse_policy == InnerAttributeParsePolicy::Permitted && + self.token == token::Semi { self.bump(); self.span_warn(span, "this inner attribute syntax is deprecated. The new syntax is \ diff --git a/src/test/parse-fail/inner-attr-after-doc-comment.rs b/src/test/parse-fail/inner-attr-after-doc-comment.rs new file mode 100644 index 00000000000..ed8342d9f5a --- /dev/null +++ b/src/test/parse-fail/inner-attr-after-doc-comment.rs @@ -0,0 +1,20 @@ +// Copyright 2016 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// compile-flags: -Z parse-only + +#![feature(lang_items)] +/** + * My module + */ + +#![recursion_limit="100"] +//~^ ERROR an inner attribute is not permitted following an outer doc comment +fn main() {} diff --git a/src/test/parse-fail/inner-attr.rs b/src/test/parse-fail/inner-attr.rs new file mode 100644 index 00000000000..8cebda66445 --- /dev/null +++ b/src/test/parse-fail/inner-attr.rs @@ -0,0 +1,16 @@ +// Copyright 2016 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// compile-flags: -Z parse-only + +#[feature(lang_items)] + +#![recursion_limit="100"] //~ ERROR an inner attribute is not permitted following an outer attribute +fn main() {} -- cgit 1.4.1-3-g733a5 From 27a18b127f4f02dbbdf12eec03b817ed3c3f0f68 Mon Sep 17 00:00:00 2001 From: Jeffrey Seyfried Date: Sat, 16 Jul 2016 20:41:43 +0000 Subject: macros: Fix bug in statement matchers --- src/libsyntax/parse/parser.rs | 80 +++++++++++++++++-------------------------- 1 file changed, 31 insertions(+), 49 deletions(-) (limited to 'src/libsyntax/parse') diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index 4656ba03e21..125f1abb062 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -3789,13 +3789,8 @@ impl<'a> Parser<'a> { /// Parse a statement. This stops just before trailing semicolons on everything but items. /// e.g. a `StmtKind::Semi` parses to a `StmtKind::Expr`, leaving the trailing `;` unconsumed. - /// - /// Also, if a macro begins an expression statement, this only parses the macro. For example, - /// ```rust - /// vec![1].into_iter(); //< `parse_stmt` only parses the "vec![1]" - /// ``` pub fn parse_stmt(&mut self) -> PResult<'a, Option> { - Ok(self.parse_stmt_()) + Ok(self.parse_stmt_(true)) } // Eat tokens until we can be relatively sure we reached the end of the @@ -3859,15 +3854,15 @@ impl<'a> Parser<'a> { } } - fn parse_stmt_(&mut self) -> Option { - self.parse_stmt_without_recovery().unwrap_or_else(|mut e| { + fn parse_stmt_(&mut self, macro_expanded: bool) -> Option { + self.parse_stmt_without_recovery(macro_expanded).unwrap_or_else(|mut e| { e.emit(); self.recover_stmt_(SemiColonMode::Break); None }) } - fn parse_stmt_without_recovery(&mut self) -> PResult<'a, Option> { + fn parse_stmt_without_recovery(&mut self, macro_expanded: bool) -> PResult<'a, Option> { maybe_whole!(Some deref self, NtStmt); let attrs = self.parse_outer_attributes()?; @@ -3930,10 +3925,34 @@ impl<'a> Parser<'a> { if id.name == keywords::Invalid.name() { let mac = spanned(lo, hi, Mac_ { path: pth, tts: tts }); + let node = if delim == token::Brace || + self.token == token::Semi || self.token == token::Eof { + StmtKind::Mac(P((mac, style, attrs.into()))) + } + // We used to incorrectly stop parsing macro-expanded statements here. + // If the next token will be an error anyway but could have parsed with the + // earlier behavior, stop parsing here and emit a warning to avoid breakage. + else if macro_expanded && self.token.can_begin_expr() && match self.token { + // These can continue an expression, so we can't stop parsing and warn. + token::OpenDelim(token::Paren) | token::OpenDelim(token::Bracket) | + token::BinOp(token::Minus) | token::BinOp(token::Star) | + token::BinOp(token::And) | token::BinOp(token::Or) | + token::AndAnd | token::OrOr | + token::DotDot | token::DotDotDot => false, + _ => true, + } { + self.warn_missing_semicolon(); + StmtKind::Mac(P((mac, style, attrs.into()))) + } else { + let e = self.mk_mac_expr(lo, hi, mac.node, ThinVec::new()); + let e = self.parse_dot_or_call_expr_with(e, lo, attrs.into())?; + let e = self.parse_assoc_expr_with(0, LhsExpr::AlreadyParsed(e))?; + StmtKind::Expr(e) + }; Stmt { id: ast::DUMMY_NODE_ID, - node: StmtKind::Mac(P((mac, style, attrs.into()))), span: mk_sp(lo, hi), + node: node, } } else { // if it has a special ident, it's definitely an item @@ -4061,49 +4080,12 @@ impl<'a> Parser<'a> { } /// Parse a statement, including the trailing semicolon. - /// This parses expression statements that begin with macros correctly (c.f. `parse_stmt`). pub fn parse_full_stmt(&mut self, macro_expanded: bool) -> PResult<'a, Option> { - let mut stmt = match self.parse_stmt_() { + let mut stmt = match self.parse_stmt_(macro_expanded) { Some(stmt) => stmt, None => return Ok(None), }; - if let StmtKind::Mac(mac) = stmt.node { - if mac.1 != MacStmtStyle::NoBraces || - self.token == token::Semi || self.token == token::Eof { - stmt.node = StmtKind::Mac(mac); - } else { - // We used to incorrectly stop parsing macro-expanded statements here. - // If the next token will be an error anyway but could have parsed with the - // earlier behavior, stop parsing here and emit a warning to avoid breakage. - if macro_expanded && self.token.can_begin_expr() && match self.token { - // These tokens can continue an expression, so we can't stop parsing and warn. - token::OpenDelim(token::Paren) | token::OpenDelim(token::Bracket) | - token::BinOp(token::Minus) | token::BinOp(token::Star) | - token::BinOp(token::And) | token::BinOp(token::Or) | - token::AndAnd | token::OrOr | - token::DotDot | token::DotDotDot => false, - _ => true, - } { - self.warn_missing_semicolon(); - stmt.node = StmtKind::Mac(mac); - return Ok(Some(stmt)); - } - - let (mac, _style, attrs) = mac.unwrap(); - let e = self.mk_mac_expr(stmt.span.lo, stmt.span.hi, mac.node, ThinVec::new()); - let e = self.parse_dot_or_call_expr_with(e, stmt.span.lo, attrs)?; - let e = self.parse_assoc_expr_with(0, LhsExpr::AlreadyParsed(e))?; - stmt.node = StmtKind::Expr(e); - } - } - - stmt = self.handle_trailing_semicolon(stmt, macro_expanded)?; - Ok(Some(stmt)) - } - - fn handle_trailing_semicolon(&mut self, mut stmt: Stmt, macro_expanded: bool) - -> PResult<'a, Stmt> { match stmt.node { StmtKind::Expr(ref expr) if self.token != token::Eof => { // expression without semicolon @@ -4133,7 +4115,7 @@ impl<'a> Parser<'a> { } stmt.span.hi = self.last_span.hi; - Ok(stmt) + Ok(Some(stmt)) } fn warn_missing_semicolon(&self) { -- cgit 1.4.1-3-g733a5 From 76ed4456224f6bf7dc35dc72cc1c2cc93186285a Mon Sep 17 00:00:00 2001 From: Jeffrey Seyfried Date: Wed, 22 Jun 2016 08:03:42 +0000 Subject: Clean up and encapsulate `syntax::ext::mtwt` --- src/librustc_driver/driver.rs | 7 +- src/librustc_driver/pretty.rs | 2 +- src/librustc_resolve/assign_ids.rs | 6 +- src/librustc_resolve/lib.rs | 20 +++-- src/libsyntax/ast.rs | 22 +----- src/libsyntax/ext/expand.rs | 44 ++++------- src/libsyntax/ext/mtwt.rs | 155 ++++++++++++++++++------------------- src/libsyntax/parse/token.rs | 5 -- 8 files changed, 110 insertions(+), 151 deletions(-) (limited to 'src/libsyntax/parse') diff --git a/src/librustc_driver/driver.rs b/src/librustc_driver/driver.rs index 3b1124a911e..6b141ca15c0 100644 --- a/src/librustc_driver/driver.rs +++ b/src/librustc_driver/driver.rs @@ -479,9 +479,8 @@ pub fn phase_1_parse_input<'a>(sess: &'a Session, input: &Input) -> PResult<'a, ast::Crate> { // These may be left in an incoherent state after a previous compile. - // `clear_tables` and `clear_ident_interner` can be used to free - // memory, but they do not restore the initial state. - syntax::ext::mtwt::reset_tables(); + syntax::ext::mtwt::reset_hygiene_data(); + // `clear_ident_interner` can be used to free memory, but it does not restore the initial state. token::reset_ident_interner(); let continue_after_error = sess.opts.continue_parse_after_error; sess.diagnostic().set_continue_after_error(continue_after_error); @@ -763,7 +762,7 @@ pub fn phase_2_configure_and_expand<'a, F>(sess: &Session, // Discard MTWT tables that aren't required past lowering to HIR. if !keep_mtwt_tables(sess) { - syntax::ext::mtwt::clear_tables(); + syntax::ext::mtwt::reset_hygiene_data(); } Ok(ExpansionResult { diff --git a/src/librustc_driver/pretty.rs b/src/librustc_driver/pretty.rs index baac455a25f..14476cc997f 100644 --- a/src/librustc_driver/pretty.rs +++ b/src/librustc_driver/pretty.rs @@ -456,7 +456,7 @@ impl<'ast> pprust::PpAnn for HygieneAnnotation<'ast> { pp::space(&mut s.s)?; // FIXME #16420: this doesn't display the connections // between syntax contexts - s.synth_comment(format!("{}#{}", nm, ctxt.0)) + s.synth_comment(format!("{}{:?}", nm, ctxt)) } pprust::NodeName(&ast::Name(nm)) => { pp::space(&mut s.s)?; diff --git a/src/librustc_resolve/assign_ids.rs b/src/librustc_resolve/assign_ids.rs index d4465822229..77facbfb617 100644 --- a/src/librustc_resolve/assign_ids.rs +++ b/src/librustc_resolve/assign_ids.rs @@ -11,7 +11,7 @@ use Resolver; use rustc::session::Session; use syntax::ast; -use syntax::ext::mtwt; +use syntax::ext::mtwt::Mark; use syntax::fold::{self, Folder}; use syntax::ptr::P; use syntax::util::move_map::MoveMap; @@ -31,7 +31,7 @@ impl<'a> Resolver<'a> { struct NodeIdAssigner<'a> { sess: &'a Session, - macros_at_scope: &'a mut HashMap>, + macros_at_scope: &'a mut HashMap>, } impl<'a> Folder for NodeIdAssigner<'a> { @@ -49,7 +49,7 @@ impl<'a> Folder for NodeIdAssigner<'a> { block.stmts = block.stmts.move_flat_map(|stmt| { if let ast::StmtKind::Item(ref item) = stmt.node { if let ast::ItemKind::Mac(..) = item.node { - macros.push(mtwt::outer_mark(item.ident.ctxt)); + macros.push(item.ident.ctxt.data().outer_mark); return None; } } diff --git a/src/librustc_resolve/lib.rs b/src/librustc_resolve/lib.rs index 2535c264ef8..36cd2ef6002 100644 --- a/src/librustc_resolve/lib.rs +++ b/src/librustc_resolve/lib.rs @@ -53,7 +53,7 @@ use rustc::ty::subst::{ParamSpace, FnSpace, TypeSpace}; use rustc::hir::{Freevar, FreevarMap, TraitCandidate, TraitMap, GlobMap}; use rustc::util::nodemap::{NodeMap, NodeSet, FnvHashMap, FnvHashSet}; -use syntax::ext::mtwt; +use syntax::ext::mtwt::Mark; use syntax::ast::{self, FloatTy}; use syntax::ast::{CRATE_NODE_ID, Name, NodeId, CrateNum, IntTy, UintTy}; use syntax::parse::token::{self, keywords}; @@ -654,7 +654,7 @@ enum RibKind<'a> { ModuleRibKind(Module<'a>), // We passed through a `macro_rules!` statement with the given expansion - MacroDefinition(ast::Mrk), + MacroDefinition(Mark), } #[derive(Copy, Clone)] @@ -933,7 +933,7 @@ pub struct Resolver<'a> { // Maps the node id of a statement to the expansions of the `macro_rules!`s // immediately above the statement (if appropriate). - macros_at_scope: HashMap>, + macros_at_scope: HashMap>, graph_root: Module<'a>, @@ -1434,10 +1434,9 @@ impl<'a> Resolver<'a> { if let MacroDefinition(mac) = self.get_ribs(ns)[i].kind { // If an invocation of this macro created `ident`, give up on `ident` // and switch to `ident`'s source from the macro definition. - if let Some((source_ident, source_macro)) = mtwt::source(ident) { - if mac == source_macro { - ident = source_ident; - } + let (source_ctxt, source_macro) = ident.ctxt.source(); + if source_macro == mac { + ident.ctxt = source_ctxt; } } } @@ -1585,10 +1584,9 @@ impl<'a> Resolver<'a> { MacroDefinition(mac) => { // If an invocation of this macro created `ident`, give up on `ident` // and switch to `ident`'s source from the macro definition. - if let Some((source_ident, source_macro)) = mtwt::source(ident) { - if mac == source_macro { - ident = source_ident; - } + let (source_ctxt, source_macro) = ident.ctxt.source(); + if source_macro == mac { + ident.ctxt = source_ctxt; } } _ => { diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs index 6b662c6779a..1f716923a16 100644 --- a/src/libsyntax/ast.rs +++ b/src/libsyntax/ast.rs @@ -19,6 +19,7 @@ pub use util::ThinVec; use syntax_pos::{mk_sp, Span, DUMMY_SP, ExpnId}; use codemap::{respan, Spanned}; use abi::Abi; +use ext::mtwt::SyntaxContext; use parse::token::{self, keywords, InternedString}; use print::pprust; use ptr::P; @@ -33,15 +34,6 @@ use serialize::{Encodable, Decodable, Encoder, Decoder}; #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct Name(pub u32); -/// A SyntaxContext represents a chain of macro-expandings -/// and renamings. Each macro expansion corresponds to -/// a fresh u32. This u32 is a reference to a table stored -/// in thread-local storage. -/// The special value EMPTY_CTXT is used to indicate an empty -/// syntax context. -#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, RustcEncodable, RustcDecodable)] -pub struct SyntaxContext(pub u32); - /// An identifier contains a Name (index into the interner /// table) and a SyntaxContext to track renaming and /// macro expansion per Flatt et al., "Macros That Work Together" @@ -81,20 +73,15 @@ impl Decodable for Name { } } -pub const EMPTY_CTXT : SyntaxContext = SyntaxContext(0); - impl Ident { - pub fn new(name: Name, ctxt: SyntaxContext) -> Ident { - Ident {name: name, ctxt: ctxt} - } pub const fn with_empty_ctxt(name: Name) -> Ident { - Ident {name: name, ctxt: EMPTY_CTXT} + Ident { name: name, ctxt: SyntaxContext::empty() } } } impl fmt::Debug for Ident { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "{}#{}", self.name, self.ctxt.0) + write!(f, "{}{:?}", self.name, self.ctxt) } } @@ -116,9 +103,6 @@ impl Decodable for Ident { } } -/// A mark represents a unique id associated with a macro expansion -pub type Mrk = u32; - #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Copy)] pub struct Lifetime { pub id: NodeId, diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index 3e9837a6995..767d1ddb8e2 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -9,20 +9,19 @@ // except according to those terms. use ast::{Block, Crate, Ident, Mac_, Name, PatKind}; -use ast::{MacStmtStyle, Mrk, Stmt, StmtKind, ItemKind}; +use ast::{MacStmtStyle, Stmt, StmtKind, ItemKind}; use ast; -use attr::HasAttrs; -use ext::mtwt; -use attr; +use ext::mtwt::Mark; +use attr::{self, HasAttrs}; use attr::AttrMetaMethods; -use codemap::{dummy_spanned, Spanned, ExpnInfo, NameAndSpan, MacroBang, MacroAttribute}; +use codemap::{dummy_spanned, ExpnInfo, NameAndSpan, MacroBang, MacroAttribute}; use syntax_pos::{self, Span, ExpnId}; use config::StripUnconfigured; use ext::base::*; use feature_gate::{self, Features}; use fold; use fold::*; -use parse::token::{fresh_mark, intern, keywords}; +use parse::token::{intern, keywords}; use ptr::P; use tokenstream::TokenTree; use util::small_vector::SmallVector; @@ -130,9 +129,9 @@ fn expand_mac_invoc(mac: ast::Mac, ident: Option, attrs: Vec(path: &ast::Path, ident: Option, tts: Vec, mark: Mrk, + fn mac_result<'a>(path: &ast::Path, ident: Option, tts: Vec, mark: Mark, attrs: Vec, call_site: Span, fld: &'a mut MacroExpander) -> Option> { // Detect use of feature-gated or invalid attributes on macro invoations @@ -708,30 +707,17 @@ pub fn expand_crate(mut cx: ExtCtxt, return (ret, cx.syntax_env.names); } -// HYGIENIC CONTEXT EXTENSION: -// all of these functions are for walking over -// ASTs and making some change to the context of every -// element that has one. a CtxtFn is a trait-ified -// version of a closure in (SyntaxContext -> SyntaxContext). -// the ones defined here include: -// Marker - add a mark to a context - // A Marker adds the given mark to the syntax context and // sets spans' `expn_id` to the given expn_id (unless it is `None`). -struct Marker { mark: Mrk, expn_id: Option } +struct Marker { mark: Mark, expn_id: Option } impl Folder for Marker { - fn fold_ident(&mut self, id: Ident) -> Ident { - ast::Ident::new(id.name, mtwt::apply_mark(self.mark, id.ctxt)) - } - fn fold_mac(&mut self, Spanned {node, span}: ast::Mac) -> ast::Mac { - Spanned { - node: Mac_ { - path: self.fold_path(node.path), - tts: self.fold_tts(&node.tts), - }, - span: self.new_span(span), - } + fn fold_ident(&mut self, mut ident: Ident) -> Ident { + ident.ctxt = ident.ctxt.apply_mark(self.mark); + ident + } + fn fold_mac(&mut self, mac: ast::Mac) -> ast::Mac { + noop_fold_mac(mac, self) } fn new_span(&mut self, mut span: Span) -> Span { @@ -743,7 +729,7 @@ impl Folder for Marker { } // apply a given mark to the given token trees. Used prior to expansion of a macro. -fn mark_tts(tts: &[TokenTree], m: Mrk) -> Vec { +fn mark_tts(tts: &[TokenTree], m: Mark) -> Vec { noop_fold_tts(tts, &mut Marker{mark:m, expn_id: None}) } diff --git a/src/libsyntax/ext/mtwt.rs b/src/libsyntax/ext/mtwt.rs index d2f6df9d5db..c4af4a1f85b 100644 --- a/src/libsyntax/ext/mtwt.rs +++ b/src/libsyntax/ext/mtwt.rs @@ -15,107 +15,104 @@ //! and definition contexts*. J. Funct. Program. 22, 2 (March 2012), 181-216. //! DOI=10.1017/S0956796812000093 http://dx.doi.org/10.1017/S0956796812000093 -pub use self::SyntaxContext_::*; - -use ast::{Ident, Mrk, SyntaxContext}; - use std::cell::RefCell; use std::collections::HashMap; +use std::fmt; -/// The SCTable contains a table of SyntaxContext_'s. It -/// represents a flattened tree structure, to avoid having -/// managed pointers everywhere (that caused an ICE). -/// The `marks` ensures that adding the same mark to the -/// same context gives you back the same context as before. -pub struct SCTable { - table: RefCell>, - marks: RefCell>, -} +/// A SyntaxContext represents a chain of macro expansions (represented by marks). +#[derive(Clone, Copy, PartialEq, Eq, Hash, RustcEncodable, RustcDecodable, Default)] +pub struct SyntaxContext(u32); -#[derive(PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy, Clone)] -pub enum SyntaxContext_ { - EmptyCtxt, - Mark (Mrk,SyntaxContext), +#[derive(Copy, Clone)] +pub struct SyntaxContextData { + pub outer_mark: Mark, + pub prev_ctxt: SyntaxContext, } -/// Extend a syntax context with a given mark -pub fn apply_mark(m: Mrk, ctxt: SyntaxContext) -> SyntaxContext { - with_sctable(|table| apply_mark_internal(m, ctxt, table)) -} +/// A mark represents a unique id associated with a macro expansion. +#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, Default)] +pub struct Mark(u32); -/// Extend a syntax context with a given mark and sctable (explicit memoization) -fn apply_mark_internal(m: Mrk, ctxt: SyntaxContext, table: &SCTable) -> SyntaxContext { - let ctxts = &mut *table.table.borrow_mut(); - match ctxts[ctxt.0 as usize] { - // Applying the same mark twice is a no-op. - Mark(outer_mark, prev_ctxt) if outer_mark == m => return prev_ctxt, - _ => *table.marks.borrow_mut().entry((ctxt, m)).or_insert_with(|| { - SyntaxContext(idx_push(ctxts, Mark(m, ctxt))) - }), +impl Mark { + pub fn fresh() -> Self { + HygieneData::with(|data| { + let next_mark = Mark(data.next_mark.0 + 1); + ::std::mem::replace(&mut data.next_mark, next_mark) + }) } } -/// Fetch the SCTable from TLS, create one if it doesn't yet exist. -pub fn with_sctable(op: F) -> T where - F: FnOnce(&SCTable) -> T, -{ - thread_local!(static SCTABLE_KEY: SCTable = new_sctable_internal()); - SCTABLE_KEY.with(move |slot| op(slot)) +struct HygieneData { + syntax_contexts: Vec, + markings: HashMap<(SyntaxContext, Mark), SyntaxContext>, + next_mark: Mark, } -// Make a fresh syntax context table with EmptyCtxt in slot zero. -fn new_sctable_internal() -> SCTable { - SCTable { - table: RefCell::new(vec![EmptyCtxt]), - marks: RefCell::new(HashMap::new()), +impl HygieneData { + fn new() -> Self { + HygieneData { + syntax_contexts: vec![SyntaxContextData { + outer_mark: Mark(0), // the null mark + prev_ctxt: SyntaxContext(0), // the empty context + }], + markings: HashMap::new(), + next_mark: Mark(1), + } } -} -/// Clear the tables from TLD to reclaim memory. -pub fn clear_tables() { - with_sctable(|table| { - *table.table.borrow_mut() = Vec::new(); - *table.marks.borrow_mut() = HashMap::new(); - }); + fn with T>(f: F) -> T { + thread_local! { + static HYGIENE_DATA: RefCell = RefCell::new(HygieneData::new()); + } + HYGIENE_DATA.with(|data| f(&mut *data.borrow_mut())) + } } -/// Reset the tables to their initial state -pub fn reset_tables() { - with_sctable(|table| { - *table.table.borrow_mut() = vec![EmptyCtxt]; - *table.marks.borrow_mut() = HashMap::new(); - }); +pub fn reset_hygiene_data() { + HygieneData::with(|data| *data = HygieneData::new()) } -/// Add a value to the end of a vec, return its index -fn idx_push(vec: &mut Vec, val: T) -> u32 { - vec.push(val); - (vec.len() - 1) as u32 -} +impl SyntaxContext { + pub const fn empty() -> Self { + SyntaxContext(0) + } + + pub fn data(self) -> SyntaxContextData { + HygieneData::with(|data| data.syntax_contexts[self.0 as usize]) + } -/// Return the outer mark for a context with a mark at the outside. -/// FAILS when outside is not a mark. -pub fn outer_mark(ctxt: SyntaxContext) -> Mrk { - with_sctable(|sctable| { - match (*sctable.table.borrow())[ctxt.0 as usize] { - Mark(mrk, _) => mrk, - _ => panic!("can't retrieve outer mark when outside is not a mark") + /// Extend a syntax context with a given mark + pub fn apply_mark(self, mark: Mark) -> SyntaxContext { + // Applying the same mark twice is a no-op + let ctxt_data = self.data(); + if mark == ctxt_data.outer_mark { + return ctxt_data.prev_ctxt; } - }) + + HygieneData::with(|data| { + let syntax_contexts = &mut data.syntax_contexts; + *data.markings.entry((self, mark)).or_insert_with(|| { + syntax_contexts.push(SyntaxContextData { + outer_mark: mark, + prev_ctxt: self, + }); + SyntaxContext(syntax_contexts.len() as u32 - 1) + }) + }) + } + + /// If `ident` is macro expanded, return the source ident from the macro definition + /// and the mark of the expansion that created the macro definition. + pub fn source(self) -> (Self /* source context */, Mark /* source macro */) { + let macro_def_ctxt = self.data().prev_ctxt.data(); + (macro_def_ctxt.prev_ctxt, macro_def_ctxt.outer_mark) + } } -/// If `ident` is macro expanded, return the source ident from the macro definition -/// and the mark of the expansion that created the macro definition. -pub fn source(ident: Ident) -> Option<(Ident /* source ident */, Mrk /* source macro */)> { - with_sctable(|sctable| { - let ctxts = sctable.table.borrow(); - if let Mark(_expansion_mark, macro_ctxt) = ctxts[ident.ctxt.0 as usize] { - if let Mark(definition_mark, orig_ctxt) = ctxts[macro_ctxt.0 as usize] { - return Some((Ident::new(ident.name, orig_ctxt), definition_mark)); - } - } - None - }) +impl fmt::Debug for SyntaxContext { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "#{}", self.0) + } } #[cfg(test)] diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index fe9d3ef7c23..f0a6f8edeec 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -633,8 +633,3 @@ pub fn fresh_name(src: ast::Ident) -> ast::Name { /*let num = rand::thread_rng().gen_uint_range(0,0xffff); gensym(format!("{}_{}",ident_to_string(src),num))*/ } - -// create a fresh mark. -pub fn fresh_mark() -> ast::Mrk { - gensym("mark").0 -} -- cgit 1.4.1-3-g733a5