From d4488b7df97e62bfeed8c30b1922ce55ff254594 Mon Sep 17 00:00:00 2001 From: Jeffrey Seyfried Date: Tue, 28 Mar 2017 05:32:43 +0000 Subject: Simplify `hygiene::Mark` application, and remove variant `Token::SubstNt` in favor of `quoted::TokenTree::MetaVar`. --- src/libsyntax/ext/base.rs | 14 --------- src/libsyntax/ext/expand.rs | 36 ++++++++++------------ src/libsyntax/ext/quote.rs | 4 +-- src/libsyntax/ext/tt/macro_parser.rs | 24 ++++++--------- src/libsyntax/ext/tt/macro_rules.rs | 11 ++++--- src/libsyntax/ext/tt/quoted.rs | 13 +++++--- src/libsyntax/ext/tt/transcribe.rs | 60 ++++++++++++++++++++---------------- src/libsyntax/fold.rs | 1 - src/libsyntax/parse/lexer/mod.rs | 42 +++++++++++++------------ src/libsyntax/parse/mod.rs | 13 +++++--- src/libsyntax/parse/parser.rs | 5 ++- src/libsyntax/parse/token.rs | 3 -- src/libsyntax/print/pprust.rs | 1 - 13 files changed, 109 insertions(+), 118 deletions(-) (limited to 'src/libsyntax') diff --git a/src/libsyntax/ext/base.rs b/src/libsyntax/ext/base.rs index 8089fad5f36..af5eabf06f8 100644 --- a/src/libsyntax/ext/base.rs +++ b/src/libsyntax/ext/base.rs @@ -903,17 +903,3 @@ pub fn get_exprs_from_tts(cx: &mut ExtCtxt, } Some(es) } - -pub struct ChangeSpan { - pub span: Span -} - -impl Folder for ChangeSpan { - fn new_span(&mut self, _sp: Span) -> Span { - self.span - } - - fn fold_mac(&mut self, mac: ast::Mac) -> ast::Mac { - fold::noop_fold_mac(mac, self) - } -} diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index f8a26287bd4..11efef45499 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -16,7 +16,7 @@ use config::{is_test_or_bench, StripUnconfigured}; use errors::FatalError; use ext::base::*; use ext::derive::{add_derived_markers, collect_derives}; -use ext::hygiene::Mark; +use ext::hygiene::{Mark, SyntaxContext}; use ext::placeholders::{placeholder, PlaceholderExpander}; use feature_gate::{self, Features, is_builtin_attr}; use fold; @@ -470,7 +470,6 @@ impl<'a, 'b> MacroExpander<'a, 'b> { Ok(()) }; - let marked_tts = noop_fold_tts(mac.node.stream(), &mut Marker(mark)); let opt_expanded = match *ext { SyntaxExtension::DeclMacro(ref expand, def_span) => { if let Err(msg) = validate_and_set_expn_info(def_span.map(|(_, s)| s), @@ -478,7 +477,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> { self.cx.span_err(path.span, &msg); return kind.dummy(span); } - kind.make_from(expand.expand(self.cx, span, marked_tts)) + kind.make_from(expand.expand(self.cx, span, mac.node.stream())) } NormalTT(ref expandfun, def_info, allow_internal_unstable) => { @@ -487,7 +486,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> { self.cx.span_err(path.span, &msg); return kind.dummy(span); } - kind.make_from(expandfun.expand(self.cx, span, marked_tts)) + kind.make_from(expandfun.expand(self.cx, span, mac.node.stream())) } IdentTT(ref expander, tt_span, allow_internal_unstable) => { @@ -506,7 +505,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> { } }); - let input: Vec<_> = marked_tts.into_trees().collect(); + let input: Vec<_> = mac.node.stream().into_trees().collect(); kind.make_from(expander.expand(self.cx, span, ident, input)) } @@ -541,21 +540,17 @@ impl<'a, 'b> MacroExpander<'a, 'b> { }, }); - let tok_result = expandfun.expand(self.cx, span, marked_tts); + let tok_result = expandfun.expand(self.cx, span, mac.node.stream()); Some(self.parse_expansion(tok_result, kind, path, span)) } }; - let expanded = if let Some(expanded) = opt_expanded { - expanded - } else { + unwrap_or!(opt_expanded, { let msg = format!("non-{kind} macro in {kind} position: {name}", name = path.segments[0].identifier.name, kind = kind.name()); self.cx.span_err(path.span, &msg); - return kind.dummy(span); - }; - - expanded.fold_with(&mut Marker(mark)) + kind.dummy(span) + }) } /// Expand a derive invocation. Returns the result of expansion. @@ -621,8 +616,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> { } }; parser.ensure_complete_parse(path, kind.name(), span); - // FIXME better span info - expansion.fold_with(&mut ChangeSpan { span: span }) + expansion } } @@ -673,7 +667,9 @@ impl<'a> Parser<'a> { if self.token != token::Eof { let msg = format!("macro expansion ignores token `{}` and any following", self.this_token_to_string()); - let mut err = self.diagnostic().struct_span_err(self.span, &msg); + let mut def_site_span = self.span; + def_site_span.ctxt = SyntaxContext::empty(); // Avoid emitting backtrace info twice. + let mut err = self.diagnostic().struct_span_err(def_site_span, &msg); let msg = format!("caused by the macro expansion here; the usage \ of `{}!` is likely invalid in {} context", macro_path, kind_name); @@ -787,12 +783,12 @@ fn stream_for_item(item: &Annotatable, parse_sess: &ParseSess) -> TokenStream { Annotatable::TraitItem(ref ti) => pprust::trait_item_to_string(ti), Annotatable::ImplItem(ref ii) => pprust::impl_item_to_string(ii), }; - string_to_stream(text, parse_sess) + string_to_stream(text, parse_sess, item.span()) } -fn string_to_stream(text: String, parse_sess: &ParseSess) -> TokenStream { +fn string_to_stream(text: String, parse_sess: &ParseSess, span: Span) -> TokenStream { let filename = String::from(""); - filemap_to_stream(parse_sess, parse_sess.codemap().new_filemap(filename, text)) + filemap_to_stream(parse_sess, parse_sess.codemap().new_filemap(filename, text), Some(span)) } impl<'a, 'b> Folder for InvocationCollector<'a, 'b> { @@ -1070,7 +1066,7 @@ impl<'feat> ExpansionConfig<'feat> { } // A Marker adds the given mark to the syntax context. -struct Marker(Mark); +pub struct Marker(pub Mark); impl Folder for Marker { fn fold_ident(&mut self, mut ident: Ident) -> Ident { diff --git a/src/libsyntax/ext/quote.rs b/src/libsyntax/ext/quote.rs index f8fac847a05..314a97496f8 100644 --- a/src/libsyntax/ext/quote.rs +++ b/src/libsyntax/ext/quote.rs @@ -364,7 +364,7 @@ pub mod rt { fn parse_tts(&self, s: String) -> Vec { let source_name = "".to_owned(); - parse::parse_stream_from_source_str(source_name, s, self.parse_sess()) + parse::parse_stream_from_source_str(source_name, s, self.parse_sess(), None) .into_trees().collect() } } @@ -700,7 +700,7 @@ fn expr_mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P { token::Underscore => "Underscore", token::Eof => "Eof", - token::Whitespace | token::SubstNt(_) | token::Comment | token::Shebang(_) => { + token::Whitespace | token::Comment | token::Shebang(_) => { panic!("unhandled token in quote!"); } }; diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs index 61d8fc2941a..e877f1fedd4 100644 --- a/src/libsyntax/ext/tt/macro_parser.rs +++ b/src/libsyntax/ext/tt/macro_parser.rs @@ -158,15 +158,10 @@ pub type NamedParseResult = ParseResult>>; pub fn count_names(ms: &[TokenTree]) -> usize { ms.iter().fold(0, |count, elt| { count + match *elt { - TokenTree::Sequence(_, ref seq) => { - seq.num_captures - } - TokenTree::Delimited(_, ref delim) => { - count_names(&delim.tts) - } - TokenTree::MetaVarDecl(..) => { - 1 - } + TokenTree::Sequence(_, ref seq) => seq.num_captures, + TokenTree::Delimited(_, ref delim) => count_names(&delim.tts), + TokenTree::MetaVar(..) => 0, + TokenTree::MetaVarDecl(..) => 1, TokenTree::Token(..) => 0, } }) @@ -244,7 +239,7 @@ fn nameize>(sess: &ParseSess, ms: &[TokenTree], mut } } } - TokenTree::Token(..) => (), + TokenTree::MetaVar(..) | TokenTree::Token(..) => (), } Ok(()) @@ -409,12 +404,11 @@ fn inner_parse_loop(sess: &ParseSess, ei.idx = 0; cur_eis.push(ei); } - TokenTree::Token(_, ref t) => { - if token_name_eq(t, token) { - ei.idx += 1; - next_eis.push(ei); - } + TokenTree::Token(_, ref t) if token_name_eq(t, token) => { + ei.idx += 1; + next_eis.push(ei); } + TokenTree::Token(..) | TokenTree::MetaVar(..) => {} } } } diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs index 9c728c9f2eb..b732f47ce6a 100644 --- a/src/libsyntax/ext/tt/macro_rules.rs +++ b/src/libsyntax/ext/tt/macro_rules.rs @@ -120,7 +120,7 @@ fn generic_extension<'cx>(cx: &'cx mut ExtCtxt, _ => cx.span_bug(sp, "malformed macro rhs"), }; // rhs has holes ( `$id` and `$(...)` that need filled) - let tts = transcribe(&cx.parse_sess.span_diagnostic, Some(named_matches), rhs); + let tts = transcribe(cx, Some(named_matches), rhs); if cx.trace_macros() { trace_macros_note(cx, sp, format!("to `{}`", tts)); @@ -292,7 +292,7 @@ fn check_lhs_no_empty_seq(sess: &ParseSess, tts: &[quoted::TokenTree]) -> bool { use self::quoted::TokenTree; for tt in tts { match *tt { - TokenTree::Token(..) | TokenTree::MetaVarDecl(..) => (), + TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => (), TokenTree::Delimited(_, ref del) => if !check_lhs_no_empty_seq(sess, &del.tts) { return false; }, @@ -372,7 +372,7 @@ impl FirstSets { let mut first = TokenSet::empty(); for tt in tts.iter().rev() { match *tt { - TokenTree::Token(..) | TokenTree::MetaVarDecl(..) => { + TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => { first.replace_with(tt.clone()); } TokenTree::Delimited(span, ref delimited) => { @@ -432,7 +432,7 @@ impl FirstSets { for tt in tts.iter() { assert!(first.maybe_empty); match *tt { - TokenTree::Token(..) | TokenTree::MetaVarDecl(..) => { + TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => { first.add_one(tt.clone()); return first; } @@ -602,7 +602,7 @@ fn check_matcher_core(sess: &ParseSess, // First, update `last` so that it corresponds to the set // of NT tokens that might end the sequence `... token`. match *token { - TokenTree::Token(..) | TokenTree::MetaVarDecl(..) => { + TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => { let can_be_followed_by_any; if let Err(bad_frag) = has_legal_fragment_specifier(sess, features, token) { let msg = format!("invalid fragment specifier `{}`", bad_frag); @@ -872,6 +872,7 @@ fn is_legal_fragment_specifier(sess: &ParseSess, fn quoted_tt_to_string(tt: "ed::TokenTree) -> String { match *tt { quoted::TokenTree::Token(_, ref tok) => ::print::pprust::token_to_string(tok), + quoted::TokenTree::MetaVar(_, name) => format!("${}", name), quoted::TokenTree::MetaVarDecl(_, name, kind) => format!("${}:{}", name, kind), _ => panic!("unexpected quoted::TokenTree::{{Sequence or Delimited}} \ in follow set checker"), diff --git a/src/libsyntax/ext/tt/quoted.rs b/src/libsyntax/ext/tt/quoted.rs index fa65e9501c2..18056f60287 100644 --- a/src/libsyntax/ext/tt/quoted.rs +++ b/src/libsyntax/ext/tt/quoted.rs @@ -78,9 +78,11 @@ pub enum KleeneOp { pub enum TokenTree { Token(Span, token::Token), Delimited(Span, Rc), - /// A kleene-style repetition sequence with a span + /// A kleene-style repetition sequence Sequence(Span, Rc), - /// Matches a nonterminal. This is only used in the left hand side of MBE macros. + /// E.g. `$var` + MetaVar(Span, ast::Ident), + /// E.g. `$var:expr`. This is only used in the left hand side of MBE macros. MetaVarDecl(Span, ast::Ident /* name to bind */, ast::Ident /* kind of nonterminal */), } @@ -130,6 +132,7 @@ impl TokenTree { pub fn span(&self) -> Span { match *self { TokenTree::Token(sp, _) | + TokenTree::MetaVar(sp, _) | TokenTree::MetaVarDecl(sp, _, _) | TokenTree::Delimited(sp, _) | TokenTree::Sequence(sp, _) => sp, @@ -144,7 +147,7 @@ pub fn parse(input: tokenstream::TokenStream, expect_matchers: bool, sess: &Pars while let Some(tree) = trees.next() { let tree = parse_tree(tree, &mut trees, expect_matchers, sess); match tree { - TokenTree::Token(start_sp, token::SubstNt(ident)) if expect_matchers => { + TokenTree::MetaVar(start_sp, ident) if expect_matchers => { let span = match trees.next() { Some(tokenstream::TokenTree::Token(span, token::Colon)) => match trees.next() { Some(tokenstream::TokenTree::Token(end_sp, ref tok)) => match tok.ident() { @@ -199,13 +202,13 @@ fn parse_tree(tree: tokenstream::TokenTree, let ident = ast::Ident { name: Symbol::intern("$crate"), ..ident }; TokenTree::Token(span, token::Ident(ident)) } else { - TokenTree::Token(span, token::SubstNt(ident)) + TokenTree::MetaVar(span, ident) } } Some(tokenstream::TokenTree::Token(span, tok)) => { let msg = format!("expected identifier, found `{}`", pprust::token_to_string(&tok)); sess.span_diagnostic.span_err(span, &msg); - TokenTree::Token(span, token::SubstNt(keywords::Invalid.ident())) + TokenTree::MetaVar(span, keywords::Invalid.ident()) } None => TokenTree::Token(span, token::Dollar), }, diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/tt/transcribe.rs index 78e755e73fa..9438e2fb0e5 100644 --- a/src/libsyntax/ext/tt/transcribe.rs +++ b/src/libsyntax/ext/tt/transcribe.rs @@ -9,10 +9,12 @@ // except according to those terms. use ast::Ident; -use errors::Handler; +use ext::base::ExtCtxt; +use ext::expand::Marker; use ext::tt::macro_parser::{NamedMatch, MatchedSeq, MatchedNonterminal}; use ext::tt::quoted; -use parse::token::{self, SubstNt, Token, NtTT}; +use fold::noop_fold_tt; +use parse::token::{self, Token, NtTT}; use syntax_pos::{Span, DUMMY_SP}; use tokenstream::{TokenStream, TokenTree, Delimited}; use util::small_vector::SmallVector; @@ -61,9 +63,9 @@ impl Iterator for Frame { } /// This can do Macro-By-Example transcription. On the other hand, if -/// `src` contains no `TokenTree::{Sequence, Match}`s, or `SubstNt`s, `interp` can +/// `src` contains no `TokenTree::{Sequence, MetaVar, MetaVarDecl}`s, `interp` can /// (and should) be None. -pub fn transcribe(sp_diag: &Handler, +pub fn transcribe(cx: &ExtCtxt, interp: Option>>, src: Vec) -> TokenStream { @@ -120,22 +122,20 @@ pub fn transcribe(sp_diag: &Handler, &interpolations, &repeats) { LockstepIterSize::Unconstrained => { - panic!(sp_diag.span_fatal( - sp, /* blame macro writer */ + cx.span_fatal(sp, /* blame macro writer */ "attempted to repeat an expression \ containing no syntax \ - variables matched as repeating at this depth")); + variables matched as repeating at this depth"); } LockstepIterSize::Contradiction(ref msg) => { // FIXME #2887 blame macro invoker instead - panic!(sp_diag.span_fatal(sp, &msg[..])); + cx.span_fatal(sp, &msg[..]); } LockstepIterSize::Constraint(len, _) => { if len == 0 { if seq.op == quoted::KleeneOp::OneOrMore { // FIXME #2887 blame invoker - panic!(sp_diag.span_fatal(sp, - "this must repeat at least once")); + cx.span_fatal(sp, "this must repeat at least once"); } } else { repeats.push((0, len)); @@ -149,29 +149,37 @@ pub fn transcribe(sp_diag: &Handler, } } // FIXME #2887: think about span stuff here - quoted::TokenTree::Token(sp, SubstNt(ident)) => { - match lookup_cur_matched(ident, &interpolations, &repeats) { - None => result.push(TokenTree::Token(sp, SubstNt(ident)).into()), - Some(cur_matched) => if let MatchedNonterminal(ref nt) = *cur_matched { - match **nt { - NtTT(ref tt) => result.push(tt.clone().into()), - _ => { - let token = TokenTree::Token(sp, token::Interpolated(nt.clone())); - result.push(token.into()); - } + quoted::TokenTree::MetaVar(mut sp, ident) => { + if let Some(cur_matched) = lookup_cur_matched(ident, &interpolations, &repeats) { + if let MatchedNonterminal(ref nt) = *cur_matched { + if let NtTT(ref tt) = **nt { + result.push(tt.clone().into()); + } else { + sp.ctxt = sp.ctxt.apply_mark(cx.current_expansion.mark); + let token = TokenTree::Token(sp, token::Interpolated(nt.clone())); + result.push(token.into()); } } else { - panic!(sp_diag.span_fatal( - sp, /* blame the macro writer */ - &format!("variable '{}' is still repeating at this depth", ident))); + cx.span_fatal(sp, /* blame the macro writer */ + &format!("variable '{}' is still repeating at this depth", ident)); } + } else { + let ident = + Ident { ctxt: ident.ctxt.apply_mark(cx.current_expansion.mark), ..ident }; + sp.ctxt = sp.ctxt.apply_mark(cx.current_expansion.mark); + result.push(TokenTree::Token(sp, token::Dollar).into()); + result.push(TokenTree::Token(sp, token::Ident(ident)).into()); } } - quoted::TokenTree::Delimited(span, delimited) => { + quoted::TokenTree::Delimited(mut span, delimited) => { + span.ctxt = span.ctxt.apply_mark(cx.current_expansion.mark); stack.push(Frame::Delimited { forest: delimited, idx: 0, span: span }); result_stack.push(mem::replace(&mut result, Vec::new())); } - quoted::TokenTree::Token(span, tok) => result.push(TokenTree::Token(span, tok).into()), + quoted::TokenTree::Token(sp, tok) => { + let mut marker = Marker(cx.current_expansion.mark); + result.push(noop_fold_tt(TokenTree::Token(sp, tok), &mut marker).into()) + } quoted::TokenTree::MetaVarDecl(..) => panic!("unexpected `TokenTree::MetaVarDecl"), } } @@ -240,7 +248,7 @@ fn lockstep_iter_size(tree: "ed::TokenTree, size + lockstep_iter_size(tt, interpolations, repeats) }) }, - TokenTree::Token(_, SubstNt(name)) | TokenTree::MetaVarDecl(_, name, _) => + TokenTree::MetaVar(_, name) | TokenTree::MetaVarDecl(_, name, _) => match lookup_cur_matched(name, interpolations, repeats) { Some(matched) => match *matched { MatchedNonterminal(_) => LockstepIterSize::Unconstrained, diff --git a/src/libsyntax/fold.rs b/src/libsyntax/fold.rs index 4c6cf49a8db..2032aecacbb 100644 --- a/src/libsyntax/fold.rs +++ b/src/libsyntax/fold.rs @@ -588,7 +588,6 @@ pub fn noop_fold_token(t: token::Token, fld: &mut T) -> token::Token }; token::Interpolated(Rc::new(fld.fold_interpolated(nt))) } - token::SubstNt(ident) => token::SubstNt(fld.fold_ident(ident)), _ => t } } diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs index e2656bea483..afc1e583d69 100644 --- a/src/libsyntax/parse/lexer/mod.rs +++ b/src/libsyntax/parse/lexer/mod.rs @@ -66,14 +66,15 @@ pub struct StringReader<'a> { token: token::Token, span: Span, open_braces: Vec<(token::DelimToken, Span)>, -} - -fn mk_sp(lo: BytePos, hi: BytePos) -> Span { - Span { lo: lo, hi: hi, ctxt: NO_EXPANSION } + pub override_span: Option, } impl<'a> StringReader<'a> { - fn next_token(&mut self) -> TokenAndSpan { + fn mk_sp(&self, lo: BytePos, hi: BytePos) -> Span { + unwrap_or!(self.override_span, Span { lo: lo, hi: hi, ctxt: NO_EXPANSION}) + } + + fn next_token(&mut self) -> TokenAndSpan where Self: Sized { let res = self.try_next_token(); self.unwrap_or_abort(res) } @@ -175,6 +176,7 @@ impl<'a> StringReader<'a> { token: token::Eof, span: syntax_pos::DUMMY_SP, open_braces: Vec::new(), + override_span: None, } } @@ -229,12 +231,12 @@ impl<'a> StringReader<'a> { /// Report a fatal error spanning [`from_pos`, `to_pos`). fn fatal_span_(&self, from_pos: BytePos, to_pos: BytePos, m: &str) -> FatalError { - self.fatal_span(mk_sp(from_pos, to_pos), m) + self.fatal_span(self.mk_sp(from_pos, to_pos), m) } /// Report a lexical error spanning [`from_pos`, `to_pos`). fn err_span_(&self, from_pos: BytePos, to_pos: BytePos, m: &str) { - self.err_span(mk_sp(from_pos, to_pos), m) + self.err_span(self.mk_sp(from_pos, to_pos), m) } /// Report a lexical error spanning [`from_pos`, `to_pos`), appending an @@ -258,7 +260,7 @@ impl<'a> StringReader<'a> { for c in c.escape_default() { m.push(c) } - self.sess.span_diagnostic.struct_span_fatal(mk_sp(from_pos, to_pos), &m[..]) + self.sess.span_diagnostic.struct_span_fatal(self.mk_sp(from_pos, to_pos), &m[..]) } /// Report a lexical error spanning [`from_pos`, `to_pos`), appending an @@ -282,7 +284,7 @@ impl<'a> StringReader<'a> { for c in c.escape_default() { m.push(c) } - self.sess.span_diagnostic.struct_span_err(mk_sp(from_pos, to_pos), &m[..]) + self.sess.span_diagnostic.struct_span_err(self.mk_sp(from_pos, to_pos), &m[..]) } /// Report a lexical error spanning [`from_pos`, `to_pos`), appending the @@ -306,11 +308,11 @@ impl<'a> StringReader<'a> { None => { if self.is_eof() { self.peek_tok = token::Eof; - self.peek_span = mk_sp(self.filemap.end_pos, self.filemap.end_pos); + self.peek_span = self.mk_sp(self.filemap.end_pos, self.filemap.end_pos); } else { let start_bytepos = self.pos; self.peek_tok = self.next_token_inner()?; - self.peek_span = mk_sp(start_bytepos, self.pos); + self.peek_span = self.mk_sp(start_bytepos, self.pos); }; } } @@ -502,7 +504,7 @@ impl<'a> StringReader<'a> { if let Some(c) = self.ch { if c.is_whitespace() { let msg = "called consume_any_line_comment, but there was whitespace"; - self.sess.span_diagnostic.span_err(mk_sp(self.pos, self.pos), msg); + self.sess.span_diagnostic.span_err(self.mk_sp(self.pos, self.pos), msg); } } @@ -545,13 +547,13 @@ impl<'a> StringReader<'a> { Some(TokenAndSpan { tok: tok, - sp: mk_sp(start_bpos, self.pos), + sp: self.mk_sp(start_bpos, self.pos), }) }) } else { Some(TokenAndSpan { tok: token::Comment, - sp: mk_sp(start_bpos, self.pos), + sp: self.mk_sp(start_bpos, self.pos), }) } } @@ -584,7 +586,7 @@ impl<'a> StringReader<'a> { } return Some(TokenAndSpan { tok: token::Shebang(self.name_from(start)), - sp: mk_sp(start, self.pos), + sp: self.mk_sp(start, self.pos), }); } } @@ -612,7 +614,7 @@ impl<'a> StringReader<'a> { } let c = Some(TokenAndSpan { tok: token::Whitespace, - sp: mk_sp(start_bpos, self.pos), + sp: self.mk_sp(start_bpos, self.pos), }); debug!("scanning whitespace: {:?}", c); c @@ -674,7 +676,7 @@ impl<'a> StringReader<'a> { Some(TokenAndSpan { tok: tok, - sp: mk_sp(start_bpos, self.pos), + sp: self.mk_sp(start_bpos, self.pos), }) }) } @@ -869,7 +871,7 @@ impl<'a> StringReader<'a> { let valid = if self.ch_is('{') { self.scan_unicode_escape(delim) && !ascii_only } else { - let span = mk_sp(start, self.pos); + let span = self.mk_sp(start, self.pos); self.sess.span_diagnostic .struct_span_err(span, "incorrect unicode escape sequence") .span_help(span, @@ -907,13 +909,13 @@ impl<'a> StringReader<'a> { }, c); if e == '\r' { - err.span_help(mk_sp(escaped_pos, pos), + err.span_help(self.mk_sp(escaped_pos, pos), "this is an isolated carriage return; consider \ checking your editor and version control \ settings"); } if (e == '{' || e == '}') && !ascii_only { - err.span_help(mk_sp(escaped_pos, pos), + err.span_help(self.mk_sp(escaped_pos, pos), "if used in a formatting string, curly braces \ are escaped with `{{` and `}}`"); } diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index 3a68a6ba764..f917eec2cd0 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -141,9 +141,10 @@ pub fn parse_stmt_from_source_str(name: String, source: String, sess: &ParseSess new_parser_from_source_str(sess, name, source).parse_stmt() } -pub fn parse_stream_from_source_str(name: String, source: String, sess: &ParseSess) - -> TokenStream { - filemap_to_stream(sess, sess.codemap().new_filemap(name, source)) +pub fn parse_stream_from_source_str(name: String, source: String, sess: &ParseSess, + override_span: Option) + -> TokenStream { + filemap_to_stream(sess, sess.codemap().new_filemap(name, source), override_span) } // Create a new parser from a source string @@ -177,7 +178,7 @@ pub fn new_sub_parser_from_file<'a>(sess: &'a ParseSess, /// Given a filemap and config, return a parser pub fn filemap_to_parser(sess: & ParseSess, filemap: Rc, ) -> Parser { let end_pos = filemap.end_pos; - let mut parser = stream_to_parser(sess, filemap_to_stream(sess, filemap)); + let mut parser = stream_to_parser(sess, filemap_to_stream(sess, filemap, None)); if parser.token == token::Eof && parser.span == syntax_pos::DUMMY_SP { parser.span = Span { lo: end_pos, hi: end_pos, ctxt: NO_EXPANSION }; @@ -212,8 +213,10 @@ fn file_to_filemap(sess: &ParseSess, path: &Path, spanopt: Option) } /// Given a filemap, produce a sequence of token-trees -pub fn filemap_to_stream(sess: &ParseSess, filemap: Rc) -> TokenStream { +pub fn filemap_to_stream(sess: &ParseSess, filemap: Rc, override_span: Option) + -> TokenStream { let mut srdr = lexer::StringReader::new(sess, filemap); + srdr.override_span = override_span; srdr.real_token(); panictry!(srdr.parse_all_token_trees()) } diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index 851a638e148..25ab46f6f9e 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -2626,7 +2626,10 @@ impl<'a> Parser<'a> { pub fn process_potential_macro_variable(&mut self) { let ident = match self.token { - token::SubstNt(name) => { + token::Dollar if self.span.ctxt != syntax_pos::hygiene::SyntaxContext::empty() && + self.look_ahead(1, |t| t.is_ident()) => { + self.bump(); + let name = match self.token { token::Ident(ident) => ident, _ => unreachable!() }; self.fatal(&format!("unknown macro variable `{}`", name)).emit(); return } diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index 77db604c56e..f208b0f56f8 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -172,9 +172,6 @@ pub enum Token { // Can be expanded into several tokens. /// Doc comment DocComment(ast::Name), - // In right-hand-sides of MBE macros: - /// A syntactic variable that will be filled in by macro expansion. - SubstNt(ast::Ident), // Junk. These carry no data because we don't really care about the data // they *would* carry, and don't really want to allocate a new ident for diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs index 34cda433d52..6c6ca556e35 100644 --- a/src/libsyntax/print/pprust.rs +++ b/src/libsyntax/print/pprust.rs @@ -270,7 +270,6 @@ pub fn token_to_string(tok: &Token) -> String { /* Other */ token::DocComment(s) => s.to_string(), - token::SubstNt(s) => format!("${}", s), token::Eof => "".to_string(), token::Whitespace => " ".to_string(), token::Comment => "/* */".to_string(), -- cgit 1.4.1-3-g733a5