diff options
| author | bors <bors@rust-lang.org> | 2018-03-23 18:38:16 +0000 |
|---|---|---|
| committer | bors <bors@rust-lang.org> | 2018-03-23 18:38:16 +0000 |
| commit | c08480fce0f39f5c9c6db6dde0dccb375ca0ab14 (patch) | |
| tree | 32db4f9e696b8c5ba9d249bf05b4af6c40974e2d /src/libsyntax | |
| parent | 55e1104dd918a809d2751d325c11d59c85485a2e (diff) | |
| parent | 0e6cd8b61a392cd80d575eaef86d12d9c3f9e2dc (diff) | |
| download | rust-c08480fce0f39f5c9c6db6dde0dccb375ca0ab14.tar.gz rust-c08480fce0f39f5c9c6db6dde0dccb375ca0ab14.zip | |
Auto merge of #49308 - alexcrichton:rollup, r=alexcrichton
Rollup of 15 pull requests - Successful merges: #48265, #48528, #48552, #48624, #48883, #48909, #49028, #49030, #49102, #49160, #49169, #49203, #49262, #49272, #49295 - Failed merges: #48942, #49035
Diffstat (limited to 'src/libsyntax')
| -rw-r--r-- | src/libsyntax/ast.rs | 2 | ||||
| -rw-r--r-- | src/libsyntax/attr.rs | 13 | ||||
| -rw-r--r-- | src/libsyntax/config.rs | 6 | ||||
| -rw-r--r-- | src/libsyntax/diagnostics/plugin.rs | 10 | ||||
| -rw-r--r-- | src/libsyntax/edition.rs (renamed from src/libsyntax/epoch.rs) | 44 | ||||
| -rw-r--r-- | src/libsyntax/ext/base.rs | 5 | ||||
| -rw-r--r-- | src/libsyntax/ext/quote.rs | 12 | ||||
| -rw-r--r-- | src/libsyntax/ext/tt/macro_parser.rs | 21 | ||||
| -rw-r--r-- | src/libsyntax/ext/tt/macro_rules.rs | 6 | ||||
| -rw-r--r-- | src/libsyntax/ext/tt/quoted.rs | 6 | ||||
| -rw-r--r-- | src/libsyntax/ext/tt/transcribe.rs | 2 | ||||
| -rw-r--r-- | src/libsyntax/feature_gate.rs | 49 | ||||
| -rw-r--r-- | src/libsyntax/fold.rs | 5 | ||||
| -rw-r--r-- | src/libsyntax/lib.rs | 2 | ||||
| -rw-r--r-- | src/libsyntax/parse/lexer/mod.rs | 74 | ||||
| -rw-r--r-- | src/libsyntax/parse/mod.rs | 24 | ||||
| -rw-r--r-- | src/libsyntax/parse/parser.rs | 29 | ||||
| -rw-r--r-- | src/libsyntax/parse/token.rs | 87 | ||||
| -rw-r--r-- | src/libsyntax/print/pprust.rs | 40 | ||||
| -rw-r--r-- | src/libsyntax/tokenstream.rs | 2 |
20 files changed, 275 insertions, 164 deletions
diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs index 1f16b728cd2..a3af6b247ee 100644 --- a/src/libsyntax/ast.rs +++ b/src/libsyntax/ast.rs @@ -112,7 +112,7 @@ impl Path { // or starts with something like `self`/`super`/`$crate`/etc. pub fn make_root(&self) -> Option<PathSegment> { if let Some(ident) = self.segments.get(0).map(|seg| seg.identifier) { - if ::parse::token::Ident(ident).is_path_segment_keyword() && + if ::parse::token::is_path_segment_keyword(ident) && ident.name != keywords::Crate.name() { return None; } diff --git a/src/libsyntax/attr.rs b/src/libsyntax/attr.rs index f2cdcda98da..5954b9eb274 100644 --- a/src/libsyntax/attr.rs +++ b/src/libsyntax/attr.rs @@ -1106,7 +1106,8 @@ impl IntType { impl MetaItem { fn tokens(&self) -> TokenStream { - let ident = TokenTree::Token(self.span, Token::Ident(Ident::with_empty_ctxt(self.name))); + let ident = TokenTree::Token(self.span, + Token::from_ast_ident(Ident::with_empty_ctxt(self.name))); TokenStream::concat(vec![ident.into(), self.node.tokens(self.span)]) } @@ -1114,9 +1115,9 @@ impl MetaItem { where I: Iterator<Item = TokenTree>, { let (span, name) = match tokens.next() { - Some(TokenTree::Token(span, Token::Ident(ident))) => (span, ident.name), + Some(TokenTree::Token(span, Token::Ident(ident, _))) => (span, ident.name), Some(TokenTree::Token(_, Token::Interpolated(ref nt))) => match nt.0 { - token::Nonterminal::NtIdent(ident) => (ident.span, ident.node.name), + token::Nonterminal::NtIdent(ident, _) => (ident.span, ident.node.name), token::Nonterminal::NtMeta(ref meta) => return Some(meta.clone()), _ => return None, }, @@ -1269,14 +1270,14 @@ impl LitKind { "true" } else { "false" - }))), + })), false), } } fn from_token(token: Token) -> Option<LitKind> { match token { - Token::Ident(ident) if ident.name == "true" => Some(LitKind::Bool(true)), - Token::Ident(ident) if ident.name == "false" => Some(LitKind::Bool(false)), + Token::Ident(ident, false) if ident.name == "true" => Some(LitKind::Bool(true)), + Token::Ident(ident, false) if ident.name == "false" => Some(LitKind::Bool(false)), Token::Interpolated(ref nt) => match nt.0 { token::NtExpr(ref v) => match v.node { ExprKind::Lit(ref lit) => Some(lit.node.clone()), diff --git a/src/libsyntax/config.rs b/src/libsyntax/config.rs index 6013c20daf2..56b1306e5b3 100644 --- a/src/libsyntax/config.rs +++ b/src/libsyntax/config.rs @@ -13,7 +13,7 @@ use feature_gate::{feature_err, EXPLAIN_STMT_ATTR_SYNTAX, Features, get_features use {fold, attr}; use ast; use codemap::Spanned; -use epoch::Epoch; +use edition::Edition; use parse::{token, ParseSess}; use ptr::P; @@ -27,7 +27,7 @@ pub struct StripUnconfigured<'a> { } // `cfg_attr`-process the crate's attributes and compute the crate's features. -pub fn features(mut krate: ast::Crate, sess: &ParseSess, should_test: bool, epoch: Epoch) +pub fn features(mut krate: ast::Crate, sess: &ParseSess, should_test: bool, edition: Edition) -> (ast::Crate, Features) { let features; { @@ -47,7 +47,7 @@ pub fn features(mut krate: ast::Crate, sess: &ParseSess, should_test: bool, epoc return (krate, Features::new()); } - features = get_features(&sess.span_diagnostic, &krate.attrs, epoch); + features = get_features(&sess.span_diagnostic, &krate.attrs, edition); // Avoid reconfiguring malformed `cfg_attr`s if err_count == sess.span_diagnostic.err_count() { diff --git a/src/libsyntax/diagnostics/plugin.rs b/src/libsyntax/diagnostics/plugin.rs index 2c91844da96..aecf32ab6af 100644 --- a/src/libsyntax/diagnostics/plugin.rs +++ b/src/libsyntax/diagnostics/plugin.rs @@ -44,7 +44,7 @@ pub fn expand_diagnostic_used<'cx>(ecx: &'cx mut ExtCtxt, token_tree: &[TokenTree]) -> Box<MacResult+'cx> { let code = match (token_tree.len(), token_tree.get(0)) { - (1, Some(&TokenTree::Token(_, token::Ident(code)))) => code, + (1, Some(&TokenTree::Token(_, token::Ident(code, _)))) => code, _ => unreachable!() }; @@ -82,10 +82,10 @@ pub fn expand_register_diagnostic<'cx>(ecx: &'cx mut ExtCtxt, token_tree.get(1), token_tree.get(2) ) { - (1, Some(&TokenTree::Token(_, token::Ident(ref code))), None, None) => { + (1, Some(&TokenTree::Token(_, token::Ident(ref code, _))), None, None) => { (code, None) }, - (3, Some(&TokenTree::Token(_, token::Ident(ref code))), + (3, Some(&TokenTree::Token(_, token::Ident(ref code, _))), Some(&TokenTree::Token(_, token::Comma)), Some(&TokenTree::Token(_, token::Literal(token::StrRaw(description, _), None)))) => { (code, Some(description)) @@ -150,9 +150,9 @@ pub fn expand_build_diagnostic_array<'cx>(ecx: &'cx mut ExtCtxt, let (crate_name, name) = match (&token_tree[0], &token_tree[2]) { ( // Crate name. - &TokenTree::Token(_, token::Ident(ref crate_name)), + &TokenTree::Token(_, token::Ident(ref crate_name, _)), // DIAGNOSTICS ident. - &TokenTree::Token(_, token::Ident(ref name)) + &TokenTree::Token(_, token::Ident(ref name, _)) ) => (*&crate_name, name), _ => unreachable!() }; diff --git a/src/libsyntax/epoch.rs b/src/libsyntax/edition.rs index 32cbc79c550..61246d4493c 100644 --- a/src/libsyntax/epoch.rs +++ b/src/libsyntax/edition.rs @@ -11,58 +11,58 @@ use std::fmt; use std::str::FromStr; -/// The epoch of the compiler (RFC 2052) +/// The edition of the compiler (RFC 2052) #[derive(Clone, Copy, Hash, PartialOrd, Ord, Eq, PartialEq, Debug)] #[non_exhaustive] -pub enum Epoch { - // epochs must be kept in order, newest to oldest +pub enum Edition { + // editions must be kept in order, newest to oldest - /// The 2015 epoch - Epoch2015, - /// The 2018 epoch - Epoch2018, + /// The 2015 edition + Edition2015, + /// The 2018 edition + Edition2018, - // when adding new epochs, be sure to update: + // when adding new editions, be sure to update: // - // - the list in the `parse_epoch` static in librustc::session::config + // - the list in the `parse_edition` static in librustc::session::config // - add a `rust_####()` function to the session // - update the enum in Cargo's sources as well // - // When -Zepoch becomes --epoch, there will - // also be a check for the epoch being nightly-only + // When -Zedition becomes --edition, there will + // also be a check for the edition being nightly-only // somewhere. That will need to be updated - // whenever we're stabilizing/introducing a new epoch + // whenever we're stabilizing/introducing a new edition // as well as changing the default Cargo template. } // must be in order from oldest to newest -pub const ALL_EPOCHS: &[Epoch] = &[Epoch::Epoch2015, Epoch::Epoch2018]; +pub const ALL_EDITIONS: &[Edition] = &[Edition::Edition2015, Edition::Edition2018]; -impl fmt::Display for Epoch { +impl fmt::Display for Edition { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { let s = match *self { - Epoch::Epoch2015 => "2015", - Epoch::Epoch2018 => "2018", + Edition::Edition2015 => "2015", + Edition::Edition2018 => "2018", }; write!(f, "{}", s) } } -impl Epoch { +impl Edition { pub fn lint_name(&self) -> &'static str { match *self { - Epoch::Epoch2015 => "epoch_2015", - Epoch::Epoch2018 => "epoch_2018", + Edition::Edition2015 => "edition_2015", + Edition::Edition2018 => "edition_2018", } } } -impl FromStr for Epoch { +impl FromStr for Edition { type Err = (); fn from_str(s: &str) -> Result<Self, ()> { match s { - "2015" => Ok(Epoch::Epoch2015), - "2018" => Ok(Epoch::Epoch2018), + "2015" => Ok(Edition::Edition2015), + "2018" => Ok(Edition::Edition2018), _ => Err(()) } } diff --git a/src/libsyntax/ext/base.rs b/src/libsyntax/ext/base.rs index 23c42972912..c3ae0fd2ca8 100644 --- a/src/libsyntax/ext/base.rs +++ b/src/libsyntax/ext/base.rs @@ -229,8 +229,9 @@ impl<F> TTMacroExpander for F impl Folder for AvoidInterpolatedIdents { fn fold_tt(&mut self, tt: tokenstream::TokenTree) -> tokenstream::TokenTree { if let tokenstream::TokenTree::Token(_, token::Interpolated(ref nt)) = tt { - if let token::NtIdent(ident) = nt.0 { - return tokenstream::TokenTree::Token(ident.span, token::Ident(ident.node)); + if let token::NtIdent(ident, is_raw) = nt.0 { + return tokenstream::TokenTree::Token(ident.span, + token::Ident(ident.node, is_raw)); } } fold::noop_fold_tt(tt, self) diff --git a/src/libsyntax/ext/quote.rs b/src/libsyntax/ext/quote.rs index 6844532e7b3..540a03ff032 100644 --- a/src/libsyntax/ext/quote.rs +++ b/src/libsyntax/ext/quote.rs @@ -75,7 +75,7 @@ pub mod rt { impl ToTokens for ast::Ident { fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> { - vec![TokenTree::Token(DUMMY_SP, token::Ident(*self))] + vec![TokenTree::Token(DUMMY_SP, Token::from_ast_ident(*self))] } } @@ -238,7 +238,9 @@ pub mod rt { if i > 0 { inner.push(TokenTree::Token(self.span, token::Colon).into()); } - inner.push(TokenTree::Token(self.span, token::Ident(segment.identifier)).into()); + inner.push(TokenTree::Token( + self.span, token::Token::from_ast_ident(segment.identifier) + ).into()); } inner.push(self.tokens.clone()); @@ -658,10 +660,10 @@ fn expr_mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P<ast::Expr> { token::Literal(token::ByteStr(i), suf) => return mk_lit!("ByteStr", suf, i), token::Literal(token::ByteStrRaw(i, n), suf) => return mk_lit!("ByteStrRaw", suf, i, n), - token::Ident(ident) => { + token::Ident(ident, is_raw) => { return cx.expr_call(sp, mk_token_path(cx, sp, "Ident"), - vec![mk_ident(cx, sp, ident)]); + vec![mk_ident(cx, sp, ident), cx.expr_bool(sp, is_raw)]); } token::Lifetime(ident) => { @@ -720,7 +722,7 @@ fn expr_mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P<ast::Expr> { fn statements_mk_tt(cx: &ExtCtxt, tt: &TokenTree, quoted: bool) -> Vec<ast::Stmt> { match *tt { - TokenTree::Token(sp, token::Ident(ident)) if quoted => { + TokenTree::Token(sp, token::Ident(ident, _)) if quoted => { // tt.extend($ident.to_tokens(ext_cx)) let e_to_toks = diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs index 667653b5f7f..8cb331c65da 100644 --- a/src/libsyntax/ext/tt/macro_parser.rs +++ b/src/libsyntax/ext/tt/macro_parser.rs @@ -364,8 +364,8 @@ pub fn parse_failure_msg(tok: Token) -> String { /// Perform a token equality check, ignoring syntax context (that is, an unhygienic comparison) fn token_name_eq(t1: &Token, t2: &Token) -> bool { - if let (Some(id1), Some(id2)) = (t1.ident(), t2.ident()) { - id1.name == id2.name + if let (Some((id1, is_raw1)), Some((id2, is_raw2))) = (t1.ident(), t2.ident()) { + id1.name == id2.name && is_raw1 == is_raw2 } else if let (&token::Lifetime(id1), &token::Lifetime(id2)) = (t1, t2) { id1.name == id2.name } else { @@ -711,9 +711,10 @@ pub fn parse( /// The token is an identifier, but not `_`. /// We prohibit passing `_` to macros expecting `ident` for now. -fn get_macro_ident(token: &Token) -> Option<Ident> { +fn get_macro_ident(token: &Token) -> Option<(Ident, bool)> { match *token { - token::Ident(ident) if ident.name != keywords::Underscore.name() => Some(ident), + token::Ident(ident, is_raw) if ident.name != keywords::Underscore.name() => + Some((ident, is_raw)), _ => None, } } @@ -737,7 +738,7 @@ fn may_begin_with(name: &str, token: &Token) -> bool { "ident" => get_macro_ident(token).is_some(), "vis" => match *token { // The follow-set of :vis + "priv" keyword + interpolated - Token::Comma | Token::Ident(_) | Token::Interpolated(_) => true, + Token::Comma | Token::Ident(..) | Token::Interpolated(_) => true, _ => token.can_begin_type(), }, "block" => match *token { @@ -746,7 +747,7 @@ fn may_begin_with(name: &str, token: &Token) -> bool { token::NtItem(_) | token::NtPat(_) | token::NtTy(_) - | token::NtIdent(_) + | token::NtIdent(..) | token::NtMeta(_) | token::NtPath(_) | token::NtVis(_) => false, // none of these may start with '{'. @@ -755,7 +756,7 @@ fn may_begin_with(name: &str, token: &Token) -> bool { _ => false, }, "path" | "meta" => match *token { - Token::ModSep | Token::Ident(_) => true, + Token::ModSep | Token::Ident(..) => true, Token::Interpolated(ref nt) => match nt.0 { token::NtPath(_) | token::NtMeta(_) => true, _ => may_be_ident(&nt.0), @@ -763,7 +764,7 @@ fn may_begin_with(name: &str, token: &Token) -> bool { _ => false, }, "pat" => match *token { - Token::Ident(_) | // box, ref, mut, and other identifiers (can stricten) + Token::Ident(..) | // box, ref, mut, and other identifiers (can stricten) Token::OpenDelim(token::Paren) | // tuple pattern Token::OpenDelim(token::Bracket) | // slice pattern Token::BinOp(token::And) | // reference @@ -823,9 +824,9 @@ fn parse_nt<'a>(p: &mut Parser<'a>, sp: Span, name: &str) -> Nonterminal { "expr" => token::NtExpr(panictry!(p.parse_expr())), "ty" => token::NtTy(panictry!(p.parse_ty())), // this could be handled like a token, since it is one - "ident" => if let Some(ident) = get_macro_ident(&p.token) { + "ident" => if let Some((ident, is_raw)) = get_macro_ident(&p.token) { p.bump(); - token::NtIdent(respan(p.prev_span, ident)) + token::NtIdent(respan(p.prev_span, ident), is_raw) } else { let token_str = pprust::token_to_string(&p.token); p.fatal(&format!("expected ident, found {}", &token_str)).emit(); diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs index a4b2c3990f5..10e5926eb9e 100644 --- a/src/libsyntax/ext/tt/macro_rules.rs +++ b/src/libsyntax/ext/tt/macro_rules.rs @@ -831,7 +831,7 @@ fn is_in_follow(tok: "ed::TokenTree, frag: &str) -> Result<bool, (String, &' "pat" => match *tok { TokenTree::Token(_, ref tok) => match *tok { FatArrow | Comma | Eq | BinOp(token::Or) => Ok(true), - Ident(i) if i.name == "if" || i.name == "in" => Ok(true), + Ident(i, false) if i.name == "if" || i.name == "in" => Ok(true), _ => Ok(false) }, _ => Ok(false), @@ -840,7 +840,7 @@ fn is_in_follow(tok: "ed::TokenTree, frag: &str) -> Result<bool, (String, &' TokenTree::Token(_, ref tok) => match *tok { OpenDelim(token::DelimToken::Brace) | OpenDelim(token::DelimToken::Bracket) | Comma | FatArrow | Colon | Eq | Gt | Semi | BinOp(token::Or) => Ok(true), - Ident(i) if i.name == "as" || i.name == "where" => Ok(true), + Ident(i, false) if i.name == "as" || i.name == "where" => Ok(true), _ => Ok(false) }, TokenTree::MetaVarDecl(_, _, frag) if frag.name == "block" => Ok(true), @@ -860,7 +860,7 @@ fn is_in_follow(tok: "ed::TokenTree, frag: &str) -> Result<bool, (String, &' match *tok { TokenTree::Token(_, ref tok) => match *tok { Comma => Ok(true), - Ident(i) if i.name != "priv" => Ok(true), + Ident(i, is_raw) if is_raw || i.name != "priv" => Ok(true), ref tok => Ok(tok.can_begin_type()) }, TokenTree::MetaVarDecl(_, _, frag) if frag.name == "ident" diff --git a/src/libsyntax/ext/tt/quoted.rs b/src/libsyntax/ext/tt/quoted.rs index 122bb9ba024..f324edeb117 100644 --- a/src/libsyntax/ext/tt/quoted.rs +++ b/src/libsyntax/ext/tt/quoted.rs @@ -200,7 +200,7 @@ pub fn parse( let span = match trees.next() { Some(tokenstream::TokenTree::Token(span, token::Colon)) => match trees.next() { Some(tokenstream::TokenTree::Token(end_sp, ref tok)) => match tok.ident() { - Some(kind) => { + Some((kind, _)) => { let span = end_sp.with_lo(start_sp.lo()); result.push(TokenTree::MetaVarDecl(span, ident, kind)); continue; @@ -289,14 +289,14 @@ where // `tree` is followed by an `ident`. This could be `$meta_var` or the `$crate` special // metavariable that names the crate of the invokation. Some(tokenstream::TokenTree::Token(ident_span, ref token)) if token.is_ident() => { - let ident = token.ident().unwrap(); + let (ident, _) = token.ident().unwrap(); let span = ident_span.with_lo(span.lo()); if ident.name == keywords::Crate.name() { let ident = ast::Ident { name: keywords::DollarCrate.name(), ..ident }; - TokenTree::Token(span, token::Ident(ident)) + TokenTree::Token(span, token::Ident(ident, false)) } else { TokenTree::MetaVar(span, ident) } diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/tt/transcribe.rs index 7883c4bbc16..3f01d5ec6dd 100644 --- a/src/libsyntax/ext/tt/transcribe.rs +++ b/src/libsyntax/ext/tt/transcribe.rs @@ -169,7 +169,7 @@ pub fn transcribe(cx: &ExtCtxt, Ident { ctxt: ident.ctxt.apply_mark(cx.current_expansion.mark), ..ident }; sp = sp.with_ctxt(sp.ctxt().apply_mark(cx.current_expansion.mark)); result.push(TokenTree::Token(sp, token::Dollar).into()); - result.push(TokenTree::Token(sp, token::Ident(ident)).into()); + result.push(TokenTree::Token(sp, token::Token::from_ast_ident(ident)).into()); } } quoted::TokenTree::Delimited(mut span, delimited) => { diff --git a/src/libsyntax/feature_gate.rs b/src/libsyntax/feature_gate.rs index 915396d29fe..69612054ae3 100644 --- a/src/libsyntax/feature_gate.rs +++ b/src/libsyntax/feature_gate.rs @@ -28,7 +28,7 @@ use self::AttributeGate::*; use abi::Abi; use ast::{self, NodeId, PatKind, RangeEnd}; use attr; -use epoch::Epoch; +use edition::Edition; use codemap::Spanned; use syntax_pos::{Span, DUMMY_SP}; use errors::{DiagnosticBuilder, Handler, FatalError}; @@ -55,13 +55,13 @@ macro_rules! set { } macro_rules! declare_features { - ($((active, $feature: ident, $ver: expr, $issue: expr, $epoch: expr),)+) => { + ($((active, $feature: ident, $ver: expr, $issue: expr, $edition: expr),)+) => { /// Represents active features that are currently being implemented or /// currently being considered for addition/removal. const ACTIVE_FEATURES: &'static [(&'static str, &'static str, Option<u32>, - Option<Epoch>, fn(&mut Features, Span))] = - &[$((stringify!($feature), $ver, $issue, $epoch, set!($feature))),+]; + Option<Edition>, fn(&mut Features, Span))] = + &[$((stringify!($feature), $ver, $issue, $edition, set!($feature))),+]; /// A set of features to be used by later passes. #[derive(Clone)] @@ -402,7 +402,7 @@ declare_features! ( (active, match_default_bindings, "1.22.0", Some(42640), None), // Trait object syntax with `dyn` prefix - (active, dyn_trait, "1.22.0", Some(44662), Some(Epoch::Epoch2018)), + (active, dyn_trait, "1.22.0", Some(44662), Some(Edition::Edition2018)), // `crate` as visibility modifier, synonymous to `pub(crate)` (active, crate_visibility_modifier, "1.23.0", Some(45388), None), @@ -451,6 +451,15 @@ declare_features! ( // `use path as _;` and `extern crate c as _;` (active, underscore_imports, "1.26.0", Some(48216), None), + + // The #[wasm_custom_section] attribute + (active, wasm_custom_section, "1.26.0", None, None), + + // The #![wasm_import_module] attribute + (active, wasm_import_module, "1.26.0", None, None), + + // Allows keywords to be escaped for use as identifiers + (active, raw_identifiers, "1.26.0", Some(48589), None), ); declare_features! ( @@ -917,6 +926,10 @@ pub const BUILTIN_ATTRIBUTES: &'static [(&'static str, AttributeType, AttributeG "the `#[no_debug]` attribute was an experimental feature that has been \ deprecated due to lack of demand", cfg_fn!(no_debug))), + ("wasm_import_module", Normal, Gated(Stability::Unstable, + "wasm_import_module", + "experimental attribute", + cfg_fn!(wasm_import_module))), ("omit_gdb_pretty_printer_section", Whitelisted, Gated(Stability::Unstable, "omit_gdb_pretty_printer_section", "the `#[omit_gdb_pretty_printer_section]` \ @@ -1004,6 +1017,11 @@ pub const BUILTIN_ATTRIBUTES: &'static [(&'static str, AttributeType, AttributeG "never will be stable", cfg_fn!(rustc_attrs))), + ("wasm_custom_section", Whitelisted, Gated(Stability::Unstable, + "wasm_custom_section", + "attribute is currently unstable", + cfg_fn!(wasm_custom_section))), + // Crate level attributes ("crate_name", CrateLevel, Ungated), ("crate_type", CrateLevel, Ungated), @@ -1800,16 +1818,16 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> { } pub fn get_features(span_handler: &Handler, krate_attrs: &[ast::Attribute], - epoch: Epoch) -> Features { + edition: Edition) -> Features { let mut features = Features::new(); let mut feature_checker = FeatureChecker::default(); - for &(.., f_epoch, set) in ACTIVE_FEATURES.iter() { - if let Some(f_epoch) = f_epoch { - if epoch >= f_epoch { + for &(.., f_edition, set) in ACTIVE_FEATURES.iter() { + if let Some(f_edition) = f_edition { + if edition >= f_edition { // FIXME(Manishearth) there is currently no way to set - // lang features by epoch + // lang features by edition set(&mut features, DUMMY_SP); } } @@ -1926,6 +1944,17 @@ pub fn check_crate(krate: &ast::Crate, parse_sess: sess, plugin_attributes, }; + + if !features.raw_identifiers { + for &span in sess.raw_identifier_spans.borrow().iter() { + if !span.allows_unstable() { + gate_feature!(&ctx, raw_identifiers, span, + "raw identifiers are experimental and subject to change" + ); + } + } + } + let visitor = &mut PostExpansionVisitor { context: &ctx }; visitor.whole_crate_feature_gates(krate); visit::walk_crate(visitor, krate); diff --git a/src/libsyntax/fold.rs b/src/libsyntax/fold.rs index 46e6027b094..05a3150c139 100644 --- a/src/libsyntax/fold.rs +++ b/src/libsyntax/fold.rs @@ -578,7 +578,7 @@ pub fn noop_fold_tts<T: Folder>(tts: TokenStream, fld: &mut T) -> TokenStream { // apply ident folder if it's an ident, apply other folds to interpolated nodes pub fn noop_fold_token<T: Folder>(t: token::Token, fld: &mut T) -> token::Token { match t { - token::Ident(id) => token::Ident(fld.fold_ident(id)), + token::Ident(id, is_raw) => token::Ident(fld.fold_ident(id), is_raw), token::Lifetime(id) => token::Lifetime(fld.fold_ident(id)), token::Interpolated(nt) => { let nt = match Lrc::try_unwrap(nt) { @@ -630,7 +630,8 @@ pub fn noop_fold_interpolated<T: Folder>(nt: token::Nonterminal, fld: &mut T) token::NtPat(pat) => token::NtPat(fld.fold_pat(pat)), token::NtExpr(expr) => token::NtExpr(fld.fold_expr(expr)), token::NtTy(ty) => token::NtTy(fld.fold_ty(ty)), - token::NtIdent(id) => token::NtIdent(Spanned::<Ident>{node: fld.fold_ident(id.node), ..id}), + token::NtIdent(id, is_raw) => + token::NtIdent(Spanned::<Ident>{node: fld.fold_ident(id.node), ..id}, is_raw), token::NtMeta(meta) => token::NtMeta(fld.fold_meta_item(meta)), token::NtPath(path) => token::NtPath(fld.fold_path(path)), token::NtTT(tt) => token::NtTT(fld.fold_tt(tt)), diff --git a/src/libsyntax/lib.rs b/src/libsyntax/lib.rs index 5f58b3bc3a0..74f1ee373ec 100644 --- a/src/libsyntax/lib.rs +++ b/src/libsyntax/lib.rs @@ -145,7 +145,7 @@ pub mod codemap; #[macro_use] pub mod config; pub mod entry; -pub mod epoch; +pub mod edition; pub mod feature_gate; pub mod fold; pub mod parse; diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs index 0e20eb49d39..068929c8948 100644 --- a/src/libsyntax/parse/lexer/mod.rs +++ b/src/libsyntax/parse/lexer/mod.rs @@ -14,7 +14,7 @@ use codemap::{CodeMap, FilePathMapping}; use errors::{FatalError, DiagnosticBuilder}; use parse::{token, ParseSess}; use str::char_at; -use symbol::Symbol; +use symbol::{Symbol, keywords}; use std_unicode::property::Pattern_White_Space; use std::borrow::Cow; @@ -1115,26 +1115,53 @@ impl<'a> StringReader<'a> { /// token, and updates the interner fn next_token_inner(&mut self) -> Result<token::Token, ()> { let c = self.ch; - if ident_start(c) && - match (c.unwrap(), self.nextch(), self.nextnextch()) { - // Note: r as in r" or r#" is part of a raw string literal, - // b as in b' is part of a byte literal. - // They are not identifiers, and are handled further down. - ('r', Some('"'), _) | - ('r', Some('#'), _) | - ('b', Some('"'), _) | - ('b', Some('\''), _) | - ('b', Some('r'), Some('"')) | - ('b', Some('r'), Some('#')) => false, - _ => true, - } { - let start = self.pos; - while ident_continue(self.ch) { - self.bump(); - } - // FIXME: perform NFKC normalization here. (Issue #2253) - return Ok(self.with_str_from(start, |string| token::Ident(self.mk_ident(string)))); + if ident_start(c) { + let (is_ident_start, is_raw_ident) = + match (c.unwrap(), self.nextch(), self.nextnextch()) { + // r# followed by an identifier starter is a raw identifier. + // This is an exception to the r# case below. + ('r', Some('#'), x) if ident_start(x) => (true, true), + // r as in r" or r#" is part of a raw string literal. + // b as in b' is part of a byte literal. + // They are not identifiers, and are handled further down. + ('r', Some('"'), _) | + ('r', Some('#'), _) | + ('b', Some('"'), _) | + ('b', Some('\''), _) | + ('b', Some('r'), Some('"')) | + ('b', Some('r'), Some('#')) => (false, false), + _ => (true, false), + }; + if is_ident_start { + let raw_start = self.pos; + if is_raw_ident { + // Consume the 'r#' characters. + self.bump(); + self.bump(); + } + + let start = self.pos; + while ident_continue(self.ch) { + self.bump(); + } + + return Ok(self.with_str_from(start, |string| { + // FIXME: perform NFKC normalization here. (Issue #2253) + let ident = self.mk_ident(string); + if is_raw_ident && (token::is_path_segment_keyword(ident) || + ident.name == keywords::Underscore.name()) { + self.fatal_span_(raw_start, self.pos, + &format!("`r#{}` is not currently supported.", ident.name) + ).raise(); + } + if is_raw_ident { + let span = self.mk_sp(raw_start, self.pos); + self.sess.raw_identifier_spans.borrow_mut().push(span); + } + token::Ident(ident, is_raw_ident) + })); + } } if is_dec_digit(c) { @@ -1773,6 +1800,7 @@ mod tests { included_mod_stack: RefCell::new(Vec::new()), code_map: cm, missing_fragment_specifiers: RefCell::new(HashSet::new()), + raw_identifier_spans: RefCell::new(Vec::new()), registered_diagnostics: Lock::new(ErrorMap::new()), non_modrs_mods: RefCell::new(vec![]), } @@ -1801,7 +1829,7 @@ mod tests { assert_eq!(string_reader.next_token().tok, token::Whitespace); let tok1 = string_reader.next_token(); let tok2 = TokenAndSpan { - tok: token::Ident(id), + tok: token::Ident(id, false), sp: Span::new(BytePos(21), BytePos(23), NO_EXPANSION), }; assert_eq!(tok1, tok2); @@ -1811,7 +1839,7 @@ mod tests { // read another token: let tok3 = string_reader.next_token(); let tok4 = TokenAndSpan { - tok: token::Ident(Ident::from_str("main")), + tok: mk_ident("main"), sp: Span::new(BytePos(24), BytePos(28), NO_EXPANSION), }; assert_eq!(tok3, tok4); @@ -1830,7 +1858,7 @@ mod tests { // make the identifier by looking up the string in the interner fn mk_ident(id: &str) -> token::Token { - token::Ident(Ident::from_str(id)) + token::Token::from_ast_ident(Ident::from_str(id)) } #[test] diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index 202dc03eaa4..7b39db16ac2 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -48,6 +48,9 @@ pub struct ParseSess { pub unstable_features: UnstableFeatures, pub config: CrateConfig, pub missing_fragment_specifiers: RefCell<HashSet<Span>>, + /// Places where raw identifiers were used. This is used for feature gating + /// raw identifiers + pub raw_identifier_spans: RefCell<Vec<Span>>, /// The registered diagnostics codes pub registered_diagnostics: Lock<ErrorMap>, // Spans where a `mod foo;` statement was included in a non-mod.rs file. @@ -74,6 +77,7 @@ impl ParseSess { unstable_features: UnstableFeatures::from_environment(), config: HashSet::new(), missing_fragment_specifiers: RefCell::new(HashSet::new()), + raw_identifier_spans: RefCell::new(Vec::new()), registered_diagnostics: Lock::new(ErrorMap::new()), included_mod_stack: RefCell::new(vec![]), code_map, @@ -739,9 +743,9 @@ mod tests { match (tts.len(), tts.get(0), tts.get(1), tts.get(2), tts.get(3)) { ( 4, - Some(&TokenTree::Token(_, token::Ident(name_macro_rules))), + Some(&TokenTree::Token(_, token::Ident(name_macro_rules, false))), Some(&TokenTree::Token(_, token::Not)), - Some(&TokenTree::Token(_, token::Ident(name_zip))), + Some(&TokenTree::Token(_, token::Ident(name_zip, false))), Some(&TokenTree::Delimited(_, ref macro_delimed)), ) if name_macro_rules.name == "macro_rules" @@ -760,7 +764,7 @@ mod tests { ( 2, Some(&TokenTree::Token(_, token::Dollar)), - Some(&TokenTree::Token(_, token::Ident(ident))), + Some(&TokenTree::Token(_, token::Ident(ident, false))), ) if first_delimed.delim == token::Paren && ident.name == "a" => {}, _ => panic!("value 3: {:?}", *first_delimed), @@ -770,7 +774,7 @@ mod tests { ( 2, Some(&TokenTree::Token(_, token::Dollar)), - Some(&TokenTree::Token(_, token::Ident(ident))), + Some(&TokenTree::Token(_, token::Ident(ident, false))), ) if second_delimed.delim == token::Paren && ident.name == "a" => {}, @@ -791,17 +795,18 @@ mod tests { let tts = string_to_stream("fn a (b : i32) { b; }".to_string()); let expected = TokenStream::concat(vec![ - TokenTree::Token(sp(0, 2), token::Ident(Ident::from_str("fn"))).into(), - TokenTree::Token(sp(3, 4), token::Ident(Ident::from_str("a"))).into(), + TokenTree::Token(sp(0, 2), token::Ident(Ident::from_str("fn"), false)).into(), + TokenTree::Token(sp(3, 4), token::Ident(Ident::from_str("a"), false)).into(), TokenTree::Delimited( sp(5, 14), tokenstream::Delimited { delim: token::DelimToken::Paren, tts: TokenStream::concat(vec![ - TokenTree::Token(sp(6, 7), token::Ident(Ident::from_str("b"))).into(), + TokenTree::Token(sp(6, 7), + token::Ident(Ident::from_str("b"), false)).into(), TokenTree::Token(sp(8, 9), token::Colon).into(), TokenTree::Token(sp(10, 13), - token::Ident(Ident::from_str("i32"))).into(), + token::Ident(Ident::from_str("i32"), false)).into(), ]).into(), }).into(), TokenTree::Delimited( @@ -809,7 +814,8 @@ mod tests { tokenstream::Delimited { delim: token::DelimToken::Brace, tts: TokenStream::concat(vec![ - TokenTree::Token(sp(17, 18), token::Ident(Ident::from_str("b"))).into(), + TokenTree::Token(sp(17, 18), + token::Ident(Ident::from_str("b"), false)).into(), TokenTree::Token(sp(18, 19), token::Semi).into(), ]).into(), }).into() diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index 6d8975197d5..a07279acae2 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -358,7 +358,7 @@ impl TokenCursor { let body = TokenTree::Delimited(sp, Delimited { delim: token::Bracket, - tts: [TokenTree::Token(sp, token::Ident(ast::Ident::from_str("doc"))), + tts: [TokenTree::Token(sp, token::Ident(ast::Ident::from_str("doc"), false)), TokenTree::Token(sp, token::Eq), TokenTree::Token(sp, token::Literal( token::StrRaw(Symbol::intern(&stripped), num_of_hashes), None))] @@ -784,7 +784,7 @@ impl<'a> Parser<'a> { fn parse_ident_common(&mut self, recover: bool) -> PResult<'a, ast::Ident> { match self.token { - token::Ident(i) => { + token::Ident(i, _) => { if self.token.is_reserved_ident() { let mut err = self.expected_ident_found(); if recover { @@ -1925,7 +1925,7 @@ impl<'a> Parser<'a> { pub fn parse_path_segment_ident(&mut self) -> PResult<'a, ast::Ident> { match self.token { - token::Ident(sid) if self.token.is_path_segment_keyword() => { + token::Ident(sid, _) if self.token.is_path_segment_keyword() => { self.bump(); Ok(sid) } @@ -2740,11 +2740,14 @@ impl<'a> Parser<'a> { } pub fn process_potential_macro_variable(&mut self) { - let ident = match self.token { + let (ident, is_raw) = match self.token { token::Dollar if self.span.ctxt() != syntax_pos::hygiene::SyntaxContext::empty() && self.look_ahead(1, |t| t.is_ident()) => { self.bump(); - let name = match self.token { token::Ident(ident) => ident, _ => unreachable!() }; + let name = match self.token { + token::Ident(ident, _) => ident, + _ => unreachable!() + }; let mut err = self.fatal(&format!("unknown macro variable `{}`", name)); err.span_label(self.span, "unknown macro variable"); err.emit(); @@ -2753,13 +2756,13 @@ impl<'a> Parser<'a> { token::Interpolated(ref nt) => { self.meta_var_span = Some(self.span); match nt.0 { - token::NtIdent(ident) => ident, + token::NtIdent(ident, is_raw) => (ident, is_raw), _ => return, } } _ => return, }; - self.token = token::Ident(ident.node); + self.token = token::Ident(ident.node, is_raw); self.span = ident.span; } @@ -4245,7 +4248,7 @@ impl<'a> Parser<'a> { -> PResult<'a, Option<P<Item>>> { let token_lo = self.span; let (ident, def) = match self.token { - token::Ident(ident) if ident.name == keywords::Macro.name() => { + token::Ident(ident, false) if ident.name == keywords::Macro.name() => { self.bump(); let ident = self.parse_ident()?; let tokens = if self.check(&token::OpenDelim(token::Brace)) { @@ -4273,7 +4276,7 @@ impl<'a> Parser<'a> { (ident, ast::MacroDef { tokens: tokens.into(), legacy: false }) } - token::Ident(ident) if ident.name == "macro_rules" && + token::Ident(ident, _) if ident.name == "macro_rules" && self.look_ahead(1, |t| *t == token::Not) => { let prev_span = self.prev_span; self.complain_if_pub_macro(&vis.node, prev_span); @@ -5078,7 +5081,9 @@ impl<'a> Parser<'a> { fn parse_self_arg(&mut self) -> PResult<'a, Option<Arg>> { let expect_ident = |this: &mut Self| match this.token { // Preserve hygienic context. - token::Ident(ident) => { let sp = this.span; this.bump(); codemap::respan(sp, ident) } + token::Ident(ident, _) => { + let sp = this.span; this.bump(); codemap::respan(sp, ident) + } _ => unreachable!() }; let isolated_self = |this: &mut Self, n| { @@ -5375,7 +5380,7 @@ impl<'a> Parser<'a> { VisibilityKind::Inherited => Ok(()), _ => { let is_macro_rules: bool = match self.token { - token::Ident(sid) => sid.name == Symbol::intern("macro_rules"), + token::Ident(sid, _) => sid.name == Symbol::intern("macro_rules"), _ => false, }; if is_macro_rules { @@ -7016,7 +7021,7 @@ impl<'a> Parser<'a> { fn parse_rename(&mut self) -> PResult<'a, Option<Ident>> { if self.eat_keyword(keywords::As) { match self.token { - token::Ident(ident) if ident.name == keywords::Underscore.name() => { + token::Ident(ident, false) if ident.name == keywords::Underscore.name() => { self.bump(); // `_` Ok(Some(Ident { name: ident.name.gensymed(), ..ident })) } diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index 4ada9e20f2c..7798a7a77ee 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -91,8 +91,8 @@ impl Lit { } } -fn ident_can_begin_expr(ident: ast::Ident) -> bool { - let ident_token: Token = Ident(ident); +fn ident_can_begin_expr(ident: ast::Ident, is_raw: bool) -> bool { + let ident_token: Token = Ident(ident, is_raw); !ident_token.is_reserved_ident() || ident_token.is_path_segment_keyword() || @@ -116,8 +116,8 @@ fn ident_can_begin_expr(ident: ast::Ident) -> bool { ].contains(&ident.name) } -fn ident_can_begin_type(ident: ast::Ident) -> bool { - let ident_token: Token = Ident(ident); +fn ident_can_begin_type(ident: ast::Ident, is_raw: bool) -> bool { + let ident_token: Token = Ident(ident, is_raw); !ident_token.is_reserved_ident() || ident_token.is_path_segment_keyword() || @@ -132,6 +132,37 @@ fn ident_can_begin_type(ident: ast::Ident) -> bool { ].contains(&ident.name) } +pub fn is_path_segment_keyword(id: ast::Ident) -> bool { + id.name == keywords::Super.name() || + id.name == keywords::SelfValue.name() || + id.name == keywords::SelfType.name() || + id.name == keywords::Extern.name() || + id.name == keywords::Crate.name() || + id.name == keywords::CrateRoot.name() || + id.name == keywords::DollarCrate.name() +} + +// Returns true for reserved identifiers used internally for elided lifetimes, +// unnamed method parameters, crate root module, error recovery etc. +pub fn is_special_ident(id: ast::Ident) -> bool { + id.name <= keywords::Underscore.name() +} + +/// Returns `true` if the token is a keyword used in the language. +pub fn is_used_keyword(id: ast::Ident) -> bool { + id.name >= keywords::As.name() && id.name <= keywords::While.name() +} + +/// Returns `true` if the token is a keyword reserved for possible future use. +pub fn is_unused_keyword(id: ast::Ident) -> bool { + id.name >= keywords::Abstract.name() && id.name <= keywords::Yield.name() +} + +/// Returns `true` if the token is either a special identifier or a keyword. +pub fn is_reserved_ident(id: ast::Ident) -> bool { + is_special_ident(id) || is_used_keyword(id) || is_unused_keyword(id) +} + #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug)] pub enum Token { /* Expression-operator symbols. */ @@ -175,7 +206,7 @@ pub enum Token { Literal(Lit, Option<ast::Name>), /* Name components */ - Ident(ast::Ident), + Ident(ast::Ident, /* is_raw */ bool), Lifetime(ast::Ident), // The `LazyTokenStream` is a pure function of the `Nonterminal`, @@ -203,6 +234,11 @@ impl Token { Token::Interpolated(Lrc::new((nt, LazyTokenStream::new()))) } + /// Recovers a `Token` from an `ast::Ident`. This creates a raw identifier if necessary. + pub fn from_ast_ident(ident: ast::Ident) -> Token { + Ident(ident, is_reserved_ident(ident) && !is_path_segment_keyword(ident)) + } + /// Returns `true` if the token starts with '>'. pub fn is_like_gt(&self) -> bool { match *self { @@ -214,7 +250,8 @@ impl Token { /// Returns `true` if the token can appear at the start of an expression. pub fn can_begin_expr(&self) -> bool { match *self { - Ident(ident) => ident_can_begin_expr(ident), // value name or keyword + Ident(ident, is_raw) => + ident_can_begin_expr(ident, is_raw), // value name or keyword OpenDelim(..) | // tuple, array or block Literal(..) | // literal Not | // operator not @@ -239,7 +276,8 @@ impl Token { /// Returns `true` if the token can appear at the start of a type. pub fn can_begin_type(&self) -> bool { match *self { - Ident(ident) => ident_can_begin_type(ident), // type name or keyword + Ident(ident, is_raw) => + ident_can_begin_type(ident, is_raw), // type name or keyword OpenDelim(Paren) | // tuple OpenDelim(Bracket) | // array Not | // never @@ -272,11 +310,11 @@ impl Token { } } - pub fn ident(&self) -> Option<ast::Ident> { + pub fn ident(&self) -> Option<(ast::Ident, bool)> { match *self { - Ident(ident) => Some(ident), + Ident(ident, is_raw) => Some((ident, is_raw)), Interpolated(ref nt) => match nt.0 { - NtIdent(ident) => Some(ident.node), + NtIdent(ident, is_raw) => Some((ident.node, is_raw)), _ => None, }, _ => None, @@ -351,19 +389,13 @@ impl Token { /// Returns `true` if the token is a given keyword, `kw`. pub fn is_keyword(&self, kw: keywords::Keyword) -> bool { - self.ident().map(|ident| ident.name == kw.name()).unwrap_or(false) + self.ident().map(|(ident, is_raw)| ident.name == kw.name() && !is_raw).unwrap_or(false) } pub fn is_path_segment_keyword(&self) -> bool { match self.ident() { - Some(id) => id.name == keywords::Super.name() || - id.name == keywords::SelfValue.name() || - id.name == keywords::SelfType.name() || - id.name == keywords::Extern.name() || - id.name == keywords::Crate.name() || - id.name == keywords::CrateRoot.name() || - id.name == keywords::DollarCrate.name(), - None => false, + Some((id, false)) => is_path_segment_keyword(id), + _ => false, } } @@ -371,7 +403,7 @@ impl Token { // unnamed method parameters, crate root module, error recovery etc. pub fn is_special_ident(&self) -> bool { match self.ident() { - Some(id) => id.name <= keywords::Underscore.name(), + Some((id, false)) => is_special_ident(id), _ => false, } } @@ -379,7 +411,7 @@ impl Token { /// Returns `true` if the token is a keyword used in the language. pub fn is_used_keyword(&self) -> bool { match self.ident() { - Some(id) => id.name >= keywords::As.name() && id.name <= keywords::While.name(), + Some((id, false)) => is_used_keyword(id), _ => false, } } @@ -387,7 +419,7 @@ impl Token { /// Returns `true` if the token is a keyword reserved for possible future use. pub fn is_unused_keyword(&self) -> bool { match self.ident() { - Some(id) => id.name >= keywords::Abstract.name() && id.name <= keywords::Yield.name(), + Some((id, false)) => is_unused_keyword(id), _ => false, } } @@ -460,7 +492,10 @@ impl Token { /// Returns `true` if the token is either a special identifier or a keyword. pub fn is_reserved_ident(&self) -> bool { - self.is_special_ident() || self.is_used_keyword() || self.is_unused_keyword() + match self.ident() { + Some((id, false)) => is_reserved_ident(id), + _ => false, + } } pub fn interpolated_to_tokenstream(&self, sess: &ParseSess, span: Span) @@ -496,8 +531,8 @@ impl Token { Nonterminal::NtImplItem(ref item) => { tokens = prepend_attrs(sess, &item.attrs, item.tokens.as_ref(), span); } - Nonterminal::NtIdent(ident) => { - let token = Token::Ident(ident.node); + Nonterminal::NtIdent(ident, is_raw) => { + let token = Token::Ident(ident.node, is_raw); tokens = Some(TokenTree::Token(ident.span, token).into()); } Nonterminal::NtLifetime(lifetime) => { @@ -529,7 +564,7 @@ pub enum Nonterminal { NtPat(P<ast::Pat>), NtExpr(P<ast::Expr>), NtTy(P<ast::Ty>), - NtIdent(ast::SpannedIdent), + NtIdent(ast::SpannedIdent, /* is_raw */ bool), /// Stuff inside brackets for attributes NtMeta(ast::MetaItem), NtPath(ast::Path), diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs index 7adb2848f8d..50577a26abf 100644 --- a/src/libsyntax/print/pprust.rs +++ b/src/libsyntax/print/pprust.rs @@ -250,7 +250,8 @@ pub fn token_to_string(tok: &Token) -> String { } /* Name components */ - token::Ident(s) => s.to_string(), + token::Ident(s, false) => s.to_string(), + token::Ident(s, true) => format!("r#{}", s), token::Lifetime(s) => s.to_string(), /* Other */ @@ -261,24 +262,25 @@ pub fn token_to_string(tok: &Token) -> String { token::Shebang(s) => format!("/* shebang: {}*/", s), token::Interpolated(ref nt) => match nt.0 { - token::NtExpr(ref e) => expr_to_string(e), - token::NtMeta(ref e) => meta_item_to_string(e), - token::NtTy(ref e) => ty_to_string(e), - token::NtPath(ref e) => path_to_string(e), - token::NtItem(ref e) => item_to_string(e), - token::NtBlock(ref e) => block_to_string(e), - token::NtStmt(ref e) => stmt_to_string(e), - token::NtPat(ref e) => pat_to_string(e), - token::NtIdent(ref e) => ident_to_string(e.node), - token::NtTT(ref tree) => tt_to_string(tree.clone()), - token::NtArm(ref e) => arm_to_string(e), - token::NtImplItem(ref e) => impl_item_to_string(e), - token::NtTraitItem(ref e) => trait_item_to_string(e), - token::NtGenerics(ref e) => generic_params_to_string(&e.params), - token::NtWhereClause(ref e) => where_clause_to_string(e), - token::NtArg(ref e) => arg_to_string(e), - token::NtVis(ref e) => vis_to_string(e), - token::NtLifetime(ref e) => lifetime_to_string(e), + token::NtExpr(ref e) => expr_to_string(e), + token::NtMeta(ref e) => meta_item_to_string(e), + token::NtTy(ref e) => ty_to_string(e), + token::NtPath(ref e) => path_to_string(e), + token::NtItem(ref e) => item_to_string(e), + token::NtBlock(ref e) => block_to_string(e), + token::NtStmt(ref e) => stmt_to_string(e), + token::NtPat(ref e) => pat_to_string(e), + token::NtIdent(ref e, false) => ident_to_string(e.node), + token::NtIdent(ref e, true) => format!("r#{}", ident_to_string(e.node)), + token::NtTT(ref tree) => tt_to_string(tree.clone()), + token::NtArm(ref e) => arm_to_string(e), + token::NtImplItem(ref e) => impl_item_to_string(e), + token::NtTraitItem(ref e) => trait_item_to_string(e), + token::NtGenerics(ref e) => generic_params_to_string(&e.params), + token::NtWhereClause(ref e) => where_clause_to_string(e), + token::NtArg(ref e) => arg_to_string(e), + token::NtVis(ref e) => vis_to_string(e), + token::NtLifetime(ref e) => lifetime_to_string(e), } } } diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index 1219e909e12..3a7a1b9a669 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -684,7 +684,7 @@ mod tests { with_globals(|| { let test0: TokenStream = Vec::<TokenTree>::new().into_iter().collect(); let test1: TokenStream = - TokenTree::Token(sp(0, 1), Token::Ident(Ident::from_str("a"))).into(); + TokenTree::Token(sp(0, 1), Token::Ident(Ident::from_str("a"), false)).into(); let test2 = string_to_ts("foo(bar::baz)"); assert_eq!(test0.is_empty(), true); |
