diff options
| author | Marcin Fatyga <marcinf@google.com> | 2016-11-01 15:26:22 +0100 |
|---|---|---|
| committer | Marcin Fatyga <marcinf@google.com> | 2016-11-01 15:26:22 +0100 |
| commit | 655effedf25e2039d283b839429bf2f42b7012a4 (patch) | |
| tree | 34fd087d891556c70a14b26a90d1bdccd0a7ccb2 /src/libsyntax | |
| parent | 4e2822c5c28bb342e5862ba7cc0b90b865c68be1 (diff) | |
| parent | ac968c466451cb9aafd9e8598ddb396ed0e6fe31 (diff) | |
| download | rust-655effedf25e2039d283b839429bf2f42b7012a4.tar.gz rust-655effedf25e2039d283b839429bf2f42b7012a4.zip | |
Merge branch 'master' of https://github.com/rust-lang/rust
Conflicts: src/libcoretest/lib.rs
Diffstat (limited to 'src/libsyntax')
37 files changed, 2337 insertions, 1976 deletions
diff --git a/src/libsyntax/abi.rs b/src/libsyntax/abi.rs index 64a71133a8c..a39cac8db99 100644 --- a/src/libsyntax/abi.rs +++ b/src/libsyntax/abi.rs @@ -24,6 +24,7 @@ pub enum Os { Netbsd, Openbsd, NaCl, + Haiku, Solaris, } @@ -32,7 +33,7 @@ pub enum Abi { // NB: This ordering MUST match the AbiDatas array below. // (This is ensured by the test indices_are_correct().) - // Single platform ABIs come first (`for_arch()` relies on this) + // Single platform ABIs Cdecl, Stdcall, Fastcall, @@ -41,7 +42,7 @@ pub enum Abi { Win64, SysV64, - // Multiplatform ABIs second + // Multiplatform / generic ABIs Rust, C, System, @@ -64,41 +65,31 @@ pub enum Architecture { pub struct AbiData { abi: Abi, - // Name of this ABI as we like it called. + /// Name of this ABI as we like it called. name: &'static str, -} -#[derive(Copy, Clone)] -pub enum AbiArchitecture { - /// Not a real ABI (e.g., intrinsic) - Rust, - /// An ABI that specifies cross-platform defaults (e.g., "C") - All, - /// Multiple architectures (bitset) - Archs(u32) + /// A generic ABI is supported on all platforms. + generic: bool, } #[allow(non_upper_case_globals)] const AbiDatas: &'static [AbiData] = &[ // Platform-specific ABIs - AbiData {abi: Abi::Cdecl, name: "cdecl" }, - AbiData {abi: Abi::Stdcall, name: "stdcall" }, - AbiData {abi: Abi::Fastcall, name: "fastcall" }, - AbiData {abi: Abi::Vectorcall, name: "vectorcall"}, - AbiData {abi: Abi::Aapcs, name: "aapcs" }, - AbiData {abi: Abi::Win64, name: "win64" }, - AbiData {abi: Abi::SysV64, name: "sysv64" }, + AbiData {abi: Abi::Cdecl, name: "cdecl", generic: false }, + AbiData {abi: Abi::Stdcall, name: "stdcall", generic: false }, + AbiData {abi: Abi::Fastcall, name: "fastcall", generic: false }, + AbiData {abi: Abi::Vectorcall, name: "vectorcall", generic: false}, + AbiData {abi: Abi::Aapcs, name: "aapcs", generic: false }, + AbiData {abi: Abi::Win64, name: "win64", generic: false }, + AbiData {abi: Abi::SysV64, name: "sysv64", generic: false }, // Cross-platform ABIs - // - // NB: Do not adjust this ordering without - // adjusting the indices below. - AbiData {abi: Abi::Rust, name: "Rust" }, - AbiData {abi: Abi::C, name: "C" }, - AbiData {abi: Abi::System, name: "system" }, - AbiData {abi: Abi::RustIntrinsic, name: "rust-intrinsic" }, - AbiData {abi: Abi::RustCall, name: "rust-call" }, - AbiData {abi: Abi::PlatformIntrinsic, name: "platform-intrinsic" } + AbiData {abi: Abi::Rust, name: "Rust", generic: true }, + AbiData {abi: Abi::C, name: "C", generic: true }, + AbiData {abi: Abi::System, name: "system", generic: true }, + AbiData {abi: Abi::RustIntrinsic, name: "rust-intrinsic", generic: true }, + AbiData {abi: Abi::RustCall, name: "rust-call", generic: true }, + AbiData {abi: Abi::PlatformIntrinsic, name: "platform-intrinsic", generic: true }, ]; /// Returns the ABI with the given name (if any). @@ -124,6 +115,10 @@ impl Abi { pub fn name(&self) -> &'static str { self.data().name } + + pub fn generic(&self) -> bool { + self.data().generic + } } impl fmt::Display for Abi { @@ -146,6 +141,7 @@ impl fmt::Display for Os { Os::Netbsd => "netbsd".fmt(f), Os::Openbsd => "openbsd".fmt(f), Os::NaCl => "nacl".fmt(f), + Os::Haiku => "haiku".fmt(f), Os::Solaris => "solaris".fmt(f), } } diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs index 40c8ba93bd5..f077ead1f8e 100644 --- a/src/libsyntax/ast.rs +++ b/src/libsyntax/ast.rs @@ -27,7 +27,9 @@ use tokenstream::{TokenTree}; use std::fmt; use std::rc::Rc; -use serialize::{Encodable, Decodable, Encoder, Decoder}; +use std::u32; + +use serialize::{self, Encodable, Decodable, Encoder, Decoder}; /// A name is a part of an identifier, representing a string or gensym. It's /// the result of interning. @@ -69,7 +71,7 @@ impl Encodable for Name { impl Decodable for Name { fn decode<D: Decoder>(d: &mut D) -> Result<Name, D::Error> { - Ok(token::intern(&d.read_str()?[..])) + Ok(token::intern(&d.read_str()?)) } } @@ -119,6 +121,7 @@ impl fmt::Debug for Lifetime { /// A lifetime definition, e.g. `'a: 'b+'c+'d` #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub struct LifetimeDef { + pub attrs: ThinVec<Attribute>, pub lifetime: Lifetime, pub bounds: Vec<Lifetime> } @@ -158,12 +161,12 @@ impl Path { Path { span: s, global: false, - segments: vec!( + segments: vec![ PathSegment { identifier: identifier, parameters: PathParameters::none() } - ), + ], } } } @@ -298,17 +301,53 @@ pub struct ParenthesizedParameterData { pub output: Option<P<Ty>>, } -pub type CrateNum = u32; +#[derive(Clone, Copy, PartialEq, PartialOrd, Eq, Ord, Hash, Debug)] +pub struct NodeId(u32); + +impl NodeId { + pub fn new(x: usize) -> NodeId { + assert!(x < (u32::MAX as usize)); + NodeId(x as u32) + } + + pub fn from_u32(x: u32) -> NodeId { + NodeId(x) + } + + pub fn as_usize(&self) -> usize { + self.0 as usize + } + + pub fn as_u32(&self) -> u32 { + self.0 + } +} + +impl fmt::Display for NodeId { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Display::fmt(&self.0, f) + } +} -pub type NodeId = u32; +impl serialize::UseSpecializedEncodable for NodeId { + fn default_encode<S: Encoder>(&self, s: &mut S) -> Result<(), S::Error> { + s.emit_u32(self.0) + } +} + +impl serialize::UseSpecializedDecodable for NodeId { + fn default_decode<D: Decoder>(d: &mut D) -> Result<NodeId, D::Error> { + d.read_u32().map(NodeId) + } +} /// Node id used to represent the root of the crate. -pub const CRATE_NODE_ID: NodeId = 0; +pub const CRATE_NODE_ID: NodeId = NodeId(0); /// When parsing and doing expansions, we initially give all AST nodes this AST /// node value. Then later, in the renumber pass, we renumber them to have /// small, positive ids. -pub const DUMMY_NODE_ID: NodeId = !0; +pub const DUMMY_NODE_ID: NodeId = NodeId(!0); /// The AST represents all type param bounds as types. /// typeck::collect::compute_bounds matches these against @@ -332,6 +371,7 @@ pub type TyParamBounds = P<[TyParamBound]>; #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub struct TyParam { + pub attrs: ThinVec<Attribute>, pub ident: Ident, pub id: NodeId, pub bounds: TyParamBounds, @@ -437,7 +477,6 @@ pub type CrateConfig = Vec<P<MetaItem>>; pub struct Crate { pub module: Mod, pub attrs: Vec<Attribute>, - pub config: CrateConfig, pub span: Span, pub exported_macros: Vec<MacroDef>, } @@ -555,7 +594,7 @@ impl Pat { PatKind::Box(ref s) | PatKind::Ref(ref s, _) => { s.walk(it) } - PatKind::Vec(ref before, ref slice, ref after) => { + PatKind::Slice(ref before, ref slice, ref after) => { before.iter().all(|p| p.walk(it)) && slice.iter().all(|p| p.walk(it)) && after.iter().all(|p| p.walk(it)) @@ -631,8 +670,8 @@ pub enum PatKind { /// A range pattern, e.g. `1...2` Range(P<Expr>, P<Expr>), /// `[a, b, ..i, y, z]` is represented as: - /// `PatKind::Vec(box [a, b], Some(i), box [y, z])` - Vec(Vec<P<Pat>>, Option<P<Pat>>, Vec<P<Pat>>), + /// `PatKind::Slice(box [a, b], Some(i), box [y, z])` + Slice(Vec<P<Pat>>, Option<P<Pat>>, Vec<P<Pat>>), /// A macro pattern; pre-expansion Mac(Mac), } @@ -860,6 +899,7 @@ pub struct Field { pub ident: SpannedIdent, pub expr: P<Expr>, pub span: Span, + pub is_shorthand: bool, } pub type SpannedIdent = Spanned<Ident>; @@ -1393,10 +1433,10 @@ pub struct BareFnTy { /// The different kinds of types recognized by the compiler #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub enum TyKind { - /// A variable-length array (`[T]`) - Vec(P<Ty>), + /// A variable-length slice (`[T]`) + Slice(P<Ty>), /// A fixed length array (`[T; n]`) - FixedLengthVec(P<Ty>, P<Expr>), + Array(P<Ty>, P<Expr>), /// A raw pointer (`*const T` or `*mut T`) Ptr(MutTy), /// A reference (`&'a T` or `&'a mut T`) @@ -1972,8 +2012,6 @@ pub struct MacroDef { pub id: NodeId, pub span: Span, pub imported_from: Option<Ident>, - pub export: bool, - pub use_locally: bool, pub allow_internal_unstable: bool, pub body: Vec<TokenTree>, } diff --git a/src/libsyntax/attr.rs b/src/libsyntax/attr.rs index 81ee96459fd..0335f210347 100644 --- a/src/libsyntax/attr.rs +++ b/src/libsyntax/attr.rs @@ -501,10 +501,7 @@ pub fn requests_inline(attrs: &[Attribute]) -> bool { } /// Tests if a cfg-pattern matches the cfg set -pub fn cfg_matches(cfgs: &[P<MetaItem>], cfg: &ast::MetaItem, - sess: &ParseSess, - features: Option<&Features>) - -> bool { +pub fn cfg_matches(cfg: &ast::MetaItem, sess: &ParseSess, features: Option<&Features>) -> bool { match cfg.node { ast::MetaItemKind::List(ref pred, ref mis) => { for mi in mis.iter() { @@ -518,10 +515,10 @@ pub fn cfg_matches(cfgs: &[P<MetaItem>], cfg: &ast::MetaItem, // that they won't fail with the loop above. match &pred[..] { "any" => mis.iter().any(|mi| { - cfg_matches(cfgs, mi.meta_item().unwrap(), sess, features) + cfg_matches(mi.meta_item().unwrap(), sess, features) }), "all" => mis.iter().all(|mi| { - cfg_matches(cfgs, mi.meta_item().unwrap(), sess, features) + cfg_matches(mi.meta_item().unwrap(), sess, features) }), "not" => { if mis.len() != 1 { @@ -529,7 +526,7 @@ pub fn cfg_matches(cfgs: &[P<MetaItem>], cfg: &ast::MetaItem, return false; } - !cfg_matches(cfgs, mis[0].meta_item().unwrap(), sess, features) + !cfg_matches(mis[0].meta_item().unwrap(), sess, features) }, p => { span_err!(sess.span_diagnostic, cfg.span, E0537, "invalid predicate `{}`", p); @@ -541,7 +538,7 @@ pub fn cfg_matches(cfgs: &[P<MetaItem>], cfg: &ast::MetaItem, if let (Some(feats), Some(gated_cfg)) = (features, GatedCfg::gate(cfg)) { gated_cfg.check_and_emit(sess, feats); } - contains(cfgs, cfg) + contains(&sess.config, cfg) } } } @@ -895,7 +892,7 @@ pub fn find_repr_attrs(diagnostic: &Handler, attr: &Attribute) -> Vec<ReprAttr> "packed" => Some(ReprPacked), "simd" => Some(ReprSimd), _ => match int_type_of_word(word) { - Some(ity) => Some(ReprInt(item.span, ity)), + Some(ity) => Some(ReprInt(ity)), None => { // Not a word we recognize span_err!(diagnostic, item.span, E0552, @@ -939,7 +936,7 @@ fn int_type_of_word(s: &str) -> Option<IntType> { #[derive(PartialEq, Debug, RustcEncodable, RustcDecodable, Copy, Clone)] pub enum ReprAttr { ReprAny, - ReprInt(Span, IntType), + ReprInt(IntType), ReprExtern, ReprPacked, ReprSimd, @@ -949,7 +946,7 @@ impl ReprAttr { pub fn is_ffi_safe(&self) -> bool { match *self { ReprAny => false, - ReprInt(_sp, ity) => ity.is_ffi_safe(), + ReprInt(ity) => ity.is_ffi_safe(), ReprExtern => true, ReprPacked => false, ReprSimd => true, diff --git a/src/libsyntax/codemap.rs b/src/libsyntax/codemap.rs index cd6f2874954..49012ad036a 100644 --- a/src/libsyntax/codemap.rs +++ b/src/libsyntax/codemap.rs @@ -364,6 +364,46 @@ impl CodeMap { } } + /// Returns `Some(span)`, a union of the lhs and rhs span. The lhs must precede the rhs. If + /// there are gaps between lhs and rhs, the resulting union will cross these gaps. + /// For this to work, the spans have to be: + /// * the expn_id of both spans much match + /// * the lhs span needs to end on the same line the rhs span begins + /// * the lhs span must start at or before the rhs span + pub fn merge_spans(&self, sp_lhs: Span, sp_rhs: Span) -> Option<Span> { + use std::cmp; + + // make sure we're at the same expansion id + if sp_lhs.expn_id != sp_rhs.expn_id { + return None; + } + + let lhs_end = match self.lookup_line(sp_lhs.hi) { + Ok(x) => x, + Err(_) => return None + }; + let rhs_begin = match self.lookup_line(sp_rhs.lo) { + Ok(x) => x, + Err(_) => return None + }; + + // if we must cross lines to merge, don't merge + if lhs_end.line != rhs_begin.line { + return None; + } + + // ensure these follow the expected order and we don't overlap + if (sp_lhs.lo <= sp_rhs.lo) && (sp_lhs.hi <= sp_rhs.lo) { + Some(Span { + lo: cmp::min(sp_lhs.lo, sp_rhs.lo), + hi: cmp::max(sp_lhs.hi, sp_rhs.hi), + expn_id: sp_lhs.expn_id, + }) + } else { + None + } + } + pub fn span_to_string(&self, sp: Span) -> String { if sp == COMMAND_LINE_SP { return "<command line option>".to_string(); @@ -764,7 +804,7 @@ impl CodeMap { } pub fn macro_backtrace(&self, span: Span) -> Vec<MacroBacktrace> { - let mut last_span = DUMMY_SP; + let mut prev_span = DUMMY_SP; let mut span = span; let mut result = vec![]; loop { @@ -787,14 +827,14 @@ impl CodeMap { None => break, Some((call_site, macro_decl_name, def_site_span)) => { // Don't print recursive invocations - if !call_site.source_equal(&last_span) { + if !call_site.source_equal(&prev_span) { result.push(MacroBacktrace { call_site: call_site, macro_decl_name: macro_decl_name, def_site_span: def_site_span, }); } - last_span = span; + prev_span = span; span = call_site; } } @@ -819,6 +859,9 @@ impl CodeMapper for CodeMap { fn macro_backtrace(&self, span: Span) -> Vec<MacroBacktrace> { self.macro_backtrace(span) } + fn merge_spans(&self, sp_lhs: Span, sp_rhs: Span) -> Option<Span> { + self.merge_spans(sp_lhs, sp_rhs) + } } // _____________________________________________________________________________ @@ -1072,6 +1115,40 @@ mod tests { blork.rs:1:1: 1:12\n `first line.`\n"); } + /// Test merging two spans on the same line + #[test] + fn span_merging() { + let cm = CodeMap::new(); + let inputtext = "bbbb BB bb CCC\n"; + let selection1 = " ~~ \n"; + let selection2 = " ~~~\n"; + cm.new_filemap_and_lines("blork.rs", None, inputtext); + let span1 = span_from_selection(inputtext, selection1); + let span2 = span_from_selection(inputtext, selection2); + + if let Some(sp) = cm.merge_spans(span1, span2) { + let sstr = cm.span_to_expanded_string(sp); + assert_eq!(sstr, "blork.rs:1:6: 1:15\n`BB bb CCC`\n"); + } + else { + assert!(false); + } + } + + /// Test failing to merge two spans on different lines + #[test] + fn span_merging_fail() { + let cm = CodeMap::new(); + let inputtext = "bbbb BB\ncc CCC\n"; + let selection1 = " ~~\n \n"; + let selection2 = " \n ~~~\n"; + cm.new_filemap_and_lines("blork.rs", None, inputtext); + let span1 = span_from_selection(inputtext, selection1); + let span2 = span_from_selection(inputtext, selection2); + + assert!(cm.merge_spans(span1, span2).is_none()); + } + /// Returns the span corresponding to the `n`th occurrence of /// `substring` in `source_text`. trait CodeMapExtension { diff --git a/src/libsyntax/config.rs b/src/libsyntax/config.rs index 3f5b294cc04..946257a16d5 100644 --- a/src/libsyntax/config.rs +++ b/src/libsyntax/config.rs @@ -9,7 +9,7 @@ // except according to those terms. use attr::HasAttrs; -use feature_gate::{emit_feature_err, EXPLAIN_STMT_ATTR_SYNTAX, Features, get_features, GateIssue}; +use feature_gate::{feature_err, EXPLAIN_STMT_ATTR_SYNTAX, Features, get_features, GateIssue}; use {fold, attr}; use ast; use codemap::{Spanned, respan}; @@ -20,7 +20,6 @@ use util::small_vector::SmallVector; /// A folder that strips out items that do not belong in the current configuration. pub struct StripUnconfigured<'a> { - pub config: &'a ast::CrateConfig, pub should_test: bool, pub sess: &'a ParseSess, pub features: Option<&'a Features>, @@ -32,7 +31,6 @@ pub fn features(mut krate: ast::Crate, sess: &ParseSess, should_test: bool) let features; { let mut strip_unconfigured = StripUnconfigured { - config: &krate.config.clone(), should_test: should_test, sess: sess, features: None, @@ -107,7 +105,7 @@ impl<'a> StripUnconfigured<'a> { use attr::cfg_matches; match (cfg.meta_item(), mi.meta_item()) { (Some(cfg), Some(mi)) => - if cfg_matches(self.config, &cfg, self.sess, self.features) { + if cfg_matches(&cfg, self.sess, self.features) { self.process_cfg_attr(respan(mi.span, ast::Attribute_ { id: attr::mk_attr_id(), style: attr.node.style, @@ -126,7 +124,7 @@ impl<'a> StripUnconfigured<'a> { } // Determine if a node with the given attributes should be included in this configuation. - fn in_cfg(&mut self, attrs: &[ast::Attribute]) -> bool { + pub fn in_cfg(&mut self, attrs: &[ast::Attribute]) -> bool { attrs.iter().all(|attr| { // When not compiling with --test we should not compile the #[test] functions if !self.should_test && is_test_or_bench(attr) { @@ -148,20 +146,24 @@ impl<'a> StripUnconfigured<'a> { return true; } - attr::cfg_matches(self.config, mis[0].meta_item().unwrap(), self.sess, self.features) + attr::cfg_matches(mis[0].meta_item().unwrap(), self.sess, self.features) }) } // Visit attributes on expression and statements (but not attributes on items in blocks). - fn visit_stmt_or_expr_attrs(&mut self, attrs: &[ast::Attribute]) { + fn visit_expr_attrs(&mut self, attrs: &[ast::Attribute]) { // flag the offending attributes for attr in attrs.iter() { if !self.features.map(|features| features.stmt_expr_attributes).unwrap_or(true) { - emit_feature_err(&self.sess.span_diagnostic, - "stmt_expr_attributes", - attr.span, - GateIssue::Language, - EXPLAIN_STMT_ATTR_SYNTAX); + let mut err = feature_err(&self.sess, + "stmt_expr_attributes", + attr.span, + GateIssue::Language, + EXPLAIN_STMT_ATTR_SYNTAX); + if attr.node.is_sugared_doc { + err.help("`///` is for documentation comments. For a plain comment, use `//`."); + } + err.emit(); } } } @@ -227,7 +229,7 @@ impl<'a> StripUnconfigured<'a> { } pub fn configure_expr(&mut self, expr: P<ast::Expr>) -> P<ast::Expr> { - self.visit_stmt_or_expr_attrs(expr.attrs()); + self.visit_expr_attrs(expr.attrs()); // If an expr is valid to cfg away it will have been removed by the // outer stmt or expression folder before descending in here. @@ -245,7 +247,6 @@ impl<'a> StripUnconfigured<'a> { } pub fn configure_stmt(&mut self, stmt: ast::Stmt) -> Option<ast::Stmt> { - self.visit_stmt_or_expr_attrs(stmt.attrs()); self.configure(stmt) } } @@ -303,6 +304,6 @@ fn is_cfg(attr: &ast::Attribute) -> bool { attr.check_name("cfg") } -fn is_test_or_bench(attr: &ast::Attribute) -> bool { +pub fn is_test_or_bench(attr: &ast::Attribute) -> bool { attr.check_name("test") || attr.check_name("bench") } diff --git a/src/libsyntax/diagnostics/plugin.rs b/src/libsyntax/diagnostics/plugin.rs index 4e50299e836..81c8e0bdb82 100644 --- a/src/libsyntax/diagnostics/plugin.rs +++ b/src/libsyntax/diagnostics/plugin.rs @@ -215,7 +215,7 @@ pub fn expand_build_diagnostic_array<'cx>(ecx: &'cx mut ExtCtxt, let ty = ecx.ty( span, - ast::TyKind::FixedLengthVec( + ast::TyKind::Array( ecx.ty( span, ast::TyKind::Tup(vec![ty_str.clone(), ty_str]) diff --git a/src/libsyntax/ext/base.rs b/src/libsyntax/ext/base.rs index fb4816d3847..cc097ab0efa 100644 --- a/src/libsyntax/ext/base.rs +++ b/src/libsyntax/ext/base.rs @@ -8,30 +8,26 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. -pub use self::SyntaxExtension::*; +pub use self::SyntaxExtension::{MultiDecorator, MultiModifier, NormalTT, IdentTT}; use ast::{self, Attribute, Name, PatKind}; use attr::HasAttrs; use codemap::{self, CodeMap, ExpnInfo, Spanned, respan}; use syntax_pos::{Span, ExpnId, NO_EXPANSION}; use errors::DiagnosticBuilder; -use ext::expand::{self, Invocation, Expansion}; +use ext::expand::{self, Expansion}; use ext::hygiene::Mark; -use ext::tt::macro_rules; -use parse; -use parse::parser; +use fold::{self, Folder}; +use parse::{self, parser}; use parse::token; use parse::token::{InternedString, str_to_ident}; use ptr::P; -use std_inject; use util::small_vector::SmallVector; -use fold::Folder; -use feature_gate; -use std::collections::HashMap; use std::path::PathBuf; use std::rc::Rc; -use tokenstream; +use std::default::Default; +use tokenstream::{self, TokenStream}; #[derive(Debug,Clone)] @@ -146,6 +142,50 @@ impl Into<Vec<Annotatable>> for Annotatable { } } +pub trait ProcMacro { + fn expand<'cx>(&self, + ecx: &'cx mut ExtCtxt, + span: Span, + ts: TokenStream) + -> TokenStream; +} + +impl<F> ProcMacro for F + where F: Fn(TokenStream) -> TokenStream +{ + fn expand<'cx>(&self, + _ecx: &'cx mut ExtCtxt, + _span: Span, + ts: TokenStream) + -> TokenStream { + // FIXME setup implicit context in TLS before calling self. + (*self)(ts) + } +} + +pub trait AttrProcMacro { + fn expand<'cx>(&self, + ecx: &'cx mut ExtCtxt, + span: Span, + annotation: TokenStream, + annotated: TokenStream) + -> TokenStream; +} + +impl<F> AttrProcMacro for F + where F: Fn(TokenStream, TokenStream) -> TokenStream +{ + fn expand<'cx>(&self, + _ecx: &'cx mut ExtCtxt, + _span: Span, + annotation: TokenStream, + annotated: TokenStream) + -> TokenStream { + // FIXME setup implicit context in TLS before calling self. + (*self)(annotation, annotated) + } +} + /// Represents a thing that maps token trees to Macro Results pub trait TTMacroExpander { fn expand<'cx>(&self, @@ -177,7 +217,8 @@ pub trait IdentMacroExpander { cx: &'cx mut ExtCtxt, sp: Span, ident: ast::Ident, - token_tree: Vec<tokenstream::TokenTree> ) + token_tree: Vec<tokenstream::TokenTree>, + attrs: Vec<ast::Attribute>) -> Box<MacResult+'cx>; } @@ -193,7 +234,8 @@ impl<F> IdentMacroExpander for F cx: &'cx mut ExtCtxt, sp: Span, ident: ast::Ident, - token_tree: Vec<tokenstream::TokenTree> ) + token_tree: Vec<tokenstream::TokenTree>, + _attrs: Vec<ast::Attribute>) -> Box<MacResult+'cx> { (*self)(cx, sp, ident, token_tree) @@ -437,11 +479,22 @@ pub enum SyntaxExtension { /// based upon it. /// /// `#[derive(...)]` is a `MultiItemDecorator`. - MultiDecorator(Box<MultiItemDecorator + 'static>), + /// + /// Prefer ProcMacro or MultiModifier since they are more flexible. + MultiDecorator(Box<MultiItemDecorator>), /// A syntax extension that is attached to an item and modifies it - /// in-place. More flexible version than Modifier. - MultiModifier(Box<MultiItemModifier + 'static>), + /// in-place. Also allows decoration, i.e., creating new items. + MultiModifier(Box<MultiItemModifier>), + + /// A function-like procedural macro. TokenStream -> TokenStream. + ProcMacro(Box<ProcMacro>), + + /// An attribute-like procedural macro. TokenStream, TokenStream -> TokenStream. + /// The first TokenSteam is the attribute, the second is the annotated item. + /// Allows modification of the input items and adding new items, similar to + /// MultiModifier, but uses TokenStreams, rather than AST nodes. + AttrProcMacro(Box<AttrProcMacro>), /// A normal, function-like syntax extension. /// @@ -449,50 +502,54 @@ pub enum SyntaxExtension { /// /// The `bool` dictates whether the contents of the macro can /// directly use `#[unstable]` things (true == yes). - NormalTT(Box<TTMacroExpander + 'static>, Option<Span>, bool), + NormalTT(Box<TTMacroExpander>, Option<Span>, bool), /// A function-like syntax extension that has an extra ident before /// the block. /// - IdentTT(Box<IdentMacroExpander + 'static>, Option<Span>, bool), + IdentTT(Box<IdentMacroExpander>, Option<Span>, bool), - /// Represents `macro_rules!` itself. - MacroRulesTT, + CustomDerive(Box<MultiItemModifier>), } pub type NamedSyntaxExtension = (Name, SyntaxExtension); pub trait Resolver { - fn load_crate(&mut self, extern_crate: &ast::Item, allows_macros: bool) -> Vec<LoadedMacro>; fn next_node_id(&mut self) -> ast::NodeId; + fn get_module_scope(&mut self, id: ast::NodeId) -> Mark; fn visit_expansion(&mut self, mark: Mark, expansion: &Expansion); - fn add_macro(&mut self, scope: Mark, ident: ast::Ident, ext: Rc<SyntaxExtension>); + fn add_macro(&mut self, scope: Mark, def: ast::MacroDef, export: bool); + fn add_ext(&mut self, ident: ast::Ident, ext: Rc<SyntaxExtension>); fn add_expansions_at_stmt(&mut self, id: ast::NodeId, macros: Vec<Mark>); fn find_attr_invoc(&mut self, attrs: &mut Vec<Attribute>) -> Option<Attribute>; - fn resolve_invoc(&mut self, invoc: &Invocation) -> Option<Rc<SyntaxExtension>>; + fn resolve_macro(&mut self, scope: Mark, path: &ast::Path, force: bool) + -> Result<Rc<SyntaxExtension>, Determinacy>; } -pub enum LoadedMacro { - Def(ast::MacroDef), - CustomDerive(String, Box<MultiItemModifier>), +#[derive(Copy, Clone, Debug)] +pub enum Determinacy { + Determined, + Undetermined, } pub struct DummyResolver; impl Resolver for DummyResolver { - fn load_crate(&mut self, _extern_crate: &ast::Item, _allows_macros: bool) -> Vec<LoadedMacro> { - Vec::new() - } fn next_node_id(&mut self) -> ast::NodeId { ast::DUMMY_NODE_ID } + fn get_module_scope(&mut self, _id: ast::NodeId) -> Mark { Mark::root() } fn visit_expansion(&mut self, _invoc: Mark, _expansion: &Expansion) {} - fn add_macro(&mut self, _scope: Mark, _ident: ast::Ident, _ext: Rc<SyntaxExtension>) {} + fn add_macro(&mut self, _scope: Mark, _def: ast::MacroDef, _export: bool) {} + fn add_ext(&mut self, _ident: ast::Ident, _ext: Rc<SyntaxExtension>) {} fn add_expansions_at_stmt(&mut self, _id: ast::NodeId, _macros: Vec<Mark>) {} fn find_attr_invoc(&mut self, _attrs: &mut Vec<Attribute>) -> Option<Attribute> { None } - fn resolve_invoc(&mut self, _invoc: &Invocation) -> Option<Rc<SyntaxExtension>> { None } + fn resolve_macro(&mut self, _scope: Mark, _path: &ast::Path, _force: bool) + -> Result<Rc<SyntaxExtension>, Determinacy> { + Err(Determinacy::Determined) + } } #[derive(Clone)] @@ -507,7 +564,9 @@ pub struct ExpansionData { pub depth: usize, pub backtrace: ExpnId, pub module: Rc<ModuleData>, - pub in_block: bool, + + // True if non-inline modules without a `#[path]` are forbidden at the root of this expansion. + pub no_noninline_mod: bool, } /// One of these is made during expansion and incrementally updated as we go; @@ -515,34 +574,30 @@ pub struct ExpansionData { /// -> expn_info of their expansion context stored into their span. pub struct ExtCtxt<'a> { pub parse_sess: &'a parse::ParseSess, - pub cfg: ast::CrateConfig, pub ecfg: expand::ExpansionConfig<'a>, pub crate_root: Option<&'static str>, pub resolver: &'a mut Resolver, - pub exported_macros: Vec<ast::MacroDef>, - pub derive_modes: HashMap<InternedString, Box<MultiItemModifier>>, + pub resolve_err_count: usize, pub current_expansion: ExpansionData, } impl<'a> ExtCtxt<'a> { - pub fn new(parse_sess: &'a parse::ParseSess, cfg: ast::CrateConfig, + pub fn new(parse_sess: &'a parse::ParseSess, ecfg: expand::ExpansionConfig<'a>, resolver: &'a mut Resolver) -> ExtCtxt<'a> { ExtCtxt { parse_sess: parse_sess, - cfg: cfg, ecfg: ecfg, crate_root: None, - exported_macros: Vec::new(), resolver: resolver, - derive_modes: HashMap::new(), + resolve_err_count: 0, current_expansion: ExpansionData { mark: Mark::root(), depth: 0, backtrace: NO_EXPANSION, module: Rc::new(ModuleData { mod_path: Vec::new(), directory: PathBuf::new() }), - in_block: false, + no_noninline_mod: false, }, } } @@ -560,11 +615,11 @@ impl<'a> ExtCtxt<'a> { pub fn new_parser_from_tts(&self, tts: &[tokenstream::TokenTree]) -> parser::Parser<'a> { - parse::tts_to_parser(self.parse_sess, tts.to_vec(), self.cfg()) + parse::tts_to_parser(self.parse_sess, tts.to_vec()) } pub fn codemap(&self) -> &'a CodeMap { self.parse_sess.codemap() } pub fn parse_sess(&self) -> &'a parse::ParseSess { self.parse_sess } - pub fn cfg(&self) -> ast::CrateConfig { self.cfg.clone() } + pub fn cfg(&self) -> &ast::CrateConfig { &self.parse_sess.config } pub fn call_site(&self) -> Span { self.codemap().with_expn_info(self.backtrace(), |ei| match ei { Some(expn_info) => expn_info.call_site, @@ -613,31 +668,6 @@ impl<'a> ExtCtxt<'a> { } pub fn bt_pop(&mut self) {} - pub fn insert_macro(&mut self, def: ast::MacroDef) { - if def.export { - self.exported_macros.push(def.clone()); - } - if def.use_locally { - let ext = macro_rules::compile(self, &def); - self.resolver.add_macro(self.current_expansion.mark, def.ident, Rc::new(ext)); - } - } - - pub fn insert_custom_derive(&mut self, name: &str, ext: Box<MultiItemModifier>, sp: Span) { - if !self.ecfg.enable_rustc_macro() { - feature_gate::emit_feature_err(&self.parse_sess.span_diagnostic, - "rustc_macro", - sp, - feature_gate::GateIssue::Language, - "loading custom derive macro crates \ - is experimentally supported"); - } - let name = token::intern_and_get_ident(name); - if self.derive_modes.insert(name.clone(), ext).is_some() { - self.span_err(sp, &format!("cannot shadow existing derive mode `{}`", name)); - } - } - pub fn struct_span_warn(&self, sp: Span, msg: &str) @@ -712,28 +742,6 @@ impl<'a> ExtCtxt<'a> { pub fn name_of(&self, st: &str) -> ast::Name { token::intern(st) } - - pub fn initialize(&mut self, user_exts: Vec<NamedSyntaxExtension>, krate: &ast::Crate) { - if std_inject::no_core(&krate) { - self.crate_root = None; - } else if std_inject::no_std(&krate) { - self.crate_root = Some("core"); - } else { - self.crate_root = Some("std"); - } - - for (name, extension) in user_exts { - let ident = ast::Ident::with_empty_ctxt(name); - self.resolver.add_macro(Mark::root(), ident, Rc::new(extension)); - } - - let mut module = ModuleData { - mod_path: vec![token::str_to_ident(&self.ecfg.crate_name)], - directory: PathBuf::from(self.parse_sess.codemap().span_to_filename(krate.span)), - }; - module.directory.pop(); - self.current_expansion.module = Rc::new(module); - } } /// Extract a string literal from the macro expanded version of `expr`, @@ -818,3 +826,17 @@ pub fn get_exprs_from_tts(cx: &mut ExtCtxt, } Some(es) } + +pub struct ChangeSpan { + pub span: Span +} + +impl Folder for ChangeSpan { + fn new_span(&mut self, _sp: Span) -> Span { + self.span + } + + fn fold_mac(&mut self, mac: ast::Mac) -> ast::Mac { + fold::noop_fold_mac(mac, self) + } +} diff --git a/src/libsyntax/ext/build.rs b/src/libsyntax/ext/build.rs index b81d95a6998..37bd83be7b4 100644 --- a/src/libsyntax/ext/build.rs +++ b/src/libsyntax/ext/build.rs @@ -73,6 +73,7 @@ pub trait AstBuilder { fn typaram(&self, span: Span, id: ast::Ident, + attrs: Vec<ast::Attribute>, bounds: ast::TyParamBounds, default: Option<P<ast::Ty>>) -> ast::TyParam; @@ -83,6 +84,7 @@ pub trait AstBuilder { fn lifetime_def(&self, span: Span, name: ast::Name, + attrs: Vec<ast::Attribute>, bounds: Vec<ast::Lifetime>) -> ast::LifetimeDef; @@ -96,7 +98,7 @@ pub trait AstBuilder { ident: ast::Ident, typ: P<ast::Ty>, ex: P<ast::Expr>) - -> P<ast::Stmt>; + -> ast::Stmt; fn stmt_let_type_only(&self, span: Span, ty: P<ast::Ty>) -> ast::Stmt; fn stmt_item(&self, sp: Span, item: P<ast::Item>) -> ast::Stmt; @@ -310,7 +312,7 @@ impl<'a> AstBuilder for ExtCtxt<'a> { self.path_all(span, false, strs, Vec::new(), Vec::new(), Vec::new()) } fn path_ident(&self, span: Span, id: ast::Ident) -> ast::Path { - self.path(span, vec!(id)) + self.path(span, vec![id]) } fn path_global(&self, span: Span, strs: Vec<ast::Ident> ) -> ast::Path { self.path_all(span, true, strs, Vec::new(), Vec::new(), Vec::new()) @@ -441,7 +443,7 @@ impl<'a> AstBuilder for ExtCtxt<'a> { true, self.std_path(&["option", "Option"]), Vec::new(), - vec!( ty ), + vec![ ty ], Vec::new())) } @@ -452,11 +454,13 @@ impl<'a> AstBuilder for ExtCtxt<'a> { fn typaram(&self, span: Span, id: ast::Ident, + attrs: Vec<ast::Attribute>, bounds: ast::TyParamBounds, default: Option<P<ast::Ty>>) -> ast::TyParam { ast::TyParam { ident: id, id: ast::DUMMY_NODE_ID, + attrs: attrs.into(), bounds: bounds, default: default, span: span @@ -473,7 +477,7 @@ impl<'a> AstBuilder for ExtCtxt<'a> { fn ty_vars_global(&self, ty_params: &P<[ast::TyParam]>) -> Vec<P<ast::Ty>> { ty_params .iter() - .map(|p| self.ty_path(self.path_global(DUMMY_SP, vec!(p.ident)))) + .map(|p| self.ty_path(self.path_global(DUMMY_SP, vec![p.ident]))) .collect() } @@ -503,9 +507,11 @@ impl<'a> AstBuilder for ExtCtxt<'a> { fn lifetime_def(&self, span: Span, name: ast::Name, + attrs: Vec<ast::Attribute>, bounds: Vec<ast::Lifetime>) -> ast::LifetimeDef { ast::LifetimeDef { + attrs: attrs.into(), lifetime: self.lifetime(span, name), bounds: bounds } @@ -556,7 +562,7 @@ impl<'a> AstBuilder for ExtCtxt<'a> { ident: ast::Ident, typ: P<ast::Ty>, ex: P<ast::Expr>) - -> P<ast::Stmt> { + -> ast::Stmt { let pat = if mutbl { let binding_mode = ast::BindingMode::ByValue(ast::Mutability::Mutable); self.pat_ident_binding_mode(sp, ident, binding_mode) @@ -571,11 +577,11 @@ impl<'a> AstBuilder for ExtCtxt<'a> { span: sp, attrs: ast::ThinVec::new(), }); - P(ast::Stmt { + ast::Stmt { id: ast::DUMMY_NODE_ID, node: ast::StmtKind::Local(local), span: sp, - }) + } } // Generate `let _: Type;`, usually used for type assertions. @@ -707,7 +713,7 @@ impl<'a> AstBuilder for ExtCtxt<'a> { self.expr(b.span, ast::ExprKind::Block(b)) } fn field_imm(&self, span: Span, name: Ident, e: P<ast::Expr>) -> ast::Field { - ast::Field { ident: respan(span, name), expr: e, span: span } + ast::Field { ident: respan(span, name), expr: e, span: span, is_shorthand: false } } fn expr_struct(&self, span: Span, path: ast::Path, fields: Vec<ast::Field>) -> P<ast::Expr> { self.expr(span, ast::ExprKind::Struct(path, fields, None)) @@ -764,7 +770,7 @@ impl<'a> AstBuilder for ExtCtxt<'a> { fn expr_some(&self, sp: Span, expr: P<ast::Expr>) -> P<ast::Expr> { let some = self.std_path(&["option", "Option", "Some"]); - self.expr_call_global(sp, some, vec!(expr)) + self.expr_call_global(sp, some, vec![expr]) } fn expr_none(&self, sp: Span) -> P<ast::Expr> { @@ -788,14 +794,14 @@ impl<'a> AstBuilder for ExtCtxt<'a> { let expr_file = self.expr_str(span, token::intern_and_get_ident(&loc.file.name)); let expr_line = self.expr_u32(span, loc.line as u32); - let expr_file_line_tuple = self.expr_tuple(span, vec!(expr_file, expr_line)); + let expr_file_line_tuple = self.expr_tuple(span, vec![expr_file, expr_line]); let expr_file_line_ptr = self.expr_addr_of(span, expr_file_line_tuple); self.expr_call_global( span, self.std_path(&["rt", "begin_panic"]), - vec!( + vec![ self.expr_str(span, msg), - expr_file_line_ptr)) + expr_file_line_ptr]) } fn expr_unreachable(&self, span: Span) -> P<ast::Expr> { @@ -806,12 +812,12 @@ impl<'a> AstBuilder for ExtCtxt<'a> { fn expr_ok(&self, sp: Span, expr: P<ast::Expr>) -> P<ast::Expr> { let ok = self.std_path(&["result", "Result", "Ok"]); - self.expr_call_global(sp, ok, vec!(expr)) + self.expr_call_global(sp, ok, vec![expr]) } fn expr_err(&self, sp: Span, expr: P<ast::Expr>) -> P<ast::Expr> { let err = self.std_path(&["result", "Result", "Err"]); - self.expr_call_global(sp, err, vec!(expr)) + self.expr_call_global(sp, err, vec![expr]) } fn expr_try(&self, sp: Span, head: P<ast::Expr>) -> P<ast::Expr> { @@ -830,17 +836,17 @@ impl<'a> AstBuilder for ExtCtxt<'a> { // Err(__try_var) (pattern and expression resp.) let err_pat = self.pat_tuple_struct(sp, err_path.clone(), vec![binding_pat]); let err_inner_expr = self.expr_call(sp, self.expr_path(err_path), - vec!(binding_expr.clone())); + vec![binding_expr.clone()]); // return Err(__try_var) let err_expr = self.expr(sp, ast::ExprKind::Ret(Some(err_inner_expr))); // Ok(__try_var) => __try_var - let ok_arm = self.arm(sp, vec!(ok_pat), binding_expr); + let ok_arm = self.arm(sp, vec![ok_pat], binding_expr); // Err(__try_var) => return Err(__try_var) - let err_arm = self.arm(sp, vec!(err_pat), err_expr); + let err_arm = self.arm(sp, vec![err_pat], err_expr); // match head { Ok() => ..., Err() => ... } - self.expr_match(sp, head, vec!(ok_arm, err_arm)) + self.expr_match(sp, head, vec![ok_arm, err_arm]) } @@ -906,7 +912,7 @@ impl<'a> AstBuilder for ExtCtxt<'a> { fn arm(&self, _span: Span, pats: Vec<P<ast::Pat>>, expr: P<ast::Expr>) -> ast::Arm { ast::Arm { - attrs: vec!(), + attrs: vec![], pats: pats, guard: None, body: expr @@ -914,7 +920,7 @@ impl<'a> AstBuilder for ExtCtxt<'a> { } fn arm_unreachable(&self, span: Span) -> ast::Arm { - self.arm(span, vec!(self.pat_wild(span)), self.expr_unreachable(span)) + self.arm(span, vec![self.pat_wild(span)], self.expr_unreachable(span)) } fn expr_match(&self, span: Span, arg: P<ast::Expr>, arms: Vec<ast::Arm>) -> P<Expr> { @@ -964,7 +970,7 @@ impl<'a> AstBuilder for ExtCtxt<'a> { } fn lambda1(&self, span: Span, blk: P<ast::Block>, ident: ast::Ident) -> P<ast::Expr> { - self.lambda(span, vec!(ident), blk) + self.lambda(span, vec![ident], blk) } fn lambda_expr(&self, span: Span, ids: Vec<ast::Ident>, diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index 62e299684b7..e3b23e239f9 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -8,22 +8,26 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. -use ast::{Block, Crate, Ident, Mac_, PatKind}; -use ast::{MacStmtStyle, StmtKind, ItemKind}; +use ast::{Block, Ident, Mac_, PatKind}; +use ast::{Name, MacStmtStyle, StmtKind, ItemKind}; use ast; use ext::hygiene::Mark; -use ext::placeholders::{self, placeholder, PlaceholderExpander}; +use ext::placeholders::{placeholder, PlaceholderExpander}; use attr::{self, HasAttrs}; use codemap::{ExpnInfo, NameAndSpan, MacroBang, MacroAttribute}; use syntax_pos::{self, Span, ExpnId}; -use config::StripUnconfigured; +use config::{is_test_or_bench, StripUnconfigured}; use ext::base::*; use feature_gate::{self, Features}; use fold; use fold::*; -use parse::token::{intern, keywords}; +use parse::{ParseSess, PResult, lexer}; +use parse::parser::Parser; +use parse::token::{self, intern, keywords}; +use print::pprust; use ptr::P; -use tokenstream::TokenTree; +use std_inject; +use tokenstream::{TokenTree, TokenStream}; use util::small_vector::SmallVector; use visit::Visitor; @@ -35,12 +39,12 @@ macro_rules! expansions { ($($kind:ident: $ty:ty [$($vec:ident, $ty_elt:ty)*], $kind_name:expr, .$make:ident, $(.$fold:ident)* $(lift .$fold_elt:ident)*, $(.$visit:ident)* $(lift .$visit_elt:ident)*;)*) => { - #[derive(Copy, Clone)] + #[derive(Copy, Clone, PartialEq, Eq)] pub enum ExpansionKind { OptExpr, $( $kind, )* } pub enum Expansion { OptExpr(Option<P<ast::Expr>>), $( $kind($ty), )* } impl ExpansionKind { - fn name(self) -> &'static str { + pub fn name(self) -> &'static str { match self { ExpansionKind::OptExpr => "expression", $( ExpansionKind::$kind => $kind_name, )* @@ -103,6 +107,12 @@ macro_rules! expansions { self.expand(Expansion::$kind(SmallVector::one(node))).$make() })*)* } + + impl<'a> MacResult for ::ext::tt::macro_rules::ParserAnyMacro<'a> { + $(fn $make(self: Box<::ext::tt::macro_rules::ParserAnyMacro<'a>>) -> Option<$ty> { + Some(self.make(ExpansionKind::$kind).$make()) + })* + } } } @@ -165,50 +175,43 @@ impl Invocation { InvocationKind::Attr { ref attr, .. } => attr.span, } } - - pub fn mark(&self) -> Mark { - self.expansion_data.mark - } } pub struct MacroExpander<'a, 'b:'a> { pub cx: &'a mut ExtCtxt<'b>, - pub single_step: bool, - pub keep_macs: bool, monotonic: bool, // c.f. `cx.monotonic_expander()` } impl<'a, 'b> MacroExpander<'a, 'b> { pub fn new(cx: &'a mut ExtCtxt<'b>, monotonic: bool) -> Self { - MacroExpander { - cx: cx, - monotonic: monotonic, - single_step: false, - keep_macs: false, - } + MacroExpander { cx: cx, monotonic: monotonic } } - fn expand_crate(&mut self, mut krate: ast::Crate) -> ast::Crate { - let err_count = self.cx.parse_sess.span_diagnostic.err_count(); - - let mut krate_item = placeholder(ExpansionKind::Items, ast::DUMMY_NODE_ID) - .make_items().pop().unwrap().unwrap(); - krate_item.node = ast::ItemKind::Mod(krate.module); - let krate_item = Expansion::Items(SmallVector::one(P(krate_item))); - - krate.module = match self.expand(krate_item).make_items().pop().unwrap().unwrap().node { - ast::ItemKind::Mod(module) => module, + pub fn expand_crate(&mut self, mut krate: ast::Crate) -> ast::Crate { + self.cx.crate_root = std_inject::injected_crate_name(&krate); + let mut module = ModuleData { + mod_path: vec![token::str_to_ident(&self.cx.ecfg.crate_name)], + directory: PathBuf::from(self.cx.codemap().span_to_filename(krate.span)), + }; + module.directory.pop(); + self.cx.current_expansion.module = Rc::new(module); + + let krate_item = Expansion::Items(SmallVector::one(P(ast::Item { + attrs: krate.attrs, + span: krate.span, + node: ast::ItemKind::Mod(krate.module), + ident: keywords::Invalid.ident(), + id: ast::DUMMY_NODE_ID, + vis: ast::Visibility::Public, + }))); + + match self.expand(krate_item).make_items().pop().unwrap().unwrap() { + ast::Item { attrs, node: ast::ItemKind::Mod(module), .. } => { + krate.attrs = attrs; + krate.module = module; + }, _ => unreachable!(), }; - krate.exported_macros = mem::replace(&mut self.cx.exported_macros, Vec::new()); - - for def in &mut krate.exported_macros { - def.id = self.cx.resolver.next_node_id() - } - - if self.cx.parse_sess.span_diagnostic.err_count() > err_count { - self.cx.parse_sess.span_diagnostic.abort_if_errors(); - } krate } @@ -221,24 +224,58 @@ impl<'a, 'b> MacroExpander<'a, 'b> { let (expansion, mut invocations) = self.collect_invocations(expansion); invocations.reverse(); - let mut expansions = vec![vec![(0, expansion)]]; - while let Some(invoc) = invocations.pop() { + let mut expansions = Vec::new(); + let mut undetermined_invocations = Vec::new(); + let (mut progress, mut force) = (false, !self.monotonic); + loop { + let invoc = if let Some(invoc) = invocations.pop() { + invoc + } else if undetermined_invocations.is_empty() { + break + } else { + invocations = mem::replace(&mut undetermined_invocations, Vec::new()); + force = !mem::replace(&mut progress, false); + continue + }; + + let scope = + if self.monotonic { invoc.expansion_data.mark } else { orig_expansion_data.mark }; + let resolution = match invoc.kind { + InvocationKind::Bang { ref mac, .. } => { + self.cx.resolver.resolve_macro(scope, &mac.node.path, force) + } + InvocationKind::Attr { ref attr, .. } => { + let ident = ast::Ident::with_empty_ctxt(intern(&*attr.name())); + let path = ast::Path::from_ident(attr.span, ident); + self.cx.resolver.resolve_macro(scope, &path, force) + } + }; + let ext = match resolution { + Ok(ext) => Some(ext), + Err(Determinacy::Determined) => None, + Err(Determinacy::Undetermined) => { + undetermined_invocations.push(invoc); + continue + } + }; + + progress = true; let ExpansionData { depth, mark, .. } = invoc.expansion_data; self.cx.current_expansion = invoc.expansion_data.clone(); - let expansion = match self.cx.resolver.resolve_invoc(&invoc) { + self.cx.current_expansion.mark = scope; + let expansion = match ext { Some(ext) => self.expand_invoc(invoc, ext), None => invoc.expansion_kind.dummy(invoc.span()), }; - self.cx.current_expansion.depth = depth + 1; let (expansion, new_invocations) = self.collect_invocations(expansion); - if expansions.len() == depth { + if expansions.len() < depth { expansions.push(Vec::new()); } - expansions[depth].push((mark.as_u32(), expansion)); - if !self.single_step { + expansions[depth - 1].push((mark.as_u32(), expansion)); + if !self.cx.ecfg.single_step { invocations.extend(new_invocations.into_iter().rev()); } } @@ -248,20 +285,17 @@ impl<'a, 'b> MacroExpander<'a, 'b> { let mut placeholder_expander = PlaceholderExpander::new(self.cx, self.monotonic); while let Some(expansions) = expansions.pop() { for (mark, expansion) in expansions.into_iter().rev() { - let expansion = expansion.fold_with(&mut placeholder_expander); - placeholder_expander.add(mark, expansion); + placeholder_expander.add(ast::NodeId::from_u32(mark), expansion); } } - placeholder_expander.remove(0) + expansion.fold_with(&mut placeholder_expander) } fn collect_invocations(&mut self, expansion: Expansion) -> (Expansion, Vec<Invocation>) { - let crate_config = mem::replace(&mut self.cx.cfg, Vec::new()); let result = { let mut collector = InvocationCollector { cfg: StripUnconfigured { - config: &crate_config, should_test: self.cx.ecfg.should_test, sess: self.cx.parse_sess, features: self.cx.ecfg.features, @@ -272,10 +306,14 @@ impl<'a, 'b> MacroExpander<'a, 'b> { }; (expansion.fold_with(&mut collector), collector.invocations) }; - self.cx.cfg = crate_config; - let mark = self.cx.current_expansion.mark; - self.cx.resolver.visit_expansion(mark, &result.0); + if self.monotonic { + let err_count = self.cx.parse_sess.span_diagnostic.err_count(); + let mark = self.cx.current_expansion.mark; + self.cx.resolver.visit_expansion(mark, &result.0); + self.cx.resolve_err_count += self.cx.parse_sess.span_diagnostic.err_count() - err_count; + } + result } @@ -294,10 +332,11 @@ impl<'a, 'b> MacroExpander<'a, 'b> { }; attr::mark_used(&attr); + let name = intern(&attr.name()); self.cx.bt_push(ExpnInfo { call_site: attr.span, callee: NameAndSpan { - format: MacroAttribute(intern(&attr.name())), + format: MacroAttribute(name), span: Some(attr.span), allow_internal_unstable: false, } @@ -315,13 +354,28 @@ impl<'a, 'b> MacroExpander<'a, 'b> { items.push(item); kind.expect_from_annotatables(items) } - _ => unreachable!(), + SyntaxExtension::AttrProcMacro(ref mac) => { + let attr_toks = TokenStream::from_tts(tts_for_attr(&attr, &self.cx.parse_sess)); + let item_toks = TokenStream::from_tts(tts_for_item(&item, &self.cx.parse_sess)); + + let tok_result = mac.expand(self.cx, attr.span, attr_toks, item_toks); + self.parse_expansion(tok_result, kind, name, attr.span) + } + SyntaxExtension::CustomDerive(_) => { + self.cx.span_err(attr.span, &format!("`{}` is a derive mode", name)); + kind.dummy(attr.span) + } + _ => { + let msg = &format!("macro `{}` may not be used in attributes", name); + self.cx.span_err(attr.span, &msg); + kind.dummy(attr.span) + } } } /// Expand a macro invocation. Returns the result of expansion. fn expand_bang_invoc(&mut self, invoc: Invocation, ext: Rc<SyntaxExtension>) -> Expansion { - let (mark, kind) = (invoc.mark(), invoc.expansion_kind); + let (mark, kind) = (invoc.expansion_data.mark, invoc.expansion_kind); let (attrs, mac, ident, span) = match invoc.kind { InvocationKind::Bang { attrs, mac, ident, span } => (attrs, mac, ident, span), _ => unreachable!(), @@ -331,7 +385,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> { // Detect use of feature-gated or invalid attributes on macro invoations // since they will not be detected after macro expansion. for attr in attrs.iter() { - feature_gate::check_attribute(&attr, &self.cx.parse_sess.span_diagnostic, + feature_gate::check_attribute(&attr, &self.cx.parse_sess, &self.cx.parse_sess.codemap(), &self.cx.ecfg.features.unwrap()); } @@ -381,53 +435,43 @@ impl<'a, 'b> MacroExpander<'a, 'b> { } }); - kind.make_from(expander.expand(self.cx, span, ident, marked_tts)) + kind.make_from(expander.expand(self.cx, span, ident, marked_tts, attrs)) } - MacroRulesTT => { - if ident.name == keywords::Invalid.name() { - self.cx.span_err(path.span, - &format!("macro {}! expects an ident argument", extname)); + MultiDecorator(..) | MultiModifier(..) | SyntaxExtension::AttrProcMacro(..) => { + self.cx.span_err(path.span, + &format!("`{}` can only be used in attributes", extname)); + return kind.dummy(span); + } + + SyntaxExtension::CustomDerive(..) => { + self.cx.span_err(path.span, &format!("`{}` is a derive mode", extname)); + return kind.dummy(span); + } + + SyntaxExtension::ProcMacro(ref expandfun) => { + if ident.name != keywords::Invalid.name() { + let msg = + format!("macro {}! expects no ident argument, given '{}'", extname, ident); + self.cx.span_err(path.span, &msg); return kind.dummy(span); - }; + } self.cx.bt_push(ExpnInfo { call_site: span, callee: NameAndSpan { format: MacroBang(extname), + // FIXME procedural macros do not have proper span info + // yet, when they do, we should use it here. span: None, - // `macro_rules!` doesn't directly allow unstable - // (this is orthogonal to whether the macro it creates allows it) + // FIXME probably want to follow macro_rules macros here. allow_internal_unstable: false, - } + }, }); - let def = ast::MacroDef { - ident: ident, - id: ast::DUMMY_NODE_ID, - span: span, - imported_from: None, - use_locally: true, - body: marked_tts, - export: attr::contains_name(&attrs, "macro_export"), - allow_internal_unstable: attr::contains_name(&attrs, "allow_internal_unstable"), - attrs: attrs, - }; - - self.cx.insert_macro(def.clone()); - - // If keep_macs is true, expands to a MacEager::items instead. - if self.keep_macs { - Some(placeholders::reconstructed_macro_rules(&def, &path)) - } else { - Some(placeholders::macro_scope_placeholder()) - } - } - - MultiDecorator(..) | MultiModifier(..) => { - self.cx.span_err(path.span, - &format!("`{}` can only be used in attributes", extname)); - return kind.dummy(span); + let toks = TokenStream::from_tts(marked_tts); + let tok_result = expandfun.expand(self.cx, span, toks); + Some(self.parse_expansion(tok_result, kind, extname, span)) } }; @@ -445,6 +489,75 @@ impl<'a, 'b> MacroExpander<'a, 'b> { expn_id: Some(self.cx.backtrace()), }) } + + fn parse_expansion(&mut self, toks: TokenStream, kind: ExpansionKind, name: Name, span: Span) + -> Expansion { + let mut parser = self.cx.new_parser_from_tts(&toks.to_tts()); + let expansion = match parser.parse_expansion(kind, false) { + Ok(expansion) => expansion, + Err(mut err) => { + err.emit(); + return kind.dummy(span); + } + }; + parser.ensure_complete_parse(name, kind.name(), span); + // FIXME better span info + expansion.fold_with(&mut ChangeSpan { span: span }) + } +} + +impl<'a> Parser<'a> { + pub fn parse_expansion(&mut self, kind: ExpansionKind, macro_legacy_warnings: bool) + -> PResult<'a, Expansion> { + Ok(match kind { + ExpansionKind::Items => { + let mut items = SmallVector::zero(); + while let Some(item) = self.parse_item()? { + items.push(item); + } + Expansion::Items(items) + } + ExpansionKind::TraitItems => { + let mut items = SmallVector::zero(); + while self.token != token::Eof { + items.push(self.parse_trait_item()?); + } + Expansion::TraitItems(items) + } + ExpansionKind::ImplItems => { + let mut items = SmallVector::zero(); + while self.token != token::Eof { + items.push(self.parse_impl_item()?); + } + Expansion::ImplItems(items) + } + ExpansionKind::Stmts => { + let mut stmts = SmallVector::zero(); + while self.token != token::Eof { + if let Some(stmt) = self.parse_full_stmt(macro_legacy_warnings)? { + stmts.push(stmt); + } + } + Expansion::Stmts(stmts) + } + ExpansionKind::Expr => Expansion::Expr(self.parse_expr()?), + ExpansionKind::OptExpr => Expansion::OptExpr(Some(self.parse_expr()?)), + ExpansionKind::Ty => Expansion::Ty(self.parse_ty()?), + ExpansionKind::Pat => Expansion::Pat(self.parse_pat()?), + }) + } + + pub fn ensure_complete_parse(&mut self, macro_name: ast::Name, kind_name: &str, span: Span) { + if self.token != token::Eof { + let msg = format!("macro expansion ignores token `{}` and any following", + self.this_token_to_string()); + let mut err = self.diagnostic().struct_span_err(self.span, &msg); + let msg = format!("caused by the macro expansion here; the usage \ + of `{}!` is likely invalid in {} context", + macro_name, kind_name); + err.span_note(span, &msg).emit(); + } + } } struct InvocationCollector<'a, 'b: 'a> { @@ -469,9 +582,13 @@ impl<'a, 'b> InvocationCollector<'a, 'b> { self.invocations.push(Invocation { kind: kind, expansion_kind: expansion_kind, - expansion_data: ExpansionData { mark: mark, ..self.cx.current_expansion.clone() }, + expansion_data: ExpansionData { + mark: mark, + depth: self.cx.current_expansion.depth + 1, + ..self.cx.current_expansion.clone() + }, }); - placeholder(expansion_kind, mark.as_u32()) + placeholder(expansion_kind, ast::NodeId::from_u32(mark.as_u32())) } fn collect_bang( @@ -500,6 +617,36 @@ impl<'a, 'b> InvocationCollector<'a, 'b> { } } +// These are pretty nasty. Ideally, we would keep the tokens around, linked from +// the AST. However, we don't so we need to create new ones. Since the item might +// have come from a macro expansion (possibly only in part), we can't use the +// existing codemap. +// +// Therefore, we must use the pretty printer (yuck) to turn the AST node into a +// string, which we then re-tokenise (double yuck), but first we have to patch +// the pretty-printed string on to the end of the existing codemap (infinity-yuck). +fn tts_for_item(item: &Annotatable, parse_sess: &ParseSess) -> Vec<TokenTree> { + let text = match *item { + Annotatable::Item(ref i) => pprust::item_to_string(i), + Annotatable::TraitItem(ref ti) => pprust::trait_item_to_string(ti), + Annotatable::ImplItem(ref ii) => pprust::impl_item_to_string(ii), + }; + string_to_tts(text, parse_sess) +} + +fn tts_for_attr(attr: &ast::Attribute, parse_sess: &ParseSess) -> Vec<TokenTree> { + string_to_tts(pprust::attr_to_string(attr), parse_sess) +} + +fn string_to_tts(text: String, parse_sess: &ParseSess) -> Vec<TokenTree> { + let filemap = parse_sess.codemap() + .new_filemap(String::from("<macro expansion>"), None, text); + + let lexer = lexer::StringReader::new(&parse_sess.span_diagnostic, filemap); + let mut parser = Parser::new(parse_sess, Box::new(lexer)); + panictry!(parser.parse_all_token_trees()) +} + impl<'a, 'b> Folder for InvocationCollector<'a, 'b> { fn fold_expr(&mut self, expr: P<ast::Expr>) -> P<ast::Expr> { let mut expr = self.cfg.configure_expr(expr).unwrap(); @@ -568,16 +715,16 @@ impl<'a, 'b> Folder for InvocationCollector<'a, 'b> { } fn fold_block(&mut self, block: P<Block>) -> P<Block> { - let orig_in_block = mem::replace(&mut self.cx.current_expansion.in_block, true); + let no_noninline_mod = mem::replace(&mut self.cx.current_expansion.no_noninline_mod, true); let result = noop_fold_block(block, self); - self.cx.current_expansion.in_block = orig_in_block; + self.cx.current_expansion.no_noninline_mod = no_noninline_mod; result } fn fold_item(&mut self, item: P<ast::Item>) -> SmallVector<P<ast::Item>> { let item = configure!(self, item); - let (item, attr) = self.classify_item(item); + let (mut item, attr) = self.classify_item(item); if let Some(attr) = attr { let item = Annotatable::Item(fully_configure!(self, item, noop_fold_item)); return self.collect_attr(attr, item, ExpansionKind::Items).make_items(); @@ -609,6 +756,7 @@ impl<'a, 'b> Folder for InvocationCollector<'a, 'b> { return noop_fold_item(item, self); } + let orig_no_noninline_mod = self.cx.current_expansion.no_noninline_mod; let mut module = (*self.cx.current_expansion.module).clone(); module.mod_path.push(item.ident); @@ -618,11 +766,14 @@ impl<'a, 'b> Folder for InvocationCollector<'a, 'b> { let inline_module = item.span.contains(inner) || inner == syntax_pos::DUMMY_SP; if inline_module { - module.directory.push(&*{ - ::attr::first_attr_value_str_by_name(&item.attrs, "path") - .unwrap_or(item.ident.name.as_str()) - }); + if let Some(path) = attr::first_attr_value_str_by_name(&item.attrs, "path") { + self.cx.current_expansion.no_noninline_mod = false; + module.directory.push(&*path); + } else { + module.directory.push(&*item.ident.name.as_str()); + } } else { + self.cx.current_expansion.no_noninline_mod = false; module.directory = PathBuf::from(self.cx.parse_sess.codemap().span_to_filename(inner)); module.directory.pop(); @@ -632,22 +783,16 @@ impl<'a, 'b> Folder for InvocationCollector<'a, 'b> { mem::replace(&mut self.cx.current_expansion.module, Rc::new(module)); let result = noop_fold_item(item, self); self.cx.current_expansion.module = orig_module; + self.cx.current_expansion.no_noninline_mod = orig_no_noninline_mod; return result; } - ast::ItemKind::ExternCrate(..) => { - // We need to error on `#[macro_use] extern crate` when it isn't at the - // crate root, because `$crate` won't work properly. - let is_crate_root = self.cx.current_expansion.module.mod_path.len() == 1; - for def in self.cx.resolver.load_crate(&*item, is_crate_root) { - match def { - LoadedMacro::Def(def) => self.cx.insert_macro(def), - LoadedMacro::CustomDerive(name, ext) => { - self.cx.insert_custom_derive(&name, ext, item.span); - } - } + // Ensure that test functions are accessible from the test harness. + ast::ItemKind::Fn(..) if self.cx.ecfg.should_test => { + if item.attrs.iter().any(|attr| is_test_or_bench(attr)) { + item = item.map(|mut item| { item.vis = ast::Visibility::Public; item }); } noop_fold_item(item, self) - }, + } _ => noop_fold_item(item, self), } } @@ -726,6 +871,8 @@ pub struct ExpansionConfig<'feat> { pub recursion_limit: usize, pub trace_mac: bool, pub should_test: bool, // If false, strip `#[test]` nodes + pub single_step: bool, + pub keep_macs: bool, } macro_rules! feature_tests { @@ -749,6 +896,8 @@ impl<'feat> ExpansionConfig<'feat> { recursion_limit: 64, trace_mac: false, should_test: false, + single_step: false, + keep_macs: false, } } @@ -761,26 +910,10 @@ impl<'feat> ExpansionConfig<'feat> { fn enable_allow_internal_unstable = allow_internal_unstable, fn enable_custom_derive = custom_derive, fn enable_pushpop_unsafe = pushpop_unsafe, - fn enable_rustc_macro = rustc_macro, + fn enable_proc_macro = proc_macro, } } -pub fn expand_crate(cx: &mut ExtCtxt, - user_exts: Vec<NamedSyntaxExtension>, - c: Crate) -> Crate { - cx.initialize(user_exts, &c); - cx.monotonic_expander().expand_crate(c) -} - -// Expands crate using supplied MacroExpander - allows for -// non-standard expansion behaviour (e.g. step-wise). -pub fn expand_crate_with_expander(expander: &mut MacroExpander, - user_exts: Vec<NamedSyntaxExtension>, - c: Crate) -> Crate { - expander.cx.initialize(user_exts, &c); - expander.expand_crate(c) -} - // A Marker adds the given mark to the syntax context and // sets spans' `expn_id` to the given expn_id (unless it is `None`). struct Marker { mark: Mark, expn_id: Option<ExpnId> } @@ -803,6 +936,6 @@ impl Folder for Marker { } // apply a given mark to the given token trees. Used prior to expansion of a macro. -fn mark_tts(tts: &[TokenTree], m: Mark) -> Vec<TokenTree> { +pub fn mark_tts(tts: &[TokenTree], m: Mark) -> Vec<TokenTree> { noop_fold_tts(tts, &mut Marker{mark:m, expn_id: None}) } diff --git a/src/libsyntax/ext/hygiene.rs b/src/libsyntax/ext/hygiene.rs index 34126fac4ac..0fd72277cca 100644 --- a/src/libsyntax/ext/hygiene.rs +++ b/src/libsyntax/ext/hygiene.rs @@ -15,6 +15,7 @@ //! and definition contexts*. J. Funct. Program. 22, 2 (March 2012), 181-216. //! DOI=10.1017/S0956796812000093 http://dx.doi.org/10.1017/S0956796812000093 +use ast::NodeId; use std::cell::RefCell; use std::collections::HashMap; use std::fmt; @@ -46,6 +47,10 @@ impl Mark { Mark(0) } + pub fn from_placeholder_id(id: NodeId) -> Self { + Mark(id.as_u32()) + } + pub fn as_u32(&self) -> u32 { self.0 } diff --git a/src/libsyntax/ext/placeholders.rs b/src/libsyntax/ext/placeholders.rs index 47f366a8876..e323dd2f623 100644 --- a/src/libsyntax/ext/placeholders.rs +++ b/src/libsyntax/ext/placeholders.rs @@ -13,7 +13,7 @@ use codemap::{DUMMY_SP, dummy_spanned}; use ext::base::ExtCtxt; use ext::expand::{Expansion, ExpansionKind}; use fold::*; -use parse::token::keywords; +use parse::token::{intern, keywords}; use ptr::P; use util::move_map::MoveMap; use util::small_vector::SmallVector; @@ -88,10 +88,11 @@ impl<'a, 'b> PlaceholderExpander<'a, 'b> { } pub fn add(&mut self, id: ast::NodeId, expansion: Expansion) { + let expansion = expansion.fold_with(self); self.expansions.insert(id, expansion); } - pub fn remove(&mut self, id: ast::NodeId) -> Expansion { + fn remove(&mut self, id: ast::NodeId) -> Expansion { self.expansions.remove(&id).unwrap() } } @@ -214,7 +215,7 @@ impl<'a, 'b> Folder for PlaceholderExpander<'a, 'b> { } } -pub fn reconstructed_macro_rules(def: &ast::MacroDef, path: &ast::Path) -> Expansion { +pub fn reconstructed_macro_rules(def: &ast::MacroDef) -> Expansion { Expansion::Items(SmallVector::one(P(ast::Item { ident: def.ident, attrs: def.attrs.clone(), @@ -222,7 +223,14 @@ pub fn reconstructed_macro_rules(def: &ast::MacroDef, path: &ast::Path) -> Expan node: ast::ItemKind::Mac(ast::Mac { span: def.span, node: ast::Mac_ { - path: path.clone(), + path: ast::Path { + span: DUMMY_SP, + global: false, + segments: vec![ast::PathSegment { + identifier: ast::Ident::with_empty_ctxt(intern("macro_rules")), + parameters: ast::PathParameters::none(), + }], + }, tts: def.body.clone(), } }), diff --git a/src/libsyntax/ext/proc_macro_shim.rs b/src/libsyntax/ext/proc_macro_shim.rs index fa37e9b54e4..dc3a01f41bc 100644 --- a/src/libsyntax/ext/proc_macro_shim.rs +++ b/src/libsyntax/ext/proc_macro_shim.rs @@ -24,7 +24,9 @@ use ext::base::*; /// Take a `ExtCtxt`, `Span`, and `TokenStream`, and produce a Macro Result that parses /// the TokenStream as a block and returns it as an `Expr`. -pub fn build_block_emitter<'cx>(cx: &'cx mut ExtCtxt, sp: Span, output: TokenStream) +pub fn build_block_emitter<'cx>(cx: &'cx mut ExtCtxt, + sp: Span, + output: TokenStream) -> Box<MacResult + 'cx> { let parser = cx.new_parser_from_tts(&output.to_tts()); @@ -60,7 +62,7 @@ pub fn build_block_emitter<'cx>(cx: &'cx mut ExtCtxt, sp: Span, output: TokenStr } pub mod prelude { - pub use ext::proc_macro_shim::build_block_emitter; + pub use super::build_block_emitter; pub use ast::Ident; pub use codemap::{DUMMY_SP, Span}; pub use ext::base::{ExtCtxt, MacResult}; diff --git a/src/libsyntax/ext/quote.rs b/src/libsyntax/ext/quote.rs index b70e270df54..f21360755bc 100644 --- a/src/libsyntax/ext/quote.rs +++ b/src/libsyntax/ext/quote.rs @@ -46,7 +46,7 @@ pub mod rt { impl ToTokens for TokenTree { fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> { - vec!(self.clone()) + vec![self.clone()] } } @@ -331,7 +331,6 @@ pub mod rt { panictry!(parse::parse_item_from_source_str( "<quote expansion>".to_string(), s, - self.cfg(), self.parse_sess())).expect("parse error") } @@ -339,7 +338,6 @@ pub mod rt { panictry!(parse::parse_stmt_from_source_str( "<quote expansion>".to_string(), s, - self.cfg(), self.parse_sess())).expect("parse error") } @@ -347,7 +345,6 @@ pub mod rt { panictry!(parse::parse_expr_from_source_str( "<quote expansion>".to_string(), s, - self.cfg(), self.parse_sess())) } @@ -355,7 +352,6 @@ pub mod rt { panictry!(parse::parse_tts_from_source_str( "<quote expansion>".to_string(), s, - self.cfg(), self.parse_sess())) } } @@ -420,7 +416,7 @@ pub fn expand_quote_expr<'cx>(cx: &'cx mut ExtCtxt, sp: Span, tts: &[TokenTree]) -> Box<base::MacResult+'cx> { - let expanded = expand_parse_call(cx, sp, "parse_expr_panic", vec!(), tts); + let expanded = expand_parse_call(cx, sp, "parse_expr_panic", vec![], tts); base::MacEager::expr(expanded) } @@ -428,7 +424,7 @@ pub fn expand_quote_item<'cx>(cx: &'cx mut ExtCtxt, sp: Span, tts: &[TokenTree]) -> Box<base::MacResult+'cx> { - let expanded = expand_parse_call(cx, sp, "parse_item_panic", vec!(), tts); + let expanded = expand_parse_call(cx, sp, "parse_item_panic", vec![], tts); base::MacEager::expr(expanded) } @@ -436,7 +432,7 @@ pub fn expand_quote_pat<'cx>(cx: &'cx mut ExtCtxt, sp: Span, tts: &[TokenTree]) -> Box<base::MacResult+'cx> { - let expanded = expand_parse_call(cx, sp, "parse_pat_panic", vec!(), tts); + let expanded = expand_parse_call(cx, sp, "parse_pat_panic", vec![], tts); base::MacEager::expr(expanded) } @@ -444,7 +440,7 @@ pub fn expand_quote_arm(cx: &mut ExtCtxt, sp: Span, tts: &[TokenTree]) -> Box<base::MacResult+'static> { - let expanded = expand_parse_call(cx, sp, "parse_arm_panic", vec!(), tts); + let expanded = expand_parse_call(cx, sp, "parse_arm_panic", vec![], tts); base::MacEager::expr(expanded) } @@ -452,7 +448,7 @@ pub fn expand_quote_ty(cx: &mut ExtCtxt, sp: Span, tts: &[TokenTree]) -> Box<base::MacResult+'static> { - let expanded = expand_parse_call(cx, sp, "parse_ty_panic", vec!(), tts); + let expanded = expand_parse_call(cx, sp, "parse_ty_panic", vec![], tts); base::MacEager::expr(expanded) } @@ -460,7 +456,7 @@ pub fn expand_quote_stmt(cx: &mut ExtCtxt, sp: Span, tts: &[TokenTree]) -> Box<base::MacResult+'static> { - let expanded = expand_parse_call(cx, sp, "parse_stmt_panic", vec!(), tts); + let expanded = expand_parse_call(cx, sp, "parse_stmt_panic", vec![], tts); base::MacEager::expr(expanded) } @@ -469,7 +465,7 @@ pub fn expand_quote_attr(cx: &mut ExtCtxt, tts: &[TokenTree]) -> Box<base::MacResult+'static> { let expanded = expand_parse_call(cx, sp, "parse_attribute_panic", - vec!(cx.expr_bool(sp, true)), tts); + vec![cx.expr_bool(sp, true)], tts); base::MacEager::expr(expanded) } @@ -478,7 +474,7 @@ pub fn expand_quote_arg(cx: &mut ExtCtxt, sp: Span, tts: &[TokenTree]) -> Box<base::MacResult+'static> { - let expanded = expand_parse_call(cx, sp, "parse_arg_panic", vec!(), tts); + let expanded = expand_parse_call(cx, sp, "parse_arg_panic", vec![], tts); base::MacEager::expr(expanded) } @@ -486,7 +482,7 @@ pub fn expand_quote_block(cx: &mut ExtCtxt, sp: Span, tts: &[TokenTree]) -> Box<base::MacResult+'static> { - let expanded = expand_parse_call(cx, sp, "parse_block_panic", vec!(), tts); + let expanded = expand_parse_call(cx, sp, "parse_block_panic", vec![], tts); base::MacEager::expr(expanded) } @@ -494,7 +490,7 @@ pub fn expand_quote_meta_item(cx: &mut ExtCtxt, sp: Span, tts: &[TokenTree]) -> Box<base::MacResult+'static> { - let expanded = expand_parse_call(cx, sp, "parse_meta_item_panic", vec!(), tts); + let expanded = expand_parse_call(cx, sp, "parse_meta_item_panic", vec![], tts); base::MacEager::expr(expanded) } @@ -503,7 +499,7 @@ pub fn expand_quote_path(cx: &mut ExtCtxt, tts: &[TokenTree]) -> Box<base::MacResult+'static> { let mode = mk_parser_path(cx, sp, &["PathStyle", "Type"]); - let expanded = expand_parse_call(cx, sp, "parse_path_panic", vec!(mode), tts); + let expanded = expand_parse_call(cx, sp, "parse_path_panic", vec![mode], tts); base::MacEager::expr(expanded) } @@ -535,7 +531,7 @@ fn mk_ident(cx: &ExtCtxt, sp: Span, ident: ast::Ident) -> P<ast::Expr> { cx.expr_method_call(sp, cx.expr_ident(sp, id_ext("ext_cx")), id_ext("ident_of"), - vec!(e_str)) + vec![e_str]) } // Lift a name to the expr that evaluates to that name @@ -544,16 +540,16 @@ fn mk_name(cx: &ExtCtxt, sp: Span, ident: ast::Ident) -> P<ast::Expr> { cx.expr_method_call(sp, cx.expr_ident(sp, id_ext("ext_cx")), id_ext("name_of"), - vec!(e_str)) + vec![e_str]) } fn mk_tt_path(cx: &ExtCtxt, sp: Span, name: &str) -> P<ast::Expr> { - let idents = vec!(id_ext("syntax"), id_ext("tokenstream"), id_ext("TokenTree"), id_ext(name)); + let idents = vec![id_ext("syntax"), id_ext("tokenstream"), id_ext("TokenTree"), id_ext(name)]; cx.expr_path(cx.path_global(sp, idents)) } fn mk_token_path(cx: &ExtCtxt, sp: Span, name: &str) -> P<ast::Expr> { - let idents = vec!(id_ext("syntax"), id_ext("parse"), id_ext("token"), id_ext(name)); + let idents = vec![id_ext("syntax"), id_ext("parse"), id_ext("token"), id_ext(name)]; cx.expr_path(cx.path_global(sp, idents)) } @@ -603,11 +599,11 @@ fn expr_mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P<ast::Expr> { } match *tok { token::BinOp(binop) => { - return cx.expr_call(sp, mk_token_path(cx, sp, "BinOp"), vec!(mk_binop(cx, sp, binop))); + return cx.expr_call(sp, mk_token_path(cx, sp, "BinOp"), vec![mk_binop(cx, sp, binop)]); } token::BinOpEq(binop) => { return cx.expr_call(sp, mk_token_path(cx, sp, "BinOpEq"), - vec!(mk_binop(cx, sp, binop))); + vec![mk_binop(cx, sp, binop)]); } token::OpenDelim(delim) => { @@ -657,13 +653,13 @@ fn expr_mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P<ast::Expr> { token::Lifetime(ident) => { return cx.expr_call(sp, mk_token_path(cx, sp, "Lifetime"), - vec!(mk_ident(cx, sp, ident))); + vec![mk_ident(cx, sp, ident)]); } token::DocComment(ident) => { return cx.expr_call(sp, mk_token_path(cx, sp, "DocComment"), - vec!(mk_name(cx, sp, ast::Ident::with_empty_ctxt(ident)))); + vec![mk_name(cx, sp, ast::Ident::with_empty_ctxt(ident))]); } token::MatchNt(name, kind) => { @@ -718,7 +714,7 @@ fn statements_mk_tt(cx: &ExtCtxt, tt: &TokenTree, matcher: bool) -> Vec<ast::Stm cx.expr_method_call(sp, cx.expr_ident(sp, ident), id_ext("to_tokens"), - vec!(cx.expr_ident(sp, id_ext("ext_cx")))); + vec![cx.expr_ident(sp, id_ext("ext_cx"))]); let e_to_toks = cx.expr_method_call(sp, e_to_toks, id_ext("into_iter"), vec![]); @@ -726,9 +722,9 @@ fn statements_mk_tt(cx: &ExtCtxt, tt: &TokenTree, matcher: bool) -> Vec<ast::Stm cx.expr_method_call(sp, cx.expr_ident(sp, id_ext("tt")), id_ext("extend"), - vec!(e_to_toks)); + vec![e_to_toks]); - vec!(cx.stmt_expr(e_push)) + vec![cx.stmt_expr(e_push)] } ref tt @ TokenTree::Token(_, MatchNt(..)) if !matcher => { let mut seq = vec![]; @@ -741,13 +737,13 @@ fn statements_mk_tt(cx: &ExtCtxt, tt: &TokenTree, matcher: bool) -> Vec<ast::Stm let e_sp = cx.expr_ident(sp, id_ext("_sp")); let e_tok = cx.expr_call(sp, mk_tt_path(cx, sp, "Token"), - vec!(e_sp, expr_mk_token(cx, sp, tok))); + vec![e_sp, expr_mk_token(cx, sp, tok)]); let e_push = cx.expr_method_call(sp, cx.expr_ident(sp, id_ext("tt")), id_ext("push"), - vec!(e_tok)); - vec!(cx.stmt_expr(e_push)) + vec![e_tok]); + vec![cx.stmt_expr(e_push)] }, TokenTree::Delimited(_, ref delimed) => { statements_mk_tt(cx, &delimed.open_tt(), matcher).into_iter() @@ -800,13 +796,13 @@ fn statements_mk_tt(cx: &ExtCtxt, tt: &TokenTree, matcher: bool) -> Vec<ast::Stm vec![e_seq_struct]); let e_tok = cx.expr_call(sp, mk_tt_path(cx, sp, "Sequence"), - vec!(e_sp, e_rc_new)); + vec![e_sp, e_rc_new]); let e_push = cx.expr_method_call(sp, cx.expr_ident(sp, id_ext("tt")), id_ext("push"), - vec!(e_tok)); - vec!(cx.stmt_expr(e_push)) + vec![e_tok]); + vec![cx.stmt_expr(e_push)] } } } @@ -871,7 +867,7 @@ fn mk_stmts_let(cx: &ExtCtxt, sp: Span) -> Vec<ast::Stmt> { let stmt_let_tt = cx.stmt_let(sp, true, id_ext("tt"), cx.expr_vec_ng(sp)); - vec!(stmt_let_sp, stmt_let_tt) + vec![stmt_let_sp, stmt_let_tt] } fn statements_mk_tts(cx: &ExtCtxt, tts: &[TokenTree], matcher: bool) -> Vec<ast::Stmt> { @@ -920,10 +916,6 @@ fn expand_parse_call(cx: &ExtCtxt, tts: &[TokenTree]) -> P<ast::Expr> { let (cx_expr, tts_expr) = expand_tts(cx, sp, tts); - let cfg_call = || cx.expr_method_call( - sp, cx.expr_ident(sp, id_ext("ext_cx")), - id_ext("cfg"), Vec::new()); - let parse_sess_call = || cx.expr_method_call( sp, cx.expr_ident(sp, id_ext("ext_cx")), id_ext("parse_sess"), Vec::new()); @@ -931,7 +923,7 @@ fn expand_parse_call(cx: &ExtCtxt, let new_parser_call = cx.expr_call(sp, cx.expr_ident(sp, id_ext("new_parser_from_tts")), - vec!(parse_sess_call(), cfg_call(), tts_expr)); + vec![parse_sess_call(), tts_expr]); let path = vec![id_ext("syntax"), id_ext("ext"), id_ext("quote"), id_ext(parse_method)]; let mut args = vec![cx.expr_mut_addr_of(sp, new_parser_call)]; diff --git a/src/libsyntax/ext/source_util.rs b/src/libsyntax/ext/source_util.rs index e75e41d0c2d..ec48cae3f76 100644 --- a/src/libsyntax/ext/source_util.rs +++ b/src/libsyntax/ext/source_util.rs @@ -92,15 +92,8 @@ pub fn expand_include<'cx>(cx: &'cx mut ExtCtxt, sp: Span, tts: &[tokenstream::T None => return DummyResult::expr(sp), }; // The file will be added to the code map by the parser - let p = - parse::new_sub_parser_from_file(cx.parse_sess(), - cx.cfg(), - &res_rel_file(cx, - sp, - Path::new(&file)), - true, - None, - sp); + let path = res_rel_file(cx, sp, Path::new(&file)); + let p = parse::new_sub_parser_from_file(cx.parse_sess(), &path, true, None, sp); struct ExpandResult<'a> { p: parse::parser::Parser<'a>, diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs index b0696a986e3..7e3fe328569 100644 --- a/src/libsyntax/ext/tt/macro_parser.rs +++ b/src/libsyntax/ext/tt/macro_parser.rs @@ -78,7 +78,6 @@ pub use self::NamedMatch::*; pub use self::ParseResult::*; use self::TokenTreeOrTokenTreeVec::*; -use ast; use ast::Ident; use syntax_pos::{self, BytePos, mk_sp, Span}; use codemap::Spanned; @@ -92,6 +91,7 @@ use parse::token; use print::pprust; use ptr::P; use tokenstream::{self, TokenTree}; +use util::small_vector::SmallVector; use std::mem; use std::rc::Rc; @@ -104,7 +104,7 @@ use std::collections::hash_map::Entry::{Vacant, Occupied}; #[derive(Clone)] enum TokenTreeOrTokenTreeVec { Tt(tokenstream::TokenTree), - TtSeq(Rc<Vec<tokenstream::TokenTree>>), + TtSeq(Vec<tokenstream::TokenTree>), } impl TokenTreeOrTokenTreeVec { @@ -161,7 +161,7 @@ pub fn count_names(ms: &[TokenTree]) -> usize { }) } -pub fn initial_matcher_pos(ms: Rc<Vec<TokenTree>>, sep: Option<Token>, lo: BytePos) +pub fn initial_matcher_pos(ms: Vec<TokenTree>, sep: Option<Token>, lo: BytePos) -> Box<MatcherPos> { let match_idx_hi = count_names(&ms[..]); let matches: Vec<_> = (0..match_idx_hi).map(|_| Vec::new()).collect(); @@ -251,14 +251,22 @@ pub fn nameize(p_s: &ParseSess, ms: &[TokenTree], res: &[Rc<NamedMatch>]) pub enum ParseResult<T> { Success(T), - /// Arm failed to match - Failure(syntax_pos::Span, String), + /// Arm failed to match. If the second parameter is `token::Eof`, it + /// indicates an unexpected end of macro invocation. Otherwise, it + /// indicates that no rules expected the given token. + Failure(syntax_pos::Span, Token), /// Fatal error (malformed macro?). Abort compilation. Error(syntax_pos::Span, String) } +pub fn parse_failure_msg(tok: Token) -> String { + match tok { + token::Eof => "unexpected end of macro invocation".to_string(), + _ => format!("no rules expected the token `{}`", pprust::token_to_string(&tok)), + } +} + pub type NamedParseResult = ParseResult<HashMap<Ident, Rc<NamedMatch>>>; -pub type PositionalParseResult = ParseResult<Vec<Rc<NamedMatch>>>; /// Perform a token equality check, ignoring syntax context (that is, an /// unhygienic comparison) @@ -271,17 +279,10 @@ pub fn token_name_eq(t1 : &Token, t2 : &Token) -> bool { } } -pub fn parse(sess: &ParseSess, - cfg: ast::CrateConfig, - mut rdr: TtReader, - ms: &[TokenTree]) - -> NamedParseResult { - let mut cur_eis = Vec::new(); - cur_eis.push(initial_matcher_pos(Rc::new(ms.iter() - .cloned() - .collect()), - None, - rdr.peek().sp.lo)); +pub fn parse(sess: &ParseSess, mut rdr: TtReader, ms: &[TokenTree]) -> NamedParseResult { + let mut cur_eis = SmallVector::one(initial_matcher_pos(ms.to_owned(), + None, + rdr.peek().sp.lo)); loop { let mut bb_eis = Vec::new(); // black-box parsed by parser.rs @@ -425,8 +426,8 @@ pub fn parse(sess: &ParseSess, cur_eis.push(ei); } TokenTree::Token(_, ref t) => { - let mut ei_t = ei.clone(); if token_name_eq(t,&tok) { + let mut ei_t = ei.clone(); ei_t.idx += 1; next_eis.push(ei_t); } @@ -446,7 +447,7 @@ pub fn parse(sess: &ParseSess, } else if eof_eis.len() > 1 { return Error(sp, "ambiguity: multiple successful parses".to_string()); } else { - return Failure(sp, "unexpected end of macro invocation".to_string()); + return Failure(sp, token::Eof); } } else { if (!bb_eis.is_empty() && !next_eis.is_empty()) @@ -467,8 +468,7 @@ pub fn parse(sess: &ParseSess, } )) } else if bb_eis.is_empty() && next_eis.is_empty() { - return Failure(sp, format!("no rules expected the token `{}`", - pprust::token_to_string(&tok))); + return Failure(sp, tok); } else if !next_eis.is_empty() { /* Now process the next token */ while !next_eis.is_empty() { @@ -476,24 +476,21 @@ pub fn parse(sess: &ParseSess, } rdr.next_token(); } else /* bb_eis.len() == 1 */ { - let mut rust_parser = Parser::new(sess, cfg.clone(), Box::new(rdr.clone())); - - let mut ei = bb_eis.pop().unwrap(); - match ei.top_elts.get_tt(ei.idx) { - TokenTree::Token(span, MatchNt(_, ident)) => { + rdr.next_tok = { + let mut rust_parser = Parser::new(sess, Box::new(&mut rdr)); + let mut ei = bb_eis.pop().unwrap(); + if let TokenTree::Token(span, MatchNt(_, ident)) = ei.top_elts.get_tt(ei.idx) { let match_cur = ei.match_cur; (&mut ei.matches[match_cur]).push(Rc::new(MatchedNonterminal( parse_nt(&mut rust_parser, span, &ident.name.as_str())))); ei.idx += 1; ei.match_cur += 1; + } else { + unreachable!() } - _ => panic!() - } - cur_eis.push(ei); - - for _ in 0..rust_parser.tokens_consumed { - let _ = rdr.next_token(); - } + cur_eis.push(ei); + Some(TokenAndSpan { tok: rust_parser.token, sp: rust_parser.span }) + }; } } diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs index 51ef45b97be..431e757368c 100644 --- a/src/libsyntax/ext/tt/macro_rules.rs +++ b/src/libsyntax/ext/tt/macro_rules.rs @@ -8,30 +8,29 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. -use ast; +use {ast, attr}; use syntax_pos::{Span, DUMMY_SP}; -use ext::base::{DummyResult, ExtCtxt, MacResult, SyntaxExtension}; -use ext::base::{NormalTT, TTMacroExpander}; +use ext::base::{DummyResult, ExtCtxt, MacEager, MacResult, SyntaxExtension}; +use ext::base::{IdentMacroExpander, NormalTT, TTMacroExpander}; +use ext::expand::{Expansion, ExpansionKind}; +use ext::placeholders; use ext::tt::macro_parser::{Success, Error, Failure}; use ext::tt::macro_parser::{MatchedSeq, MatchedNonterminal}; -use ext::tt::macro_parser::parse; +use ext::tt::macro_parser::{parse, parse_failure_msg}; +use parse::ParseSess; use parse::lexer::new_tt_reader; use parse::parser::{Parser, Restrictions}; use parse::token::{self, gensym_ident, NtTT, Token}; use parse::token::Token::*; use print; -use ptr::P; use tokenstream::{self, TokenTree}; -use util::small_vector::SmallVector; - -use std::cell::RefCell; use std::collections::{HashMap}; use std::collections::hash_map::{Entry}; use std::rc::Rc; -struct ParserAnyMacro<'a> { - parser: RefCell<Parser<'a>>, +pub struct ParserAnyMacro<'a> { + parser: Parser<'a>, /// Span of the expansion site of the macro this parser is for site_span: Span, @@ -40,115 +39,25 @@ struct ParserAnyMacro<'a> { } impl<'a> ParserAnyMacro<'a> { - /// Make sure we don't have any tokens left to parse, so we don't - /// silently drop anything. `allow_semi` is so that "optional" - /// semicolons at the end of normal expressions aren't complained - /// about e.g. the semicolon in `macro_rules! kapow { () => { - /// panic!(); } }` doesn't get picked up by .parse_expr(), but it's - /// allowed to be there. - fn ensure_complete_parse(&self, allow_semi: bool, context: &str) { - let mut parser = self.parser.borrow_mut(); - if allow_semi && parser.token == token::Semi { + pub fn make(mut self: Box<ParserAnyMacro<'a>>, kind: ExpansionKind) -> Expansion { + let ParserAnyMacro { site_span, macro_ident, ref mut parser } = *self; + let expansion = panictry!(parser.parse_expansion(kind, true)); + + // We allow semicolons at the end of expressions -- e.g. the semicolon in + // `macro_rules! m { () => { panic!(); } }` isn't parsed by `.parse_expr()`, + // but `m!()` is allowed in expression positions (c.f. issue #34706). + if kind == ExpansionKind::Expr && parser.token == token::Semi { parser.bump(); } - if parser.token != token::Eof { - let token_str = parser.this_token_to_string(); - let msg = format!("macro expansion ignores token `{}` and any \ - following", - token_str); - let span = parser.span; - let mut err = parser.diagnostic().struct_span_err(span, &msg[..]); - let msg = format!("caused by the macro expansion here; the usage \ - of `{}!` is likely invalid in {} context", - self.macro_ident, context); - err.span_note(self.site_span, &msg[..]) - .emit(); - } - } -} - -impl<'a> MacResult for ParserAnyMacro<'a> { - fn make_expr(self: Box<ParserAnyMacro<'a>>) -> Option<P<ast::Expr>> { - let ret = panictry!(self.parser.borrow_mut().parse_expr()); - self.ensure_complete_parse(true, "expression"); - Some(ret) - } - fn make_pat(self: Box<ParserAnyMacro<'a>>) -> Option<P<ast::Pat>> { - let ret = panictry!(self.parser.borrow_mut().parse_pat()); - self.ensure_complete_parse(false, "pattern"); - Some(ret) - } - fn make_items(self: Box<ParserAnyMacro<'a>>) -> Option<SmallVector<P<ast::Item>>> { - let mut ret = SmallVector::zero(); - while let Some(item) = panictry!(self.parser.borrow_mut().parse_item()) { - ret.push(item); - } - self.ensure_complete_parse(false, "item"); - Some(ret) - } - - fn make_impl_items(self: Box<ParserAnyMacro<'a>>) - -> Option<SmallVector<ast::ImplItem>> { - let mut ret = SmallVector::zero(); - loop { - let mut parser = self.parser.borrow_mut(); - match parser.token { - token::Eof => break, - _ => ret.push(panictry!(parser.parse_impl_item())) - } - } - self.ensure_complete_parse(false, "item"); - Some(ret) - } - - fn make_trait_items(self: Box<ParserAnyMacro<'a>>) - -> Option<SmallVector<ast::TraitItem>> { - let mut ret = SmallVector::zero(); - loop { - let mut parser = self.parser.borrow_mut(); - match parser.token { - token::Eof => break, - _ => ret.push(panictry!(parser.parse_trait_item())) - } - } - self.ensure_complete_parse(false, "item"); - Some(ret) - } - - - fn make_stmts(self: Box<ParserAnyMacro<'a>>) - -> Option<SmallVector<ast::Stmt>> { - let mut ret = SmallVector::zero(); - loop { - let mut parser = self.parser.borrow_mut(); - match parser.token { - token::Eof => break, - _ => match parser.parse_full_stmt(true) { - Ok(maybe_stmt) => match maybe_stmt { - Some(stmt) => ret.push(stmt), - None => (), - }, - Err(mut e) => { - e.emit(); - break; - } - } - } - } - self.ensure_complete_parse(false, "statement"); - Some(ret) - } - fn make_ty(self: Box<ParserAnyMacro<'a>>) -> Option<P<ast::Ty>> { - let ret = panictry!(self.parser.borrow_mut().parse_ty()); - self.ensure_complete_parse(false, "type"); - Some(ret) + // Make sure we don't have any tokens left to parse so we don't silently drop anything. + parser.ensure_complete_parse(macro_ident.name, kind.name(), site_span); + expansion } } struct MacroRulesMacroExpander { name: ast::Ident, - imported_from: Option<ast::Ident>, lhses: Vec<TokenTree>, rhses: Vec<TokenTree>, valid: bool, @@ -166,7 +75,6 @@ impl TTMacroExpander for MacroRulesMacroExpander { generic_extension(cx, sp, self.name, - self.imported_from, arg, &self.lhses, &self.rhses) @@ -177,7 +85,6 @@ impl TTMacroExpander for MacroRulesMacroExpander { fn generic_extension<'cx>(cx: &'cx ExtCtxt, sp: Span, name: ast::Ident, - imported_from: Option<ast::Ident>, arg: &[TokenTree], lhses: &[TokenTree], rhses: &[TokenTree]) @@ -190,7 +97,7 @@ fn generic_extension<'cx>(cx: &'cx ExtCtxt, // Which arm's failure should we report? (the one furthest along) let mut best_fail_spot = DUMMY_SP; - let mut best_fail_msg = "internal error: ran no matchers".to_string(); + let mut best_fail_tok = None; for (i, lhs) in lhses.iter().enumerate() { // try each arm's matchers let lhs_tt = match *lhs { @@ -206,13 +113,11 @@ fn generic_extension<'cx>(cx: &'cx ExtCtxt, _ => cx.span_bug(sp, "malformed macro rhs"), }; // rhs has holes ( `$id` and `$(...)` that need filled) - let trncbr = new_tt_reader(&cx.parse_sess().span_diagnostic, - Some(named_matches), - imported_from, - rhs); - let mut p = Parser::new(cx.parse_sess(), cx.cfg(), Box::new(trncbr)); + let trncbr = + new_tt_reader(&cx.parse_sess.span_diagnostic, Some(named_matches), rhs); + let mut p = Parser::new(cx.parse_sess(), Box::new(trncbr)); p.directory = cx.current_expansion.module.directory.clone(); - p.restrictions = match cx.current_expansion.in_block { + p.restrictions = match cx.current_expansion.no_noninline_mod { true => Restrictions::NO_NONINLINE_MOD, false => Restrictions::empty(), }; @@ -220,7 +125,7 @@ fn generic_extension<'cx>(cx: &'cx ExtCtxt, // Let the context choose how to interpret the result. // Weird, but useful for X-macros. return Box::new(ParserAnyMacro { - parser: RefCell::new(p), + parser: p, // Pass along the original expansion site and the name of the macro // so we can print a useful error message if the parse of the expanded @@ -229,9 +134,9 @@ fn generic_extension<'cx>(cx: &'cx ExtCtxt, macro_ident: name }) } - Failure(sp, ref msg) => if sp.lo >= best_fail_spot.lo { + Failure(sp, tok) => if sp.lo >= best_fail_spot.lo { best_fail_spot = sp; - best_fail_msg = (*msg).clone(); + best_fail_tok = Some(tok); }, Error(err_sp, ref msg) => { cx.span_fatal(err_sp.substitute_dummy(sp), &msg[..]) @@ -239,7 +144,40 @@ fn generic_extension<'cx>(cx: &'cx ExtCtxt, } } - cx.span_fatal(best_fail_spot.substitute_dummy(sp), &best_fail_msg[..]); + let best_fail_msg = parse_failure_msg(best_fail_tok.expect("ran no matchers")); + cx.span_fatal(best_fail_spot.substitute_dummy(sp), &best_fail_msg); +} + +pub struct MacroRulesExpander; +impl IdentMacroExpander for MacroRulesExpander { + fn expand(&self, + cx: &mut ExtCtxt, + span: Span, + ident: ast::Ident, + tts: Vec<tokenstream::TokenTree>, + attrs: Vec<ast::Attribute>) + -> Box<MacResult> { + let export = attr::contains_name(&attrs, "macro_export"); + let def = ast::MacroDef { + ident: ident, + id: ast::DUMMY_NODE_ID, + span: span, + imported_from: None, + body: tts, + allow_internal_unstable: attr::contains_name(&attrs, "allow_internal_unstable"), + attrs: attrs, + }; + + // If keep_macs is true, expands to a MacEager::items instead. + let result = if cx.ecfg.keep_macs { + MacEager::items(placeholders::reconstructed_macro_rules(&def).make_items()) + } else { + MacEager::items(placeholders::macro_scope_placeholder().make_items()) + }; + + cx.resolver.add_macro(cx.current_expansion.mark, def, export); + result + } } // Note that macro-by-example's input is also matched against a token tree: @@ -248,9 +186,7 @@ fn generic_extension<'cx>(cx: &'cx ExtCtxt, // Holy self-referential! /// Converts a `macro_rules!` invocation into a syntax extension. -pub fn compile<'cx>(cx: &'cx mut ExtCtxt, - def: &ast::MacroDef) -> SyntaxExtension { - +pub fn compile(sess: &ParseSess, def: &ast::MacroDef) -> SyntaxExtension { let lhs_nm = gensym_ident("lhs"); let rhs_nm = gensym_ident("rhs"); @@ -282,19 +218,16 @@ pub fn compile<'cx>(cx: &'cx mut ExtCtxt, ]; // Parse the macro_rules! invocation (`none` is for no interpolations): - let arg_reader = new_tt_reader(&cx.parse_sess().span_diagnostic, - None, - None, - def.body.clone()); - - let argument_map = match parse(cx.parse_sess(), - cx.cfg(), - arg_reader, - &argument_gram) { + let arg_reader = new_tt_reader(&sess.span_diagnostic, None, def.body.clone()); + + let argument_map = match parse(sess, arg_reader, &argument_gram) { Success(m) => m, - Failure(sp, str) | Error(sp, str) => { - panic!(cx.parse_sess().span_diagnostic - .span_fatal(sp.substitute_dummy(def.span), &str[..])); + Failure(sp, tok) => { + let s = parse_failure_msg(tok); + panic!(sess.span_diagnostic.span_fatal(sp.substitute_dummy(def.span), &s)); + } + Error(sp, s) => { + panic!(sess.span_diagnostic.span_fatal(sp.substitute_dummy(def.span), &s)); } }; @@ -305,32 +238,36 @@ pub fn compile<'cx>(cx: &'cx mut ExtCtxt, MatchedSeq(ref s, _) => { s.iter().map(|m| match **m { MatchedNonterminal(NtTT(ref tt)) => { - valid &= check_lhs_nt_follows(cx, tt); + valid &= check_lhs_nt_follows(sess, tt); (**tt).clone() } - _ => cx.span_bug(def.span, "wrong-structured lhs") - }).collect() + _ => sess.span_diagnostic.span_bug(def.span, "wrong-structured lhs") + }).collect::<Vec<TokenTree>>() } - _ => cx.span_bug(def.span, "wrong-structured lhs") + _ => sess.span_diagnostic.span_bug(def.span, "wrong-structured lhs") }; let rhses = match **argument_map.get(&rhs_nm).unwrap() { MatchedSeq(ref s, _) => { s.iter().map(|m| match **m { MatchedNonterminal(NtTT(ref tt)) => (**tt).clone(), - _ => cx.span_bug(def.span, "wrong-structured rhs") + _ => sess.span_diagnostic.span_bug(def.span, "wrong-structured rhs") }).collect() } - _ => cx.span_bug(def.span, "wrong-structured rhs") + _ => sess.span_diagnostic.span_bug(def.span, "wrong-structured rhs") }; for rhs in &rhses { - valid &= check_rhs(cx, rhs); + valid &= check_rhs(sess, rhs); + } + + // don't abort iteration early, so that errors for multiple lhses can be reported + for lhs in &lhses { + valid &= check_lhs_no_empty_seq(sess, &[lhs.clone()]) } let exp: Box<_> = Box::new(MacroRulesMacroExpander { name: def.ident, - imported_from: def.imported_from, lhses: lhses, rhses: rhses, valid: valid, @@ -339,14 +276,14 @@ pub fn compile<'cx>(cx: &'cx mut ExtCtxt, NormalTT(exp, Some(def.span), def.allow_internal_unstable) } -fn check_lhs_nt_follows(cx: &mut ExtCtxt, lhs: &TokenTree) -> bool { +fn check_lhs_nt_follows(sess: &ParseSess, lhs: &TokenTree) -> bool { // lhs is going to be like TokenTree::Delimited(...), where the // entire lhs is those tts. Or, it can be a "bare sequence", not wrapped in parens. match lhs { - &TokenTree::Delimited(_, ref tts) => check_matcher(cx, &tts.tts), + &TokenTree::Delimited(_, ref tts) => check_matcher(sess, &tts.tts), _ => { - cx.span_err(lhs.get_span(), "invalid macro matcher; matchers must \ - be contained in balanced delimiters"); + let msg = "invalid macro matcher; matchers must be contained in balanced delimiters"; + sess.span_diagnostic.span_err(lhs.get_span(), msg); false } } @@ -354,20 +291,52 @@ fn check_lhs_nt_follows(cx: &mut ExtCtxt, lhs: &TokenTree) -> bool { // after parsing/expansion. we can report every error in every macro this way. } -fn check_rhs(cx: &mut ExtCtxt, rhs: &TokenTree) -> bool { +/// Check that the lhs contains no repetition which could match an empty token +/// tree, because then the matcher would hang indefinitely. +fn check_lhs_no_empty_seq(sess: &ParseSess, tts: &[TokenTree]) -> bool { + for tt in tts { + match *tt { + TokenTree::Token(_, _) => (), + TokenTree::Delimited(_, ref del) => if !check_lhs_no_empty_seq(sess, &del.tts) { + return false; + }, + TokenTree::Sequence(span, ref seq) => { + if seq.separator.is_none() { + if seq.tts.iter().all(|seq_tt| { + match *seq_tt { + TokenTree::Sequence(_, ref sub_seq) => + sub_seq.op == tokenstream::KleeneOp::ZeroOrMore, + _ => false, + } + }) { + sess.span_diagnostic.span_err(span, "repetition matches empty token tree"); + return false; + } + } + if !check_lhs_no_empty_seq(sess, &seq.tts) { + return false; + } + } + } + } + + true +} + +fn check_rhs(sess: &ParseSess, rhs: &TokenTree) -> bool { match *rhs { TokenTree::Delimited(..) => return true, - _ => cx.span_err(rhs.get_span(), "macro rhs must be delimited") + _ => sess.span_diagnostic.span_err(rhs.get_span(), "macro rhs must be delimited") } false } -fn check_matcher(cx: &mut ExtCtxt, matcher: &[TokenTree]) -> bool { +fn check_matcher(sess: &ParseSess, matcher: &[TokenTree]) -> bool { let first_sets = FirstSets::new(matcher); let empty_suffix = TokenSet::empty(); - let err = cx.parse_sess.span_diagnostic.err_count(); - check_matcher_core(cx, &first_sets, matcher, &empty_suffix); - err == cx.parse_sess.span_diagnostic.err_count() + let err = sess.span_diagnostic.err_count(); + check_matcher_core(sess, &first_sets, matcher, &empty_suffix); + err == sess.span_diagnostic.err_count() } // The FirstSets for a matcher is a mapping from subsequences in the @@ -605,7 +574,7 @@ impl TokenSet { // // Requires that `first_sets` is pre-computed for `matcher`; // see `FirstSets::new`. -fn check_matcher_core(cx: &mut ExtCtxt, +fn check_matcher_core(sess: &ParseSess, first_sets: &FirstSets, matcher: &[TokenTree], follow: &TokenSet) -> TokenSet { @@ -637,7 +606,8 @@ fn check_matcher_core(cx: &mut ExtCtxt, TokenTree::Token(sp, ref tok) => { let can_be_followed_by_any; if let Err(bad_frag) = has_legal_fragment_specifier(tok) { - cx.struct_span_err(sp, &format!("invalid fragment specifier `{}`", bad_frag)) + let msg = format!("invalid fragment specifier `{}`", bad_frag); + sess.span_diagnostic.struct_span_err(sp, &msg) .help("valid fragment specifiers are `ident`, `block`, \ `stmt`, `expr`, `pat`, `ty`, `path`, `meta`, `tt` \ and `item`") @@ -662,7 +632,7 @@ fn check_matcher_core(cx: &mut ExtCtxt, } TokenTree::Delimited(_, ref d) => { let my_suffix = TokenSet::singleton((d.close_span, Token::CloseDelim(d.delim))); - check_matcher_core(cx, first_sets, &d.tts, &my_suffix); + check_matcher_core(sess, first_sets, &d.tts, &my_suffix); // don't track non NT tokens last.replace_with_irrelevant(); @@ -694,7 +664,7 @@ fn check_matcher_core(cx: &mut ExtCtxt, // At this point, `suffix_first` is built, and // `my_suffix` is some TokenSet that we can use // for checking the interior of `seq_rep`. - let next = check_matcher_core(cx, first_sets, &seq_rep.tts, my_suffix); + let next = check_matcher_core(sess, first_sets, &seq_rep.tts, my_suffix); if next.maybe_empty { last.add_all(&next); } else { @@ -714,9 +684,9 @@ fn check_matcher_core(cx: &mut ExtCtxt, 'each_last: for &(_sp, ref t) in &last.tokens { if let MatchNt(ref name, ref frag_spec) = *t { for &(sp, ref next_token) in &suffix_first.tokens { - match is_in_follow(cx, next_token, &frag_spec.name.as_str()) { + match is_in_follow(next_token, &frag_spec.name.as_str()) { Err((msg, help)) => { - cx.struct_span_err(sp, &msg).help(help).emit(); + sess.span_diagnostic.struct_span_err(sp, &msg).help(help).emit(); // don't bother reporting every source of // conflict for a particular element of `last`. continue 'each_last; @@ -731,7 +701,7 @@ fn check_matcher_core(cx: &mut ExtCtxt, "may be" }; - cx.span_err( + sess.span_diagnostic.span_err( sp, &format!("`${name}:{frag}` {may_be} followed by `{next}`, which \ is not allowed for `{frag}` fragments", @@ -788,7 +758,7 @@ fn frag_can_be_followed_by_any(frag: &str) -> bool { /// break macros that were relying on that binary operator as a /// separator. // when changing this do not forget to update doc/book/macros.md! -fn is_in_follow(_: &ExtCtxt, tok: &Token, frag: &str) -> Result<bool, (String, &'static str)> { +fn is_in_follow(tok: &Token, frag: &str) -> Result<bool, (String, &'static str)> { if let &CloseDelim(_) = tok { // closing a token tree can never be matched by any fragment; // iow, we always require that `(` and `)` match, etc. diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/tt/transcribe.rs index 939425378de..8a6a8e53a3e 100644 --- a/src/libsyntax/ext/tt/transcribe.rs +++ b/src/libsyntax/ext/tt/transcribe.rs @@ -10,14 +10,15 @@ use self::LockstepIterSize::*; use ast::Ident; -use syntax_pos::{Span, DUMMY_SP}; use errors::{Handler, DiagnosticBuilder}; use ext::tt::macro_parser::{NamedMatch, MatchedSeq, MatchedNonterminal}; use parse::token::{DocComment, MatchNt, SubstNt}; -use parse::token::{Token, Interpolated, NtIdent, NtTT, SpecialMacroVar}; +use parse::token::{Token, Interpolated, NtIdent, NtTT}; use parse::token; use parse::lexer::TokenAndSpan; +use syntax_pos::{Span, DUMMY_SP}; use tokenstream::{self, TokenTree}; +use util::small_vector::SmallVector; use std::rc::Rc; use std::ops::Add; @@ -36,18 +37,16 @@ struct TtFrame { pub struct TtReader<'a> { pub sp_diag: &'a Handler, /// the unzipped tree: - stack: Vec<TtFrame>, + stack: SmallVector<TtFrame>, /* for MBE-style macro transcription */ interpolations: HashMap<Ident, Rc<NamedMatch>>, - imported_from: Option<Ident>, - // Some => return imported_from as the next token - crate_name_next: Option<Span>, repeat_idx: Vec<usize>, repeat_len: Vec<usize>, /* cached: */ pub cur_tok: Token, pub cur_span: Span, + pub next_tok: Option<TokenAndSpan>, /// Transform doc comments. Only useful in macro invocations pub desugar_doc_comments: bool, pub fatal_errs: Vec<DiagnosticBuilder<'a>>, @@ -58,10 +57,9 @@ pub struct TtReader<'a> { /// (and should) be None. pub fn new_tt_reader(sp_diag: &Handler, interp: Option<HashMap<Ident, Rc<NamedMatch>>>, - imported_from: Option<Ident>, src: Vec<tokenstream::TokenTree>) -> TtReader { - new_tt_reader_with_doc_flag(sp_diag, interp, imported_from, src, false) + new_tt_reader_with_doc_flag(sp_diag, interp, src, false) } /// The extra `desugar_doc_comments` flag enables reading doc comments @@ -72,13 +70,12 @@ pub fn new_tt_reader(sp_diag: &Handler, /// (and should) be None. pub fn new_tt_reader_with_doc_flag(sp_diag: &Handler, interp: Option<HashMap<Ident, Rc<NamedMatch>>>, - imported_from: Option<Ident>, src: Vec<tokenstream::TokenTree>, desugar_doc_comments: bool) -> TtReader { let mut r = TtReader { sp_diag: sp_diag, - stack: vec!(TtFrame { + stack: SmallVector::one(TtFrame { forest: TokenTree::Sequence(DUMMY_SP, Rc::new(tokenstream::SequenceRepetition { tts: src, // doesn't matter. This merely holds the root unzipping. @@ -92,14 +89,13 @@ pub fn new_tt_reader_with_doc_flag(sp_diag: &Handler, None => HashMap::new(), Some(x) => x, }, - imported_from: imported_from, - crate_name_next: None, repeat_idx: Vec::new(), repeat_len: Vec::new(), desugar_doc_comments: desugar_doc_comments, /* dummy values, never read: */ cur_tok: token::Eof, cur_span: DUMMY_SP, + next_tok: None, fatal_errs: Vec::new(), }; tt_next_token(&mut r); /* get cur_tok and cur_span set up */ @@ -178,20 +174,15 @@ fn lockstep_iter_size(t: &TokenTree, r: &TtReader) -> LockstepIterSize { /// Return the next token from the TtReader. /// EFFECT: advances the reader's token field pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan { + if let Some(tok) = r.next_tok.take() { + return tok; + } // FIXME(pcwalton): Bad copy? let ret_val = TokenAndSpan { tok: r.cur_tok.clone(), sp: r.cur_span.clone(), }; loop { - match r.crate_name_next.take() { - None => (), - Some(sp) => { - r.cur_span = sp; - r.cur_tok = token::Ident(r.imported_from.unwrap()); - return ret_val; - }, - } let should_pop = match r.stack.last() { None => { assert_eq!(ret_val.tok, token::Eof); @@ -341,18 +332,6 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan { sep: None }); } - TokenTree::Token(sp, token::SpecialVarNt(SpecialMacroVar::CrateMacroVar)) => { - r.stack.last_mut().unwrap().idx += 1; - - if r.imported_from.is_some() { - r.cur_span = sp; - r.cur_tok = token::ModSep; - r.crate_name_next = Some(sp); - return ret_val; - } - - // otherwise emit nothing and proceed to the next token - } TokenTree::Token(sp, tok) => { r.cur_span = sp; r.cur_tok = tok; diff --git a/src/libsyntax/feature_gate.rs b/src/libsyntax/feature_gate.rs index 27b97a0ad66..129e4a82338 100644 --- a/src/libsyntax/feature_gate.rs +++ b/src/libsyntax/feature_gate.rs @@ -30,12 +30,13 @@ use ast::{self, NodeId, PatKind}; use attr; use codemap::{CodeMap, Spanned}; use syntax_pos::Span; -use errors::Handler; +use errors::{DiagnosticBuilder, Handler}; use visit::{self, FnKind, Visitor}; use parse::ParseSess; use parse::token::InternedString; use std::ascii::AsciiExt; +use std::env; macro_rules! setter { ($field: ident) => {{ @@ -166,6 +167,9 @@ declare_features! ( // RFC 1238 (active, dropck_parametricity, "1.3.0", Some(28498)), + // Allows using the may_dangle attribute; RFC 1327 + (active, dropck_eyepatch, "1.10.0", Some(34761)), + // Allows the use of custom attributes; RFC 572 (active, custom_attribute, "1.0.0", Some(29642)), @@ -252,9 +256,6 @@ declare_features! ( // a...b and ...b (active, inclusive_range_syntax, "1.7.0", Some(28237)), - // `expr?` - (active, question_mark, "1.9.0", Some(31436)), - // impl specialization (RFC 1210) (active, specialization, "1.7.0", Some(31844)), @@ -291,7 +292,7 @@ declare_features! ( (active, item_like_imports, "1.13.0", Some(35120)), // Macros 1.1 - (active, rustc_macro, "1.13.0", Some(35900)), + (active, proc_macro, "1.13.0", Some(35900)), // Allows untagged unions `union U { ... }` (active, untagged_unions, "1.13.0", Some(32836)), @@ -302,6 +303,12 @@ declare_features! ( // Used to identify the `compiler_builtins` crate // rustc internal (active, compiler_builtins, "1.13.0", None), + + // Allows attributes on lifetime/type formal parameters in generics (RFC 1327) + (active, generic_param_attrs, "1.11.0", Some(34761)), + + // Allows field shorthands (`x` meaning `x: x`) in struct literal expressions. + (active, field_init_shorthand, "1.14.0", Some(37340)), ); declare_features! ( @@ -344,6 +351,8 @@ declare_features! ( (accepted, while_let, "1.0.0", None), // Allows `#[deprecated]` attribute (accepted, deprecated, "1.9.0", Some(29935)), + // `expr?` + (accepted, question_mark, "1.14.0", Some(31436)), ); // (changing above list without updating src/doc/reference.md makes @cmr sad) @@ -365,17 +374,34 @@ pub enum AttributeType { pub enum AttributeGate { /// Is gated by a given feature gate, reason /// and function to check if enabled - Gated(&'static str, &'static str, fn(&Features) -> bool), + Gated(Stability, &'static str, &'static str, fn(&Features) -> bool), /// Ungated attribute, can be used on all release channels Ungated, } +impl AttributeGate { + fn is_deprecated(&self) -> bool { + match *self { + Gated(Stability::Deprecated(_), ..) => true, + _ => false, + } + } +} + +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub enum Stability { + Unstable, + // Argument is tracking issue link. + Deprecated(&'static str), +} + // fn() is not Debug impl ::std::fmt::Debug for AttributeGate { fn fmt(&self, fmt: &mut ::std::fmt::Formatter) -> ::std::fmt::Result { match *self { - Gated(ref name, ref expl, _) => write!(fmt, "Gated({}, {})", name, expl), + Gated(ref stab, ref name, ref expl, _) => + write!(fmt, "Gated({:?}, {}, {})", stab, name, expl), Ungated => write!(fmt, "Ungated") } } @@ -390,6 +416,10 @@ macro_rules! cfg_fn { }} } +pub fn deprecated_attributes() -> Vec<&'static (&'static str, AttributeType, AttributeGate)> { + KNOWN_ATTRIBUTES.iter().filter(|a| a.2.is_deprecated()).collect() +} + // Attributes that have a special meaning to rustc or rustdoc pub const KNOWN_ATTRIBUTES: &'static [(&'static str, AttributeType, AttributeGate)] = &[ // Normal attributes @@ -426,7 +456,8 @@ pub const KNOWN_ATTRIBUTES: &'static [(&'static str, AttributeType, AttributeGat ("macro_escape", Normal, Ungated), // RFC #1445. - ("structural_match", Whitelisted, Gated("structural_match", + ("structural_match", Whitelisted, Gated(Stability::Unstable, + "structural_match", "the semantics of constant patterns is \ not yet settled", cfg_fn!(structural_match))), @@ -434,140 +465,181 @@ pub const KNOWN_ATTRIBUTES: &'static [(&'static str, AttributeType, AttributeGat // Not used any more, but we can't feature gate it ("no_stack_check", Normal, Ungated), - ("plugin", CrateLevel, Gated("plugin", + ("plugin", CrateLevel, Gated(Stability::Unstable, + "plugin", "compiler plugins are experimental \ and possibly buggy", cfg_fn!(plugin))), ("no_std", CrateLevel, Ungated), - ("no_core", CrateLevel, Gated("no_core", + ("no_core", CrateLevel, Gated(Stability::Unstable, + "no_core", "no_core is experimental", cfg_fn!(no_core))), - ("lang", Normal, Gated("lang_items", + ("lang", Normal, Gated(Stability::Unstable, + "lang_items", "language items are subject to change", cfg_fn!(lang_items))), - ("linkage", Whitelisted, Gated("linkage", + ("linkage", Whitelisted, Gated(Stability::Unstable, + "linkage", "the `linkage` attribute is experimental \ and not portable across platforms", cfg_fn!(linkage))), - ("thread_local", Whitelisted, Gated("thread_local", + ("thread_local", Whitelisted, Gated(Stability::Unstable, + "thread_local", "`#[thread_local]` is an experimental feature, and does \ not currently handle destructors. There is no \ corresponding `#[task_local]` mapping to the task \ model", cfg_fn!(thread_local))), - ("rustc_on_unimplemented", Normal, Gated("on_unimplemented", + ("rustc_on_unimplemented", Normal, Gated(Stability::Unstable, + "on_unimplemented", "the `#[rustc_on_unimplemented]` attribute \ is an experimental feature", cfg_fn!(on_unimplemented))), - ("allocator", Whitelisted, Gated("allocator", + ("allocator", Whitelisted, Gated(Stability::Unstable, + "allocator", "the `#[allocator]` attribute is an experimental feature", cfg_fn!(allocator))), - ("needs_allocator", Normal, Gated("needs_allocator", + ("needs_allocator", Normal, Gated(Stability::Unstable, + "needs_allocator", "the `#[needs_allocator]` \ attribute is an experimental \ feature", cfg_fn!(needs_allocator))), - ("panic_runtime", Whitelisted, Gated("panic_runtime", + ("panic_runtime", Whitelisted, Gated(Stability::Unstable, + "panic_runtime", "the `#[panic_runtime]` attribute is \ an experimental feature", cfg_fn!(panic_runtime))), - ("needs_panic_runtime", Whitelisted, Gated("needs_panic_runtime", + ("needs_panic_runtime", Whitelisted, Gated(Stability::Unstable, + "needs_panic_runtime", "the `#[needs_panic_runtime]` \ attribute is an experimental \ feature", cfg_fn!(needs_panic_runtime))), - ("rustc_variance", Normal, Gated("rustc_attrs", + ("rustc_variance", Normal, Gated(Stability::Unstable, + "rustc_attrs", "the `#[rustc_variance]` attribute \ is just used for rustc unit tests \ and will never be stable", cfg_fn!(rustc_attrs))), - ("rustc_error", Whitelisted, Gated("rustc_attrs", + ("rustc_error", Whitelisted, Gated(Stability::Unstable, + "rustc_attrs", "the `#[rustc_error]` attribute \ is just used for rustc unit tests \ and will never be stable", cfg_fn!(rustc_attrs))), - ("rustc_if_this_changed", Whitelisted, Gated("rustc_attrs", + ("rustc_if_this_changed", Whitelisted, Gated(Stability::Unstable, + "rustc_attrs", "the `#[rustc_if_this_changed]` attribute \ is just used for rustc unit tests \ and will never be stable", cfg_fn!(rustc_attrs))), - ("rustc_then_this_would_need", Whitelisted, Gated("rustc_attrs", + ("rustc_then_this_would_need", Whitelisted, Gated(Stability::Unstable, + "rustc_attrs", "the `#[rustc_if_this_changed]` attribute \ is just used for rustc unit tests \ and will never be stable", cfg_fn!(rustc_attrs))), - ("rustc_dirty", Whitelisted, Gated("rustc_attrs", + ("rustc_dirty", Whitelisted, Gated(Stability::Unstable, + "rustc_attrs", "the `#[rustc_dirty]` attribute \ is just used for rustc unit tests \ and will never be stable", cfg_fn!(rustc_attrs))), - ("rustc_clean", Whitelisted, Gated("rustc_attrs", + ("rustc_clean", Whitelisted, Gated(Stability::Unstable, + "rustc_attrs", "the `#[rustc_clean]` attribute \ is just used for rustc unit tests \ and will never be stable", cfg_fn!(rustc_attrs))), - ("rustc_partition_reused", Whitelisted, Gated("rustc_attrs", + ("rustc_metadata_dirty", Whitelisted, Gated(Stability::Unstable, + "rustc_attrs", + "the `#[rustc_metadata_dirty]` attribute \ + is just used for rustc unit tests \ + and will never be stable", + cfg_fn!(rustc_attrs))), + ("rustc_metadata_clean", Whitelisted, Gated(Stability::Unstable, + "rustc_attrs", + "the `#[rustc_metadata_clean]` attribute \ + is just used for rustc unit tests \ + and will never be stable", + cfg_fn!(rustc_attrs))), + ("rustc_partition_reused", Whitelisted, Gated(Stability::Unstable, + "rustc_attrs", "this attribute \ is just used for rustc unit tests \ and will never be stable", cfg_fn!(rustc_attrs))), - ("rustc_partition_translated", Whitelisted, Gated("rustc_attrs", + ("rustc_partition_translated", Whitelisted, Gated(Stability::Unstable, + "rustc_attrs", "this attribute \ is just used for rustc unit tests \ and will never be stable", cfg_fn!(rustc_attrs))), - ("rustc_symbol_name", Whitelisted, Gated("rustc_attrs", + ("rustc_symbol_name", Whitelisted, Gated(Stability::Unstable, + "rustc_attrs", "internal rustc attributes will never be stable", cfg_fn!(rustc_attrs))), - ("rustc_item_path", Whitelisted, Gated("rustc_attrs", + ("rustc_item_path", Whitelisted, Gated(Stability::Unstable, + "rustc_attrs", "internal rustc attributes will never be stable", cfg_fn!(rustc_attrs))), - ("rustc_move_fragments", Normal, Gated("rustc_attrs", + ("rustc_move_fragments", Normal, Gated(Stability::Unstable, + "rustc_attrs", "the `#[rustc_move_fragments]` attribute \ is just used for rustc unit tests \ and will never be stable", cfg_fn!(rustc_attrs))), - ("rustc_mir", Whitelisted, Gated("rustc_attrs", + ("rustc_mir", Whitelisted, Gated(Stability::Unstable, + "rustc_attrs", "the `#[rustc_mir]` attribute \ is just used for rustc unit tests \ and will never be stable", cfg_fn!(rustc_attrs))), - ("rustc_inherit_overflow_checks", Whitelisted, Gated("rustc_attrs", + ("rustc_inherit_overflow_checks", Whitelisted, Gated(Stability::Unstable, + "rustc_attrs", "the `#[rustc_inherit_overflow_checks]` \ attribute is just used to control \ overflow checking behavior of several \ libcore functions that are inlined \ across crates and will never be stable", cfg_fn!(rustc_attrs))), - ("compiler_builtins", Whitelisted, Gated("compiler_builtins", + ("compiler_builtins", Whitelisted, Gated(Stability::Unstable, + "compiler_builtins", "the `#[compiler_builtins]` attribute is used to \ identify the `compiler_builtins` crate which \ contains compiler-rt intrinsics and will never be \ stable", cfg_fn!(compiler_builtins))), - ("allow_internal_unstable", Normal, Gated("allow_internal_unstable", + ("allow_internal_unstable", Normal, Gated(Stability::Unstable, + "allow_internal_unstable", EXPLAIN_ALLOW_INTERNAL_UNSTABLE, cfg_fn!(allow_internal_unstable))), - ("fundamental", Whitelisted, Gated("fundamental", + ("fundamental", Whitelisted, Gated(Stability::Unstable, + "fundamental", "the `#[fundamental]` attribute \ is an experimental feature", cfg_fn!(fundamental))), - ("linked_from", Normal, Gated("linked_from", + ("linked_from", Normal, Gated(Stability::Unstable, + "linked_from", "the `#[linked_from]` attribute \ is an experimental feature", cfg_fn!(linked_from))), - ("rustc_macro_derive", Normal, Gated("rustc_macro", - "the `#[rustc_macro_derive]` attribute \ - is an experimental feature", - cfg_fn!(rustc_macro))), + ("proc_macro_derive", Normal, Gated(Stability::Unstable, + "proc_macro", + "the `#[proc_macro_derive]` attribute \ + is an experimental feature", + cfg_fn!(proc_macro))), - ("rustc_copy_clone_marker", Whitelisted, Gated("rustc_attrs", + ("rustc_copy_clone_marker", Whitelisted, Gated(Stability::Unstable, + "rustc_attrs", "internal implementation detail", cfg_fn!(rustc_attrs))), @@ -577,7 +649,8 @@ pub const KNOWN_ATTRIBUTES: &'static [(&'static str, AttributeType, AttributeGat // FIXME: #14406 these are processed in trans, which happens after the // lint pass ("cold", Whitelisted, Ungated), - ("naked", Whitelisted, Gated("naked_functions", + ("naked", Whitelisted, Gated(Stability::Unstable, + "naked_functions", "the `#[naked]` attribute \ is an experimental feature", cfg_fn!(naked_functions))), @@ -588,26 +661,38 @@ pub const KNOWN_ATTRIBUTES: &'static [(&'static str, AttributeType, AttributeGat ("link_section", Whitelisted, Ungated), ("no_builtins", Whitelisted, Ungated), ("no_mangle", Whitelisted, Ungated), - ("no_debug", Whitelisted, Gated("no_debug", - "the `#[no_debug]` attribute \ - is an experimental feature", - cfg_fn!(no_debug))), - ("omit_gdb_pretty_printer_section", Whitelisted, Gated("omit_gdb_pretty_printer_section", + ("no_debug", Whitelisted, Gated( + Stability::Deprecated("https://github.com/rust-lang/rust/issues/29721"), + "no_debug", + "the `#[no_debug]` attribute is an experimental feature", + cfg_fn!(no_debug))), + ("omit_gdb_pretty_printer_section", Whitelisted, Gated(Stability::Unstable, + "omit_gdb_pretty_printer_section", "the `#[omit_gdb_pretty_printer_section]` \ attribute is just used for the Rust test \ suite", cfg_fn!(omit_gdb_pretty_printer_section))), ("unsafe_destructor_blind_to_params", Normal, - Gated("dropck_parametricity", + Gated(Stability::Unstable, + "dropck_parametricity", "unsafe_destructor_blind_to_params has unstable semantics \ and may be removed in the future", cfg_fn!(dropck_parametricity))), - ("unwind", Whitelisted, Gated("unwind_attributes", "#[unwind] is experimental", + ("may_dangle", + Normal, + Gated(Stability::Unstable, + "dropck_eyepatch", + "may_dangle has unstable semantics and may be removed in the future", + cfg_fn!(dropck_eyepatch))), + ("unwind", Whitelisted, Gated(Stability::Unstable, + "unwind_attributes", + "#[unwind] is experimental", cfg_fn!(unwind_attributes))), // used in resolve - ("prelude_import", Whitelisted, Gated("prelude_import", + ("prelude_import", Whitelisted, Gated(Stability::Unstable, + "prelude_import", "`#[prelude_import]` is for use by rustc only", cfg_fn!(prelude_import))), @@ -619,10 +704,12 @@ pub const KNOWN_ATTRIBUTES: &'static [(&'static str, AttributeType, AttributeGat ("unstable", Whitelisted, Ungated), ("deprecated", Normal, Ungated), - ("rustc_paren_sugar", Normal, Gated("unboxed_closures", + ("rustc_paren_sugar", Normal, Gated(Stability::Unstable, + "unboxed_closures", "unboxed_closures are still evolving", cfg_fn!(unboxed_closures))), - ("rustc_reflect_like", Whitelisted, Gated("reflect", + ("rustc_reflect_like", Whitelisted, Gated(Stability::Unstable, + "reflect", "defining reflective traits is still evolving", cfg_fn!(reflect))), @@ -644,7 +731,7 @@ const GATED_CFGS: &'static [(&'static str, &'static str, fn(&Features) -> bool)] ("target_vendor", "cfg_target_vendor", cfg_fn!(cfg_target_vendor)), ("target_thread_local", "cfg_target_thread_local", cfg_fn!(cfg_target_thread_local)), ("target_has_atomic", "cfg_target_has_atomic", cfg_fn!(cfg_target_has_atomic)), - ("rustc_macro", "rustc_macro", cfg_fn!(rustc_macro)), + ("proc_macro", "proc_macro", cfg_fn!(proc_macro)), ]; #[derive(Debug, Eq, PartialEq)] @@ -669,16 +756,15 @@ impl GatedCfg { pub fn check_and_emit(&self, sess: &ParseSess, features: &Features) { let (cfg, feature, has_feature) = GATED_CFGS[self.index]; if !has_feature(features) && !sess.codemap().span_allows_unstable(self.span) { - let diagnostic = &sess.span_diagnostic; let explain = format!("`cfg({})` is experimental and subject to change", cfg); - emit_feature_err(diagnostic, feature, self.span, GateIssue::Language, &explain); + emit_feature_err(sess, feature, self.span, GateIssue::Language, &explain); } } } struct Context<'a> { features: &'a Features, - span_handler: &'a Handler, + parse_sess: &'a ParseSess, cm: &'a CodeMap, plugin_attributes: &'a [(String, AttributeType)], } @@ -689,7 +775,7 @@ macro_rules! gate_feature_fn { let has_feature: bool = has_feature(&$cx.features); debug!("gate_feature(feature = {:?}, span = {:?}); has? {}", name, span, has_feature); if !has_feature && !cx.cm.span_allows_unstable(span) { - emit_feature_err(cx.span_handler, name, span, GateIssue::Language, explain); + emit_feature_err(cx.parse_sess, name, span, GateIssue::Language, explain); } }} } @@ -706,7 +792,7 @@ impl<'a> Context<'a> { let name = &*attr.name(); for &(n, ty, ref gateage) in KNOWN_ATTRIBUTES { if n == name { - if let &Gated(ref name, ref desc, ref has_feature) = gateage { + if let &Gated(_, ref name, ref desc, ref has_feature) = gateage { gate_feature_fn!(self, has_feature, attr.span, name, desc); } debug!("check_attribute: {:?} is known, {:?}, {:?}", name, ty, gateage); @@ -746,10 +832,10 @@ impl<'a> Context<'a> { } } -pub fn check_attribute(attr: &ast::Attribute, handler: &Handler, +pub fn check_attribute(attr: &ast::Attribute, parse_sess: &ParseSess, cm: &CodeMap, features: &Features) { let cx = Context { - features: features, span_handler: handler, + features: features, parse_sess: parse_sess, cm: cm, plugin_attributes: &[] }; cx.check_attribute(attr, true); @@ -778,8 +864,15 @@ pub enum GateIssue { Library(Option<u32>) } -pub fn emit_feature_err(diag: &Handler, feature: &str, span: Span, issue: GateIssue, +pub fn emit_feature_err(sess: &ParseSess, feature: &str, span: Span, issue: GateIssue, explain: &str) { + feature_err(sess, feature, span, issue, explain).emit(); +} + +pub fn feature_err<'a>(sess: &'a ParseSess, feature: &str, span: Span, issue: GateIssue, + explain: &str) -> DiagnosticBuilder<'a> { + let diag = &sess.span_diagnostic; + let issue = match issue { GateIssue::Language => find_lang_feature_issue(feature), GateIssue::Library(lib) => lib, @@ -792,14 +885,13 @@ pub fn emit_feature_err(diag: &Handler, feature: &str, span: Span, issue: GateIs }; // #23973: do not suggest `#![feature(...)]` if we are in beta/stable - if option_env!("CFG_DISABLE_UNSTABLE_FEATURES").is_some() { - err.emit(); - return; + if sess.unstable_features.is_nightly_build() { + err.help(&format!("add #![feature({})] to the \ + crate attributes to enable", + feature)); } - err.help(&format!("add #![feature({})] to the \ - crate attributes to enable", - feature)); - err.emit(); + + err } const EXPLAIN_BOX_SYNTAX: &'static str = @@ -823,7 +915,12 @@ pub const EXPLAIN_ALLOW_INTERNAL_UNSTABLE: &'static str = "allow_internal_unstable side-steps feature gating and stability checks"; pub const EXPLAIN_CUSTOM_DERIVE: &'static str = - "`#[derive]` for custom traits is not stable enough for use and is subject to change"; + "`#[derive]` for custom traits is not stable enough for use. It is deprecated and will \ + be removed in v1.15"; + +pub const EXPLAIN_DEPR_CUSTOM_DERIVE: &'static str = + "`#[derive]` for custom traits is deprecated and will be removed in v1.15. Prefer using \ + procedural macro custom derive"; pub const EXPLAIN_DERIVE_UNDERSCORE: &'static str = "attributes of the form `#[derive_*]` are reserved for the compiler"; @@ -952,9 +1049,10 @@ impl<'a> Visitor for PostExpansionVisitor<'a> { if attr::contains_name(&i.attrs[..], "simd") { gate_feature_post!(&self, simd, i.span, "SIMD types are experimental and possibly buggy"); - self.context.span_handler.span_warn(i.span, - "the `#[simd]` attribute is deprecated, \ - use `#[repr(simd)]` instead"); + self.context.parse_sess.span_diagnostic.span_warn(i.span, + "the `#[simd]` attribute \ + is deprecated, use \ + `#[repr(simd)]` instead"); } for attr in &i.attrs { if attr.name() == "repr" { @@ -1057,12 +1155,17 @@ impl<'a> Visitor for PostExpansionVisitor<'a> { e.span, "inclusive range syntax is experimental"); } - ast::ExprKind::Try(..) => { - gate_feature_post!(&self, question_mark, e.span, "the `?` operator is not stable"); - } ast::ExprKind::InPlace(..) => { gate_feature_post!(&self, placement_in_syntax, e.span, EXPLAIN_PLACEMENT_IN); } + ast::ExprKind::Struct(_, ref fields, _) => { + for field in fields { + if field.is_shorthand { + gate_feature_post!(&self, field_init_shorthand, field.span, + "struct field shorthands are unstable"); + } + } + } _ => {} } visit::walk_expr(self, e); @@ -1070,14 +1173,14 @@ impl<'a> Visitor for PostExpansionVisitor<'a> { fn visit_pat(&mut self, pattern: &ast::Pat) { match pattern.node { - PatKind::Vec(_, Some(_), ref last) if !last.is_empty() => { + PatKind::Slice(_, Some(_), ref last) if !last.is_empty() => { gate_feature_post!(&self, advanced_slice_patterns, pattern.span, "multiple-element slice matches anywhere \ but at the end of a slice (e.g. \ `[0, ..xs, 0]`) are experimental") } - PatKind::Vec(..) => { + PatKind::Slice(..) => { gate_feature_post!(&self, slice_patterns, pattern.span, "slice pattern syntax is experimental"); @@ -1208,6 +1311,24 @@ impl<'a> Visitor for PostExpansionVisitor<'a> { visit::walk_vis(self, vis) } + + fn visit_generics(&mut self, g: &ast::Generics) { + for t in &g.ty_params { + if !t.attrs.is_empty() { + gate_feature_post!(&self, generic_param_attrs, t.attrs[0].span, + "attributes on type parameter bindings are experimental"); + } + } + visit::walk_generics(self, g) + } + + fn visit_lifetime_def(&mut self, lifetime_def: &ast::LifetimeDef) { + if !lifetime_def.attrs.is_empty() { + gate_feature_post!(&self, generic_param_attrs, lifetime_def.attrs[0].span, + "attributes on lifetime bindings are experimental"); + } + visit::walk_lifetime_def(self, lifetime_def) + } } pub fn get_features(span_handler: &Handler, krate_attrs: &[ast::Attribute]) -> Features { @@ -1263,7 +1384,7 @@ pub fn check_crate(krate: &ast::Crate, maybe_stage_features(&sess.span_diagnostic, krate, unstable); let ctx = Context { features: features, - span_handler: &sess.span_diagnostic, + parse_sess: sess, cm: sess.codemap(), plugin_attributes: plugin_attributes, }; @@ -1275,7 +1396,7 @@ pub enum UnstableFeatures { /// Hard errors for unstable features are active, as on /// beta/stable channels. Disallow, - /// Allow features to me activated, as on nightly. + /// Allow features to be activated, as on nightly. Allow, /// Errors are bypassed for bootstrapping. This is required any time /// during the build that feature-related lints are set to warn or above @@ -1284,6 +1405,27 @@ pub enum UnstableFeatures { Cheat } +impl UnstableFeatures { + pub fn from_environment() -> UnstableFeatures { + // Whether this is a feature-staged build, i.e. on the beta or stable channel + let disable_unstable_features = option_env!("CFG_DISABLE_UNSTABLE_FEATURES").is_some(); + // Whether we should enable unstable features for bootstrapping + let bootstrap = env::var("RUSTC_BOOTSTRAP").is_ok(); + match (disable_unstable_features, bootstrap) { + (_, true) => UnstableFeatures::Cheat, + (true, _) => UnstableFeatures::Disallow, + (false, _) => UnstableFeatures::Allow + } + } + + pub fn is_nightly_build(&self) -> bool { + match *self { + UnstableFeatures::Allow | UnstableFeatures::Cheat => true, + _ => false, + } + } +} + fn maybe_stage_features(span_handler: &Handler, krate: &ast::Crate, unstable: UnstableFeatures) { let allow_features = match unstable { diff --git a/src/libsyntax/fold.rs b/src/libsyntax/fold.rs index 9fb4d0203f4..4bf6e55d674 100644 --- a/src/libsyntax/fold.rs +++ b/src/libsyntax/fold.rs @@ -356,7 +356,7 @@ pub fn noop_fold_ty<T: Folder>(t: P<Ty>, fld: &mut T) -> P<Ty> { id: fld.new_id(id), node: match node { TyKind::Infer | TyKind::ImplicitSelf => node, - TyKind::Vec(ty) => TyKind::Vec(fld.fold_ty(ty)), + TyKind::Slice(ty) => TyKind::Slice(fld.fold_ty(ty)), TyKind::Ptr(mt) => TyKind::Ptr(fld.fold_mt(mt)), TyKind::Rptr(region, mt) => { TyKind::Rptr(fld.fold_opt_lifetime(region), fld.fold_mt(mt)) @@ -385,8 +385,8 @@ pub fn noop_fold_ty<T: Folder>(t: P<Ty>, fld: &mut T) -> P<Ty> { TyKind::ObjectSum(fld.fold_ty(ty), fld.fold_bounds(bounds)) } - TyKind::FixedLengthVec(ty, e) => { - TyKind::FixedLengthVec(fld.fold_ty(ty), fld.fold_expr(e)) + TyKind::Array(ty, e) => { + TyKind::Array(fld.fold_ty(ty), fld.fold_expr(e)) } TyKind::Typeof(expr) => { TyKind::Typeof(fld.fold_expr(expr)) @@ -478,8 +478,8 @@ pub fn noop_fold_parenthesized_parameter_data<T: Folder>(data: ParenthesizedPara pub fn noop_fold_local<T: Folder>(l: P<Local>, fld: &mut T) -> P<Local> { l.map(|Local {id, pat, ty, init, span, attrs}| Local { id: fld.new_id(id), - ty: ty.map(|t| fld.fold_ty(t)), pat: fld.fold_pat(pat), + ty: ty.map(|t| fld.fold_ty(t)), init: init.map(|e| fld.fold_expr(e)), span: fld.new_span(span), attrs: fold_attrs(attrs.into(), fld).into(), @@ -662,8 +662,13 @@ pub fn noop_fold_ty_param_bound<T>(tpb: TyParamBound, fld: &mut T) } pub fn noop_fold_ty_param<T: Folder>(tp: TyParam, fld: &mut T) -> TyParam { - let TyParam {id, ident, bounds, default, span} = tp; + let TyParam {attrs, id, ident, bounds, default, span} = tp; + let attrs: Vec<_> = attrs.into(); TyParam { + attrs: attrs.into_iter() + .flat_map(|x| fld.fold_attribute(x).into_iter()) + .collect::<Vec<_>>() + .into(), id: fld.new_id(id), ident: ident, bounds: fld.fold_bounds(bounds), @@ -687,7 +692,12 @@ pub fn noop_fold_lifetime<T: Folder>(l: Lifetime, fld: &mut T) -> Lifetime { pub fn noop_fold_lifetime_def<T: Folder>(l: LifetimeDef, fld: &mut T) -> LifetimeDef { + let attrs: Vec<_> = l.attrs.into(); LifetimeDef { + attrs: attrs.into_iter() + .flat_map(|x| fld.fold_attribute(x).into_iter()) + .collect::<Vec<_>>() + .into(), lifetime: fld.fold_lifetime(l.lifetime), bounds: fld.fold_lifetimes(l.bounds), } @@ -813,11 +823,12 @@ pub fn noop_fold_struct_field<T: Folder>(f: StructField, fld: &mut T) -> StructF } } -pub fn noop_fold_field<T: Folder>(Field {ident, expr, span}: Field, folder: &mut T) -> Field { +pub fn noop_fold_field<T: Folder>(f: Field, folder: &mut T) -> Field { Field { - ident: respan(ident.span, folder.fold_ident(ident.node)), - expr: folder.fold_expr(expr), - span: folder.new_span(span) + ident: respan(f.ident.span, folder.fold_ident(f.ident.node)), + expr: folder.fold_expr(f.expr), + span: folder.new_span(f.span), + is_shorthand: f.is_shorthand, } } @@ -860,14 +871,10 @@ pub fn noop_fold_item_kind<T: Folder>(i: ItemKind, folder: &mut T) -> ItemKind { ItemKind::Const(folder.fold_ty(t), folder.fold_expr(e)) } ItemKind::Fn(decl, unsafety, constness, abi, generics, body) => { - ItemKind::Fn( - folder.fold_fn_decl(decl), - unsafety, - constness, - abi, - folder.fold_generics(generics), - folder.fold_block(body) - ) + let generics = folder.fold_generics(generics); + let decl = folder.fold_fn_decl(decl); + let body = folder.fold_block(body); + ItemKind::Fn(decl, unsafety, constness, abi, generics, body) } ItemKind::Mod(m) => ItemKind::Mod(folder.fold_mod(m)), ItemKind::ForeignMod(nm) => ItemKind::ForeignMod(folder.fold_foreign_mod(nm)), @@ -875,50 +882,35 @@ pub fn noop_fold_item_kind<T: Folder>(i: ItemKind, folder: &mut T) -> ItemKind { ItemKind::Ty(folder.fold_ty(t), folder.fold_generics(generics)) } ItemKind::Enum(enum_definition, generics) => { - ItemKind::Enum( - ast::EnumDef { - variants: enum_definition.variants.move_map(|x| folder.fold_variant(x)), - }, - folder.fold_generics(generics)) + let generics = folder.fold_generics(generics); + let variants = enum_definition.variants.move_map(|x| folder.fold_variant(x)); + ItemKind::Enum(ast::EnumDef { variants: variants }, generics) } ItemKind::Struct(struct_def, generics) => { - let struct_def = folder.fold_variant_data(struct_def); - ItemKind::Struct(struct_def, folder.fold_generics(generics)) + let generics = folder.fold_generics(generics); + ItemKind::Struct(folder.fold_variant_data(struct_def), generics) } ItemKind::Union(struct_def, generics) => { - let struct_def = folder.fold_variant_data(struct_def); - ItemKind::Union(struct_def, folder.fold_generics(generics)) + let generics = folder.fold_generics(generics); + ItemKind::Union(folder.fold_variant_data(struct_def), generics) } ItemKind::DefaultImpl(unsafety, ref trait_ref) => { ItemKind::DefaultImpl(unsafety, folder.fold_trait_ref((*trait_ref).clone())) } - ItemKind::Impl(unsafety, polarity, generics, ifce, ty, impl_items) => { - let new_impl_items = impl_items.move_flat_map(|item| { - folder.fold_impl_item(item) - }); - let ifce = match ifce { - None => None, - Some(ref trait_ref) => { - Some(folder.fold_trait_ref((*trait_ref).clone())) - } - }; - ItemKind::Impl(unsafety, - polarity, - folder.fold_generics(generics), - ifce, - folder.fold_ty(ty), - new_impl_items) - } - ItemKind::Trait(unsafety, generics, bounds, items) => { - let bounds = folder.fold_bounds(bounds); - let items = items.move_flat_map(|item| { - folder.fold_trait_item(item) - }); - ItemKind::Trait(unsafety, - folder.fold_generics(generics), - bounds, - items) - } + ItemKind::Impl(unsafety, polarity, generics, ifce, ty, impl_items) => ItemKind::Impl( + unsafety, + polarity, + folder.fold_generics(generics), + ifce.map(|trait_ref| folder.fold_trait_ref(trait_ref.clone())), + folder.fold_ty(ty), + impl_items.move_flat_map(|item| folder.fold_impl_item(item)), + ), + ItemKind::Trait(unsafety, generics, bounds, items) => ItemKind::Trait( + unsafety, + folder.fold_generics(generics), + folder.fold_bounds(bounds), + items.move_flat_map(|item| folder.fold_trait_item(item)), + ), ItemKind::Mac(m) => ItemKind::Mac(folder.fold_mac(m)), } } @@ -954,9 +946,9 @@ pub fn noop_fold_impl_item<T: Folder>(i: ImplItem, folder: &mut T) -> SmallVector<ImplItem> { SmallVector::one(ImplItem { id: folder.new_id(i.id), + vis: folder.fold_vis(i.vis), ident: folder.fold_ident(i.ident), attrs: fold_attrs(i.attrs, folder), - vis: folder.fold_vis(i.vis), defaultness: i.defaultness, node: match i.node { ast::ImplItemKind::Const(ty, expr) => { @@ -980,10 +972,8 @@ pub fn noop_fold_mod<T: Folder>(Mod {inner, items}: Mod, folder: &mut T) -> Mod } } -pub fn noop_fold_crate<T: Folder>(Crate {module, attrs, config, mut exported_macros, span}: Crate, +pub fn noop_fold_crate<T: Folder>(Crate {module, attrs, mut exported_macros, span}: Crate, folder: &mut T) -> Crate { - let config = folder.fold_meta_items(config); - let mut items = folder.fold_item(P(ast::Item { ident: keywords::Invalid.ident(), attrs: attrs, @@ -1017,7 +1007,6 @@ pub fn noop_fold_crate<T: Folder>(Crate {module, attrs, config, mut exported_mac Crate { module: module, attrs: attrs, - config: config, exported_macros: exported_macros, span: span, } @@ -1031,15 +1020,12 @@ pub fn noop_fold_item<T: Folder>(i: P<Item>, folder: &mut T) -> SmallVector<P<It // fold one item into exactly one item pub fn noop_fold_item_simple<T: Folder>(Item {id, ident, attrs, node, vis, span}: Item, folder: &mut T) -> Item { - let id = folder.new_id(id); - let node = folder.fold_item_kind(node); - Item { - id: id, + id: folder.new_id(id), + vis: folder.fold_vis(vis), ident: folder.fold_ident(ident), attrs: fold_attrs(attrs, folder), - node: node, - vis: folder.fold_vis(vis), + node: folder.fold_item_kind(node), span: folder.new_span(span) } } @@ -1047,6 +1033,7 @@ pub fn noop_fold_item_simple<T: Folder>(Item {id, ident, attrs, node, vis, span} pub fn noop_fold_foreign_item<T: Folder>(ni: ForeignItem, folder: &mut T) -> ForeignItem { ForeignItem { id: folder.new_id(ni.id), + vis: folder.fold_vis(ni.vis), ident: folder.fold_ident(ni.ident), attrs: fold_attrs(ni.attrs, folder), node: match ni.node { @@ -1057,7 +1044,6 @@ pub fn noop_fold_foreign_item<T: Folder>(ni: ForeignItem, folder: &mut T) -> For ForeignItemKind::Static(folder.fold_ty(t), m) } }, - vis: folder.fold_vis(ni.vis), span: folder.new_span(ni.span) } } @@ -1114,8 +1100,8 @@ pub fn noop_fold_pat<T: Folder>(p: P<Pat>, folder: &mut T) -> P<Pat> { PatKind::Range(e1, e2) => { PatKind::Range(folder.fold_expr(e1), folder.fold_expr(e2)) }, - PatKind::Vec(before, slice, after) => { - PatKind::Vec(before.move_map(|x| folder.fold_pat(x)), + PatKind::Slice(before, slice, after) => { + PatKind::Slice(before.move_map(|x| folder.fold_pat(x)), slice.map(|x| folder.fold_pat(x)), after.move_map(|x| folder.fold_pat(x))) } diff --git a/src/libsyntax/json.rs b/src/libsyntax/json.rs index a40c30b3e33..a1c273baeea 100644 --- a/src/libsyntax/json.rs +++ b/src/libsyntax/json.rs @@ -38,14 +38,24 @@ pub struct JsonEmitter { } impl JsonEmitter { + pub fn stderr(registry: Option<Registry>, + code_map: Rc<CodeMap>) -> JsonEmitter { + JsonEmitter { + dst: Box::new(io::stderr()), + registry: registry, + cm: code_map, + } + } + pub fn basic() -> JsonEmitter { JsonEmitter::stderr(None, Rc::new(CodeMap::new())) } - pub fn stderr(registry: Option<Registry>, - code_map: Rc<CodeMap>) -> JsonEmitter { + pub fn new(dst: Box<Write + Send>, + registry: Option<Registry>, + code_map: Rc<CodeMap>) -> JsonEmitter { JsonEmitter { - dst: Box::new(io::stderr()), + dst: dst, registry: registry, cm: code_map, } diff --git a/src/libsyntax/lib.rs b/src/libsyntax/lib.rs index 4a2c9aff2d2..b86e508d899 100644 --- a/src/libsyntax/lib.rs +++ b/src/libsyntax/lib.rs @@ -31,9 +31,12 @@ #![feature(staged_api)] #![feature(str_escape)] #![feature(unicode)] -#![feature(question_mark)] +#![cfg_attr(stage0, feature(question_mark))] #![feature(rustc_diagnostic_macros)] +#![feature(specialization)] +#![feature(dotdot_in_tuple_patterns)] +extern crate core; extern crate serialize; extern crate term; extern crate libc; diff --git a/src/libsyntax/parse/attr.rs b/src/libsyntax/parse/attr.rs index a0defbc09dc..3cb34fa3c91 100644 --- a/src/libsyntax/parse/attr.rs +++ b/src/libsyntax/parse/attr.rs @@ -125,8 +125,8 @@ impl<'a> Parser<'a> { self.expect(&token::OpenDelim(token::Bracket))?; let meta_item = self.parse_meta_item()?; - let hi = self.last_span.hi; self.expect(&token::CloseDelim(token::Bracket))?; + let hi = self.prev_span.hi; (mk_sp(lo, hi), meta_item, style) } @@ -231,16 +231,16 @@ impl<'a> Parser<'a> { token::Eq => { self.bump(); let lit = self.parse_unsuffixed_lit()?; - let hi = self.last_span.hi; + let hi = self.prev_span.hi; Ok(P(spanned(lo, hi, ast::MetaItemKind::NameValue(name, lit)))) } token::OpenDelim(token::Paren) => { let inner_items = self.parse_meta_seq()?; - let hi = self.last_span.hi; + let hi = self.prev_span.hi; Ok(P(spanned(lo, hi, ast::MetaItemKind::List(name, inner_items)))) } _ => { - let hi = self.last_span.hi; + let hi = self.prev_span.hi; Ok(P(spanned(lo, hi, ast::MetaItemKind::Word(name)))) } } @@ -253,14 +253,14 @@ impl<'a> Parser<'a> { match self.parse_unsuffixed_lit() { Ok(lit) => { - return Ok(spanned(lo, self.last_span.hi, ast::NestedMetaItemKind::Literal(lit))) + return Ok(spanned(lo, self.prev_span.hi, ast::NestedMetaItemKind::Literal(lit))) } Err(ref mut err) => self.diagnostic().cancel(err) } match self.parse_meta_item() { Ok(mi) => { - return Ok(spanned(lo, self.last_span.hi, ast::NestedMetaItemKind::MetaItem(mi))) + return Ok(spanned(lo, self.prev_span.hi, ast::NestedMetaItemKind::MetaItem(mi))) } Err(ref mut err) => self.diagnostic().cancel(err) } diff --git a/src/libsyntax/parse/lexer/comments.rs b/src/libsyntax/parse/lexer/comments.rs index 5eb5605ea71..ba83a55ea79 100644 --- a/src/libsyntax/parse/lexer/comments.rs +++ b/src/libsyntax/parse/lexer/comments.rs @@ -24,7 +24,7 @@ use str::char_at; use std::io::Read; use std::usize; -#[derive(Clone, Copy, PartialEq)] +#[derive(Clone, Copy, PartialEq, Debug)] pub enum CommentStyle { /// No code on either side of each line of the comment Isolated, @@ -149,25 +149,24 @@ fn push_blank_line_comment(rdr: &StringReader, comments: &mut Vec<Comment>) { comments.push(Comment { style: BlankLine, lines: Vec::new(), - pos: rdr.last_pos, + pos: rdr.pos, }); } fn consume_whitespace_counting_blank_lines(rdr: &mut StringReader, comments: &mut Vec<Comment>) { - while is_pattern_whitespace(rdr.curr) && !rdr.is_eof() { - if rdr.col == CharPos(0) && rdr.curr_is('\n') { + while is_pattern_whitespace(rdr.ch) && !rdr.is_eof() { + if rdr.ch_is('\n') { push_blank_line_comment(rdr, &mut *comments); } rdr.bump(); } } - fn read_shebang_comment(rdr: &mut StringReader, code_to_the_left: bool, comments: &mut Vec<Comment>) { debug!(">>> shebang comment"); - let p = rdr.last_pos; + let p = rdr.pos; debug!("<<< shebang comment"); comments.push(Comment { style: if code_to_the_left { Trailing } else { Isolated }, @@ -180,9 +179,9 @@ fn read_line_comments(rdr: &mut StringReader, code_to_the_left: bool, comments: &mut Vec<Comment>) { debug!(">>> line comments"); - let p = rdr.last_pos; + let p = rdr.pos; let mut lines: Vec<String> = Vec::new(); - while rdr.curr_is('/') && rdr.nextch_is('/') { + while rdr.ch_is('/') && rdr.nextch_is('/') { let line = rdr.read_one_line_comment(); debug!("{}", line); // Doc comments are not put in comments. @@ -240,7 +239,7 @@ fn read_block_comment(rdr: &mut StringReader, code_to_the_left: bool, comments: &mut Vec<Comment>) { debug!(">>> block comment"); - let p = rdr.last_pos; + let p = rdr.pos; let mut lines: Vec<String> = Vec::new(); let col = rdr.col; rdr.bump(); @@ -249,9 +248,9 @@ fn read_block_comment(rdr: &mut StringReader, let mut curr_line = String::from("/*"); // doc-comments are not really comments, they are attributes - if (rdr.curr_is('*') && !rdr.nextch_is('*')) || rdr.curr_is('!') { - while !(rdr.curr_is('*') && rdr.nextch_is('/')) && !rdr.is_eof() { - curr_line.push(rdr.curr.unwrap()); + if (rdr.ch_is('*') && !rdr.nextch_is('*')) || rdr.ch_is('!') { + while !(rdr.ch_is('*') && rdr.nextch_is('/')) && !rdr.is_eof() { + curr_line.push(rdr.ch.unwrap()); rdr.bump(); } if !rdr.is_eof() { @@ -271,19 +270,19 @@ fn read_block_comment(rdr: &mut StringReader, if rdr.is_eof() { panic!(rdr.fatal("unterminated block comment")); } - if rdr.curr_is('\n') { + if rdr.ch_is('\n') { trim_whitespace_prefix_and_push_line(&mut lines, curr_line, col); curr_line = String::new(); rdr.bump(); } else { - curr_line.push(rdr.curr.unwrap()); - if rdr.curr_is('/') && rdr.nextch_is('*') { + curr_line.push(rdr.ch.unwrap()); + if rdr.ch_is('/') && rdr.nextch_is('*') { rdr.bump(); rdr.bump(); curr_line.push('*'); level += 1; } else { - if rdr.curr_is('*') && rdr.nextch_is('/') { + if rdr.ch_is('*') && rdr.nextch_is('/') { rdr.bump(); rdr.bump(); curr_line.push('/'); @@ -305,7 +304,7 @@ fn read_block_comment(rdr: &mut StringReader, Isolated }; rdr.consume_non_eol_whitespace(); - if !rdr.is_eof() && !rdr.curr_is('\n') && lines.len() == 1 { + if !rdr.is_eof() && !rdr.ch_is('\n') && lines.len() == 1 { style = Mixed; } debug!("<<< block comment"); @@ -317,14 +316,22 @@ fn read_block_comment(rdr: &mut StringReader, } -fn consume_comment(rdr: &mut StringReader, code_to_the_left: bool, comments: &mut Vec<Comment>) { +fn consume_comment(rdr: &mut StringReader, + comments: &mut Vec<Comment>, + code_to_the_left: &mut bool, + anything_to_the_left: &mut bool) { debug!(">>> consume comment"); - if rdr.curr_is('/') && rdr.nextch_is('/') { - read_line_comments(rdr, code_to_the_left, comments); - } else if rdr.curr_is('/') && rdr.nextch_is('*') { - read_block_comment(rdr, code_to_the_left, comments); - } else if rdr.curr_is('#') && rdr.nextch_is('!') { - read_shebang_comment(rdr, code_to_the_left, comments); + if rdr.ch_is('/') && rdr.nextch_is('/') { + read_line_comments(rdr, *code_to_the_left, comments); + *code_to_the_left = false; + *anything_to_the_left = false; + } else if rdr.ch_is('/') && rdr.nextch_is('*') { + read_block_comment(rdr, *code_to_the_left, comments); + *anything_to_the_left = true; + } else if rdr.ch_is('#') && rdr.nextch_is('!') { + read_shebang_comment(rdr, *code_to_the_left, comments); + *code_to_the_left = false; + *anything_to_the_left = false; } else { panic!(); } @@ -352,24 +359,30 @@ pub fn gather_comments_and_literals(span_diagnostic: &errors::Handler, let mut comments: Vec<Comment> = Vec::new(); let mut literals: Vec<Literal> = Vec::new(); - let mut first_read: bool = true; + let mut code_to_the_left = false; // Only code + let mut anything_to_the_left = false; // Code or comments while !rdr.is_eof() { loop { - let mut code_to_the_left = !first_read; + // Eat all the whitespace and count blank lines. rdr.consume_non_eol_whitespace(); - if rdr.curr_is('\n') { - code_to_the_left = false; + if rdr.ch_is('\n') { + if anything_to_the_left { + rdr.bump(); // The line is not blank, do not count. + } consume_whitespace_counting_blank_lines(&mut rdr, &mut comments); + code_to_the_left = false; + anything_to_the_left = false; } - while rdr.peeking_at_comment() { - consume_comment(&mut rdr, code_to_the_left, &mut comments); - consume_whitespace_counting_blank_lines(&mut rdr, &mut comments); + // Eat one comment group + if rdr.peeking_at_comment() { + consume_comment(&mut rdr, &mut comments, + &mut code_to_the_left, &mut anything_to_the_left); + } else { + break } - break; } - - let bstart = rdr.last_pos; + let bstart = rdr.pos; rdr.next_token(); // discard, and look ahead; we're working with internal state let TokenAndSpan { tok, sp } = rdr.peek(); @@ -384,7 +397,8 @@ pub fn gather_comments_and_literals(span_diagnostic: &errors::Handler, } else { debug!("tok: {}", pprust::token_to_string(&tok)); } - first_read = false; + code_to_the_left = true; + anything_to_the_left = true; } (comments, literals) diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs index 9e9ea096460..5e20f6e4192 100644 --- a/src/libsyntax/parse/lexer/mod.rs +++ b/src/libsyntax/parse/lexer/mod.rs @@ -74,17 +74,29 @@ pub struct TokenAndSpan { pub sp: Span, } +impl Default for TokenAndSpan { + fn default() -> Self { + TokenAndSpan { tok: token::Underscore, sp: syntax_pos::DUMMY_SP } + } +} + pub struct StringReader<'a> { pub span_diagnostic: &'a Handler, /// The absolute offset within the codemap of the next character to read + pub next_pos: BytePos, + /// The absolute offset within the codemap of the current character pub pos: BytePos, - /// The absolute offset within the codemap of the last character read(curr) - pub last_pos: BytePos, /// The column of the next character to read pub col: CharPos, - /// The last character to be read - pub curr: Option<char>, + /// The current character (which has been read from self.pos) + pub ch: Option<char>, pub filemap: Rc<syntax_pos::FileMap>, + /// If Some, stop reading the source at this position (inclusive). + pub terminator: Option<BytePos>, + /// Whether to record new-lines in filemap. This is only necessary the first + /// time a filemap is lexed. If part of a filemap is being re-lexed, this + /// should be set to false. + pub save_new_lines: bool, // cached: pub peek_tok: token::Token, pub peek_span: Span, @@ -96,7 +108,14 @@ pub struct StringReader<'a> { impl<'a> Reader for StringReader<'a> { fn is_eof(&self) -> bool { - self.curr.is_none() + if self.ch.is_none() { + return true; + } + + match self.terminator { + Some(t) => self.next_pos > t, + None => false, + } } /// Return the next token. EFFECT: advances the string_reader. fn try_next_token(&mut self) -> Result<TokenAndSpan, ()> { @@ -131,7 +150,7 @@ impl<'a> Reader for StringReader<'a> { impl<'a> Reader for TtReader<'a> { fn is_eof(&self) -> bool { - self.cur_tok == token::Eof + self.peek().tok == token::Eof } fn try_next_token(&mut self) -> Result<TokenAndSpan, ()> { assert!(self.fatal_errs.is_empty()); @@ -152,18 +171,47 @@ impl<'a> Reader for TtReader<'a> { self.fatal_errs.clear(); } fn peek(&self) -> TokenAndSpan { - TokenAndSpan { + self.next_tok.clone().unwrap_or(TokenAndSpan { tok: self.cur_tok.clone(), sp: self.cur_span, - } + }) + } +} + +impl<'a, 'b> Reader for &'b mut TtReader<'a> { + fn is_eof(&self) -> bool { + (**self).is_eof() + } + fn try_next_token(&mut self) -> Result<TokenAndSpan, ()> { + (**self).try_next_token() + } + fn fatal(&self, m: &str) -> FatalError { + (**self).fatal(m) + } + fn err(&self, m: &str) { + (**self).err(m) + } + fn emit_fatal_errors(&mut self) { + (**self).emit_fatal_errors() + } + fn peek(&self) -> TokenAndSpan { + (**self).peek() } } impl<'a> StringReader<'a> { - /// For comments.rs, which hackily pokes into pos and curr + /// For comments.rs, which hackily pokes into next_pos and ch pub fn new_raw<'b>(span_diagnostic: &'b Handler, filemap: Rc<syntax_pos::FileMap>) -> StringReader<'b> { + let mut sr = StringReader::new_raw_internal(span_diagnostic, filemap); + sr.bump(); + sr + } + + fn new_raw_internal<'b>(span_diagnostic: &'b Handler, + filemap: Rc<syntax_pos::FileMap>) + -> StringReader<'b> { if filemap.src.is_none() { span_diagnostic.bug(&format!("Cannot lex filemap \ without source: {}", @@ -172,21 +220,21 @@ impl<'a> StringReader<'a> { let source_text = (*filemap.src.as_ref().unwrap()).clone(); - let mut sr = StringReader { + StringReader { span_diagnostic: span_diagnostic, + next_pos: filemap.start_pos, pos: filemap.start_pos, - last_pos: filemap.start_pos, col: CharPos(0), - curr: Some('\n'), + ch: Some('\n'), filemap: filemap, + terminator: None, + save_new_lines: true, // dummy values; not read peek_tok: token::Eof, peek_span: syntax_pos::DUMMY_SP, source_text: source_text, fatal_errs: Vec::new(), - }; - sr.bump(); - sr + } } pub fn new<'b>(span_diagnostic: &'b Handler, @@ -200,8 +248,8 @@ impl<'a> StringReader<'a> { sr } - pub fn curr_is(&self, c: char) -> bool { - self.curr == Some(c) + pub fn ch_is(&self, c: char) -> bool { + self.ch == Some(c) } /// Report a fatal lexical error with a given span. @@ -296,9 +344,9 @@ impl<'a> StringReader<'a> { self.peek_tok = token::Eof; self.peek_span = syntax_pos::mk_sp(self.filemap.end_pos, self.filemap.end_pos); } else { - let start_bytepos = self.last_pos; + let start_bytepos = self.pos; self.peek_tok = self.next_token_inner()?; - self.peek_span = syntax_pos::mk_sp(start_bytepos, self.last_pos); + self.peek_span = syntax_pos::mk_sp(start_bytepos, self.pos); }; } } @@ -310,19 +358,19 @@ impl<'a> StringReader<'a> { } /// Calls `f` with a string slice of the source text spanning from `start` - /// up to but excluding `self.last_pos`, meaning the slice does not include - /// the character `self.curr`. + /// up to but excluding `self.pos`, meaning the slice does not include + /// the character `self.ch`. pub fn with_str_from<T, F>(&self, start: BytePos, f: F) -> T where F: FnOnce(&str) -> T { - self.with_str_from_to(start, self.last_pos, f) + self.with_str_from_to(start, self.pos, f) } /// Create a Name from a given offset to the current offset, each /// adjusted 1 towards each other (assumes that on either side there is a /// single-byte delimiter). pub fn name_from(&self, start: BytePos) -> ast::Name { - debug!("taking an ident from {:?} to {:?}", start, self.last_pos); + debug!("taking an ident from {:?} to {:?}", start, self.pos); self.with_str_from(start, token::intern) } @@ -393,32 +441,35 @@ impl<'a> StringReader<'a> { /// Advance the StringReader by one character. If a newline is /// discovered, add it to the FileMap's list of line start offsets. pub fn bump(&mut self) { - self.last_pos = self.pos; - let current_byte_offset = self.byte_offset(self.pos).to_usize(); - if current_byte_offset < self.source_text.len() { - assert!(self.curr.is_some()); - let last_char = self.curr.unwrap(); - let ch = char_at(&self.source_text, current_byte_offset); - let next = current_byte_offset + ch.len_utf8(); - let byte_offset_diff = next - current_byte_offset; - self.pos = self.pos + Pos::from_usize(byte_offset_diff); - self.curr = Some(ch); - self.col = self.col + CharPos(1); - if last_char == '\n' { - self.filemap.next_line(self.last_pos); + let new_pos = self.next_pos; + let new_byte_offset = self.byte_offset(new_pos).to_usize(); + if new_byte_offset < self.source_text.len() { + let old_ch_is_newline = self.ch.unwrap() == '\n'; + let new_ch = char_at(&self.source_text, new_byte_offset); + let new_ch_len = new_ch.len_utf8(); + + self.ch = Some(new_ch); + self.pos = new_pos; + self.next_pos = new_pos + Pos::from_usize(new_ch_len); + if old_ch_is_newline { + if self.save_new_lines { + self.filemap.next_line(self.pos); + } self.col = CharPos(0); + } else { + self.col = self.col + CharPos(1); } - - if byte_offset_diff > 1 { - self.filemap.record_multibyte_char(self.last_pos, byte_offset_diff); + if new_ch_len > 1 { + self.filemap.record_multibyte_char(self.pos, new_ch_len); } } else { - self.curr = None; + self.ch = None; + self.pos = new_pos; } } pub fn nextch(&self) -> Option<char> { - let offset = self.byte_offset(self.pos).to_usize(); + let offset = self.byte_offset(self.next_pos).to_usize(); if offset < self.source_text.len() { Some(char_at(&self.source_text, offset)) } else { @@ -431,7 +482,7 @@ impl<'a> StringReader<'a> { } pub fn nextnextch(&self) -> Option<char> { - let offset = self.byte_offset(self.pos).to_usize(); + let offset = self.byte_offset(self.next_pos).to_usize(); let s = &self.source_text[..]; if offset >= s.len() { return None; @@ -450,11 +501,11 @@ impl<'a> StringReader<'a> { /// Eats <XID_start><XID_continue>*, if possible. fn scan_optional_raw_name(&mut self) -> Option<ast::Name> { - if !ident_start(self.curr) { + if !ident_start(self.ch) { return None; } - let start = self.last_pos; - while ident_continue(self.curr) { + let start = self.pos; + while ident_continue(self.ch) { self.bump(); } @@ -467,41 +518,37 @@ impl<'a> StringReader<'a> { }) } - /// PRECONDITION: self.curr is not whitespace + /// PRECONDITION: self.ch is not whitespace /// Eats any kind of comment. fn scan_comment(&mut self) -> Option<TokenAndSpan> { - if let Some(c) = self.curr { + if let Some(c) = self.ch { if c.is_whitespace() { - self.span_diagnostic.span_err(syntax_pos::mk_sp(self.last_pos, self.last_pos), + self.span_diagnostic.span_err(syntax_pos::mk_sp(self.pos, self.pos), "called consume_any_line_comment, but there \ was whitespace"); } } - if self.curr_is('/') { + if self.ch_is('/') { match self.nextch() { Some('/') => { self.bump(); self.bump(); // line comments starting with "///" or "//!" are doc-comments - let doc_comment = self.curr_is('/') || self.curr_is('!'); - let start_bpos = if doc_comment { - self.pos - BytePos(3) - } else { - self.last_pos - BytePos(2) - }; + let doc_comment = self.ch_is('/') || self.ch_is('!'); + let start_bpos = self.pos - BytePos(2); while !self.is_eof() { - match self.curr.unwrap() { + match self.ch.unwrap() { '\n' => break, '\r' => { if self.nextch_is('\n') { // CRLF break; } else if doc_comment { - self.err_span_(self.last_pos, - self.pos, + self.err_span_(self.pos, + self.next_pos, "bare CR not allowed in doc-comment"); } } @@ -521,13 +568,13 @@ impl<'a> StringReader<'a> { Some(TokenAndSpan { tok: tok, - sp: syntax_pos::mk_sp(start_bpos, self.last_pos), + sp: syntax_pos::mk_sp(start_bpos, self.pos), }) }) } else { Some(TokenAndSpan { tok: token::Comment, - sp: syntax_pos::mk_sp(start_bpos, self.last_pos), + sp: syntax_pos::mk_sp(start_bpos, self.pos), }) }; } @@ -538,7 +585,7 @@ impl<'a> StringReader<'a> { } _ => None, } - } else if self.curr_is('#') { + } else if self.ch_is('#') { if self.nextch_is('!') { // Parse an inner attribute. @@ -550,17 +597,17 @@ impl<'a> StringReader<'a> { // we're at the beginning of the file... let cmap = CodeMap::new(); cmap.files.borrow_mut().push(self.filemap.clone()); - let loc = cmap.lookup_char_pos_adj(self.last_pos); + let loc = cmap.lookup_char_pos_adj(self.pos); debug!("Skipping a shebang"); if loc.line == 1 && loc.col == CharPos(0) { // FIXME: Add shebang "token", return it - let start = self.last_pos; - while !self.curr_is('\n') && !self.is_eof() { + let start = self.pos; + while !self.ch_is('\n') && !self.is_eof() { self.bump(); } return Some(TokenAndSpan { tok: token::Shebang(self.name_from(start)), - sp: syntax_pos::mk_sp(start, self.last_pos), + sp: syntax_pos::mk_sp(start, self.pos), }); } } @@ -573,7 +620,7 @@ impl<'a> StringReader<'a> { /// If there is whitespace, shebang, or a comment, scan it. Otherwise, /// return None. fn scan_whitespace_or_comment(&mut self) -> Option<TokenAndSpan> { - match self.curr.unwrap_or('\0') { + match self.ch.unwrap_or('\0') { // # to handle shebang at start of file -- this is the entry point // for skipping over all "junk" '/' | '#' => { @@ -582,13 +629,13 @@ impl<'a> StringReader<'a> { c }, c if is_pattern_whitespace(Some(c)) => { - let start_bpos = self.last_pos; - while is_pattern_whitespace(self.curr) { + let start_bpos = self.pos; + while is_pattern_whitespace(self.ch) { self.bump(); } let c = Some(TokenAndSpan { tok: token::Whitespace, - sp: syntax_pos::mk_sp(start_bpos, self.last_pos), + sp: syntax_pos::mk_sp(start_bpos, self.pos), }); debug!("scanning whitespace: {:?}", c); c @@ -600,8 +647,8 @@ impl<'a> StringReader<'a> { /// Might return a sugared-doc-attr fn scan_block_comment(&mut self) -> Option<TokenAndSpan> { // block comments starting with "/**" or "/*!" are doc-comments - let is_doc_comment = self.curr_is('*') || self.curr_is('!'); - let start_bpos = self.last_pos - BytePos(2); + let is_doc_comment = self.ch_is('*') || self.ch_is('!'); + let start_bpos = self.pos - BytePos(2); let mut level: isize = 1; let mut has_cr = false; @@ -612,10 +659,10 @@ impl<'a> StringReader<'a> { } else { "unterminated block comment" }; - let last_bpos = self.last_pos; + let last_bpos = self.pos; panic!(self.fatal_span_(start_bpos, last_bpos, msg)); } - let n = self.curr.unwrap(); + let n = self.ch.unwrap(); match n { '/' if self.nextch_is('*') => { level += 1; @@ -650,7 +697,7 @@ impl<'a> StringReader<'a> { Some(TokenAndSpan { tok: tok, - sp: syntax_pos::mk_sp(start_bpos, self.last_pos), + sp: syntax_pos::mk_sp(start_bpos, self.pos), }) }) } @@ -665,7 +712,7 @@ impl<'a> StringReader<'a> { assert!(real_radix <= scan_radix); let mut len = 0; loop { - let c = self.curr; + let c = self.ch; if c == Some('_') { debug!("skipping a _"); self.bump(); @@ -677,8 +724,8 @@ impl<'a> StringReader<'a> { // check that the hypothetical digit is actually // in range for the true radix if c.unwrap().to_digit(real_radix).is_none() { - self.err_span_(self.last_pos, - self.pos, + self.err_span_(self.pos, + self.next_pos, &format!("invalid digit for a base {} literal", real_radix)); } len += 1; @@ -693,12 +740,12 @@ impl<'a> StringReader<'a> { fn scan_number(&mut self, c: char) -> token::Lit { let num_digits; let mut base = 10; - let start_bpos = self.last_pos; + let start_bpos = self.pos; self.bump(); if c == '0' { - match self.curr.unwrap_or('\0') { + match self.ch.unwrap_or('\0') { 'b' => { self.bump(); base = 2; @@ -730,7 +777,7 @@ impl<'a> StringReader<'a> { if num_digits == 0 { self.err_span_(start_bpos, - self.last_pos, + self.pos, "no valid digits found for number"); return token::Integer(token::intern("0")); } @@ -738,26 +785,26 @@ impl<'a> StringReader<'a> { // might be a float, but don't be greedy if this is actually an // integer literal followed by field/method access or a range pattern // (`0..2` and `12.foo()`) - if self.curr_is('.') && !self.nextch_is('.') && + if self.ch_is('.') && !self.nextch_is('.') && !self.nextch() .unwrap_or('\0') .is_xid_start() { // might have stuff after the ., and if it does, it needs to start // with a number self.bump(); - if self.curr.unwrap_or('\0').is_digit(10) { + if self.ch.unwrap_or('\0').is_digit(10) { self.scan_digits(10, 10); self.scan_float_exponent(); } - let last_pos = self.last_pos; - self.check_float_base(start_bpos, last_pos, base); + let pos = self.pos; + self.check_float_base(start_bpos, pos, base); return token::Float(self.name_from(start_bpos)); } else { // it might be a float if it has an exponent - if self.curr_is('e') || self.curr_is('E') { + if self.ch_is('e') || self.ch_is('E') { self.scan_float_exponent(); - let last_pos = self.last_pos; - self.check_float_base(start_bpos, last_pos, base); + let pos = self.pos; + self.check_float_base(start_bpos, pos, base); return token::Float(self.name_from(start_bpos)); } // but we certainly have an integer! @@ -769,30 +816,30 @@ impl<'a> StringReader<'a> { /// error if too many or too few digits are encountered. fn scan_hex_digits(&mut self, n_digits: usize, delim: char, below_0x7f_only: bool) -> bool { debug!("scanning {} digits until {:?}", n_digits, delim); - let start_bpos = self.last_pos; + let start_bpos = self.pos; let mut accum_int = 0; let mut valid = true; for _ in 0..n_digits { if self.is_eof() { - let last_bpos = self.last_pos; + let last_bpos = self.pos; panic!(self.fatal_span_(start_bpos, last_bpos, "unterminated numeric character escape")); } - if self.curr_is(delim) { - let last_bpos = self.last_pos; + if self.ch_is(delim) { + let last_bpos = self.pos; self.err_span_(start_bpos, last_bpos, "numeric character escape is too short"); valid = false; break; } - let c = self.curr.unwrap_or('\x00'); + let c = self.ch.unwrap_or('\x00'); accum_int *= 16; accum_int += c.to_digit(16).unwrap_or_else(|| { - self.err_span_char(self.last_pos, - self.pos, + self.err_span_char(self.pos, + self.next_pos, "invalid character in numeric character escape", c); @@ -804,7 +851,7 @@ impl<'a> StringReader<'a> { if below_0x7f_only && accum_int >= 0x80 { self.err_span_(start_bpos, - self.last_pos, + self.pos, "this form of character escape may only be used with characters in \ the range [\\x00-\\x7f]"); valid = false; @@ -813,7 +860,7 @@ impl<'a> StringReader<'a> { match char::from_u32(accum_int) { Some(_) => valid, None => { - let last_bpos = self.last_pos; + let last_bpos = self.pos; self.err_span_(start_bpos, last_bpos, "invalid numeric character escape"); false } @@ -834,8 +881,8 @@ impl<'a> StringReader<'a> { match first_source_char { '\\' => { // '\X' for some X must be a character constant: - let escaped = self.curr; - let escaped_pos = self.last_pos; + let escaped = self.ch; + let escaped_pos = self.pos; self.bump(); match escaped { None => {} // EOF here is an error that will be checked later. @@ -844,10 +891,10 @@ impl<'a> StringReader<'a> { 'n' | 'r' | 't' | '\\' | '\'' | '"' | '0' => true, 'x' => self.scan_byte_escape(delim, !ascii_only), 'u' => { - let valid = if self.curr_is('{') { + let valid = if self.ch_is('{') { self.scan_unicode_escape(delim) && !ascii_only } else { - let span = syntax_pos::mk_sp(start, self.last_pos); + let span = syntax_pos::mk_sp(start, self.pos); self.span_diagnostic .struct_span_err(span, "incorrect unicode escape sequence") .span_help(span, @@ -858,7 +905,7 @@ impl<'a> StringReader<'a> { }; if ascii_only { self.err_span_(start, - self.last_pos, + self.pos, "unicode escape sequences cannot be used as a \ byte or in a byte string"); } @@ -869,14 +916,14 @@ impl<'a> StringReader<'a> { self.consume_whitespace(); true } - '\r' if delim == '"' && self.curr_is('\n') => { + '\r' if delim == '"' && self.ch_is('\n') => { self.consume_whitespace(); true } c => { - let last_pos = self.last_pos; + let pos = self.pos; let mut err = self.struct_err_span_char(escaped_pos, - last_pos, + pos, if ascii_only { "unknown byte escape" } else { @@ -885,13 +932,13 @@ impl<'a> StringReader<'a> { }, c); if e == '\r' { - err.span_help(syntax_pos::mk_sp(escaped_pos, last_pos), + err.span_help(syntax_pos::mk_sp(escaped_pos, pos), "this is an isolated carriage return; consider \ checking your editor and version control \ settings"); } if (e == '{' || e == '}') && !ascii_only { - err.span_help(syntax_pos::mk_sp(escaped_pos, last_pos), + err.span_help(syntax_pos::mk_sp(escaped_pos, pos), "if used in a formatting string, curly braces \ are escaped with `{{` and `}}`"); } @@ -903,9 +950,9 @@ impl<'a> StringReader<'a> { } } '\t' | '\n' | '\r' | '\'' if delim == '\'' => { - let last_pos = self.last_pos; + let pos = self.pos; self.err_span_char(start, - last_pos, + pos, if ascii_only { "byte constant must be escaped" } else { @@ -915,21 +962,21 @@ impl<'a> StringReader<'a> { return false; } '\r' => { - if self.curr_is('\n') { + if self.ch_is('\n') { self.bump(); return true; } else { self.err_span_(start, - self.last_pos, + self.pos, "bare CR not allowed in string, use \\r instead"); return false; } } _ => { if ascii_only && first_source_char > '\x7F' { - let last_pos = self.last_pos; + let pos = self.pos; self.err_span_(start, - last_pos, + pos, "byte constant must be ASCII. Use a \\xHH escape for a \ non-ASCII byte"); return false; @@ -945,29 +992,29 @@ impl<'a> StringReader<'a> { /// will read at least one digit, and up to 6, and pass over the }. fn scan_unicode_escape(&mut self, delim: char) -> bool { self.bump(); // past the { - let start_bpos = self.last_pos; + let start_bpos = self.pos; let mut count = 0; let mut accum_int = 0; let mut valid = true; - while !self.curr_is('}') && count <= 6 { - let c = match self.curr { + while !self.ch_is('}') && count <= 6 { + let c = match self.ch { Some(c) => c, None => { panic!(self.fatal_span_(start_bpos, - self.last_pos, + self.pos, "unterminated unicode escape (found EOF)")); } }; accum_int *= 16; accum_int += c.to_digit(16).unwrap_or_else(|| { if c == delim { - panic!(self.fatal_span_(self.last_pos, - self.pos, + panic!(self.fatal_span_(self.pos, + self.next_pos, "unterminated unicode escape (needed a `}`)")); } else { - self.err_span_char(self.last_pos, - self.pos, + self.err_span_char(self.pos, + self.next_pos, "invalid character in unicode escape", c); } @@ -980,14 +1027,14 @@ impl<'a> StringReader<'a> { if count > 6 { self.err_span_(start_bpos, - self.last_pos, + self.pos, "overlong unicode escape (can have at most 6 hex digits)"); valid = false; } if valid && (char::from_u32(accum_int).is_none() || count == 0) { self.err_span_(start_bpos, - self.last_pos, + self.pos, "invalid unicode character escape"); valid = false; } @@ -998,14 +1045,14 @@ impl<'a> StringReader<'a> { /// Scan over a float exponent. fn scan_float_exponent(&mut self) { - if self.curr_is('e') || self.curr_is('E') { + if self.ch_is('e') || self.ch_is('E') { self.bump(); - if self.curr_is('-') || self.curr_is('+') { + if self.ch_is('-') || self.ch_is('+') { self.bump(); } if self.scan_digits(10, 10) == 0 { - self.err_span_(self.last_pos, - self.pos, + self.err_span_(self.pos, + self.next_pos, "expected at least one digit in exponent") } } @@ -1036,7 +1083,7 @@ impl<'a> StringReader<'a> { fn binop(&mut self, op: token::BinOpToken) -> token::Token { self.bump(); - if self.curr_is('=') { + if self.ch_is('=') { self.bump(); return token::BinOpEq(op); } else { @@ -1047,7 +1094,7 @@ impl<'a> StringReader<'a> { /// Return the next token from the string, advances the input past that /// token, and updates the interner fn next_token_inner(&mut self) -> Result<token::Token, ()> { - let c = self.curr; + let c = self.ch; if ident_start(c) && match (c.unwrap(), self.nextch(), self.nextnextch()) { // Note: r as in r" or r#" is part of a raw string literal, @@ -1061,8 +1108,8 @@ impl<'a> StringReader<'a> { ('b', Some('r'), Some('#')) => false, _ => true, } { - let start = self.last_pos; - while ident_continue(self.curr) { + let start = self.pos; + while ident_continue(self.ch) { self.bump(); } @@ -1095,9 +1142,9 @@ impl<'a> StringReader<'a> { } '.' => { self.bump(); - return if self.curr_is('.') { + return if self.ch_is('.') { self.bump(); - if self.curr_is('.') { + if self.ch_is('.') { self.bump(); Ok(token::DotDotDot) } else { @@ -1149,7 +1196,7 @@ impl<'a> StringReader<'a> { } ':' => { self.bump(); - if self.curr_is(':') { + if self.ch_is(':') { self.bump(); return Ok(token::ModSep); } else { @@ -1165,10 +1212,10 @@ impl<'a> StringReader<'a> { // Multi-byte tokens. '=' => { self.bump(); - if self.curr_is('=') { + if self.ch_is('=') { self.bump(); return Ok(token::EqEq); - } else if self.curr_is('>') { + } else if self.ch_is('>') { self.bump(); return Ok(token::FatArrow); } else { @@ -1177,7 +1224,7 @@ impl<'a> StringReader<'a> { } '!' => { self.bump(); - if self.curr_is('=') { + if self.ch_is('=') { self.bump(); return Ok(token::Ne); } else { @@ -1186,7 +1233,7 @@ impl<'a> StringReader<'a> { } '<' => { self.bump(); - match self.curr.unwrap_or('\x00') { + match self.ch.unwrap_or('\x00') { '=' => { self.bump(); return Ok(token::Le); @@ -1196,7 +1243,7 @@ impl<'a> StringReader<'a> { } '-' => { self.bump(); - match self.curr.unwrap_or('\x00') { + match self.ch.unwrap_or('\x00') { _ => { return Ok(token::LArrow); } @@ -1209,7 +1256,7 @@ impl<'a> StringReader<'a> { } '>' => { self.bump(); - match self.curr.unwrap_or('\x00') { + match self.ch.unwrap_or('\x00') { '=' => { self.bump(); return Ok(token::Ge); @@ -1224,25 +1271,25 @@ impl<'a> StringReader<'a> { } '\'' => { // Either a character constant 'a' OR a lifetime name 'abc - let start_with_quote = self.last_pos; + let start_with_quote = self.pos; self.bump(); - let start = self.last_pos; + let start = self.pos; // the eof will be picked up by the final `'` check below - let c2 = self.curr.unwrap_or('\x00'); + let c2 = self.ch.unwrap_or('\x00'); self.bump(); // If the character is an ident start not followed by another single // quote, then this is a lifetime name: - if ident_start(Some(c2)) && !self.curr_is('\'') { - while ident_continue(self.curr) { + if ident_start(Some(c2)) && !self.ch_is('\'') { + while ident_continue(self.ch) { self.bump(); } // lifetimes shouldn't end with a single quote // if we find one, then this is an invalid character literal - if self.curr_is('\'') { + if self.ch_is('\'') { panic!(self.fatal_span_verbose( - start_with_quote, self.pos, + start_with_quote, self.next_pos, String::from("character literal may only contain one codepoint"))); } @@ -1260,7 +1307,7 @@ impl<'a> StringReader<'a> { str_to_ident(lifetime_name) }); let keyword_checking_token = &token::Ident(keyword_checking_ident); - let last_bpos = self.last_pos; + let last_bpos = self.pos; if keyword_checking_token.is_any_keyword() && !keyword_checking_token.is_keyword(keywords::Static) { self.err_span_(start, last_bpos, "lifetimes cannot use keyword names"); @@ -1275,9 +1322,9 @@ impl<'a> StringReader<'a> { false, '\''); - if !self.curr_is('\'') { + if !self.ch_is('\'') { panic!(self.fatal_span_verbose( - start_with_quote, self.last_pos, + start_with_quote, self.pos, String::from("character literal may only contain one codepoint"))); } @@ -1286,13 +1333,13 @@ impl<'a> StringReader<'a> { } else { token::intern("0") }; - self.bump(); // advance curr past token + self.bump(); // advance ch past token let suffix = self.scan_optional_raw_name(); return Ok(token::Literal(token::Char(id), suffix)); } 'b' => { self.bump(); - let lit = match self.curr { + let lit = match self.ch { Some('\'') => self.scan_byte(), Some('"') => self.scan_byte_string(), Some('r') => self.scan_raw_byte_string(), @@ -1302,19 +1349,19 @@ impl<'a> StringReader<'a> { return Ok(token::Literal(lit, suffix)); } '"' => { - let start_bpos = self.last_pos; + let start_bpos = self.pos; let mut valid = true; self.bump(); - while !self.curr_is('"') { + while !self.ch_is('"') { if self.is_eof() { - let last_bpos = self.last_pos; + let last_bpos = self.pos; panic!(self.fatal_span_(start_bpos, last_bpos, "unterminated double quote string")); } - let ch_start = self.last_pos; - let ch = self.curr.unwrap(); + let ch_start = self.pos; + let ch = self.ch.unwrap(); self.bump(); valid &= self.scan_char_or_byte(ch_start, ch, @@ -1333,20 +1380,20 @@ impl<'a> StringReader<'a> { return Ok(token::Literal(token::Str_(id), suffix)); } 'r' => { - let start_bpos = self.last_pos; + let start_bpos = self.pos; self.bump(); let mut hash_count = 0; - while self.curr_is('#') { + while self.ch_is('#') { self.bump(); hash_count += 1; } if self.is_eof() { - let last_bpos = self.last_pos; + let last_bpos = self.pos; panic!(self.fatal_span_(start_bpos, last_bpos, "unterminated raw string")); - } else if !self.curr_is('"') { - let last_bpos = self.last_pos; - let curr_char = self.curr.unwrap(); + } else if !self.ch_is('"') { + let last_bpos = self.pos; + let curr_char = self.ch.unwrap(); panic!(self.fatal_span_char(start_bpos, last_bpos, "found invalid character; only `#` is allowed \ @@ -1354,27 +1401,27 @@ impl<'a> StringReader<'a> { curr_char)); } self.bump(); - let content_start_bpos = self.last_pos; + let content_start_bpos = self.pos; let mut content_end_bpos; let mut valid = true; 'outer: loop { if self.is_eof() { - let last_bpos = self.last_pos; + let last_bpos = self.pos; panic!(self.fatal_span_(start_bpos, last_bpos, "unterminated raw string")); } - // if self.curr_is('"') { - // content_end_bpos = self.last_pos; + // if self.ch_is('"') { + // content_end_bpos = self.pos; // for _ in 0..hash_count { // self.bump(); - // if !self.curr_is('#') { + // if !self.ch_is('#') { // continue 'outer; - let c = self.curr.unwrap(); + let c = self.ch.unwrap(); match c { '"' => { - content_end_bpos = self.last_pos; + content_end_bpos = self.pos; for _ in 0..hash_count { self.bump(); - if !self.curr_is('#') { + if !self.ch_is('#') { continue 'outer; } } @@ -1382,7 +1429,7 @@ impl<'a> StringReader<'a> { } '\r' => { if !self.nextch_is('\n') { - let last_bpos = self.last_pos; + let last_bpos = self.pos; self.err_span_(start_bpos, last_bpos, "bare CR not allowed in raw string, use \\r \ @@ -1449,8 +1496,8 @@ impl<'a> StringReader<'a> { return Ok(self.binop(token::Percent)); } c => { - let last_bpos = self.last_pos; - let bpos = self.pos; + let last_bpos = self.pos; + let bpos = self.next_pos; let mut err = self.struct_fatal_span_char(last_bpos, bpos, "unknown start of token", @@ -1463,18 +1510,18 @@ impl<'a> StringReader<'a> { } fn consume_whitespace(&mut self) { - while is_pattern_whitespace(self.curr) && !self.is_eof() { + while is_pattern_whitespace(self.ch) && !self.is_eof() { self.bump(); } } fn read_to_eol(&mut self) -> String { let mut val = String::new(); - while !self.curr_is('\n') && !self.is_eof() { - val.push(self.curr.unwrap()); + while !self.ch_is('\n') && !self.is_eof() { + val.push(self.ch.unwrap()); self.bump(); } - if self.curr_is('\n') { + if self.ch_is('\n') { self.bump(); } return val; @@ -1488,23 +1535,23 @@ impl<'a> StringReader<'a> { } fn consume_non_eol_whitespace(&mut self) { - while is_pattern_whitespace(self.curr) && !self.curr_is('\n') && !self.is_eof() { + while is_pattern_whitespace(self.ch) && !self.ch_is('\n') && !self.is_eof() { self.bump(); } } fn peeking_at_comment(&self) -> bool { - (self.curr_is('/') && self.nextch_is('/')) || (self.curr_is('/') && self.nextch_is('*')) || + (self.ch_is('/') && self.nextch_is('/')) || (self.ch_is('/') && self.nextch_is('*')) || // consider shebangs comments, but not inner attributes - (self.curr_is('#') && self.nextch_is('!') && !self.nextnextch_is('[')) + (self.ch_is('#') && self.nextch_is('!') && !self.nextnextch_is('[')) } fn scan_byte(&mut self) -> token::Lit { self.bump(); - let start = self.last_pos; + let start = self.pos; // the eof will be picked up by the final `'` check below - let c2 = self.curr.unwrap_or('\x00'); + let c2 = self.ch.unwrap_or('\x00'); self.bump(); let valid = self.scan_char_or_byte(start, @@ -1512,13 +1559,13 @@ impl<'a> StringReader<'a> { // ascii_only = true, '\''); - if !self.curr_is('\'') { + if !self.ch_is('\'') { // Byte offsetting here is okay because the // character before position `start` are an // ascii single quote and ascii 'b'. - let last_pos = self.last_pos; + let pos = self.pos; panic!(self.fatal_span_verbose(start - BytePos(2), - last_pos, + pos, "unterminated byte constant".to_string())); } @@ -1527,7 +1574,7 @@ impl<'a> StringReader<'a> { } else { token::intern("?") }; - self.bump(); // advance curr past token + self.bump(); // advance ch past token return token::Byte(id); } @@ -1537,17 +1584,17 @@ impl<'a> StringReader<'a> { fn scan_byte_string(&mut self) -> token::Lit { self.bump(); - let start = self.last_pos; + let start = self.pos; let mut valid = true; - while !self.curr_is('"') { + while !self.ch_is('"') { if self.is_eof() { - let last_pos = self.last_pos; - panic!(self.fatal_span_(start, last_pos, "unterminated double quote byte string")); + let pos = self.pos; + panic!(self.fatal_span_(start, pos, "unterminated double quote byte string")); } - let ch_start = self.last_pos; - let ch = self.curr.unwrap(); + let ch_start = self.pos; + let ch = self.ch.unwrap(); self.bump(); valid &= self.scan_char_or_byte(ch_start, ch, @@ -1565,40 +1612,40 @@ impl<'a> StringReader<'a> { } fn scan_raw_byte_string(&mut self) -> token::Lit { - let start_bpos = self.last_pos; + let start_bpos = self.pos; self.bump(); let mut hash_count = 0; - while self.curr_is('#') { + while self.ch_is('#') { self.bump(); hash_count += 1; } if self.is_eof() { - let last_pos = self.last_pos; - panic!(self.fatal_span_(start_bpos, last_pos, "unterminated raw string")); - } else if !self.curr_is('"') { - let last_pos = self.last_pos; - let ch = self.curr.unwrap(); + let pos = self.pos; + panic!(self.fatal_span_(start_bpos, pos, "unterminated raw string")); + } else if !self.ch_is('"') { + let pos = self.pos; + let ch = self.ch.unwrap(); panic!(self.fatal_span_char(start_bpos, - last_pos, + pos, "found invalid character; only `#` is allowed in raw \ string delimitation", ch)); } self.bump(); - let content_start_bpos = self.last_pos; + let content_start_bpos = self.pos; let mut content_end_bpos; 'outer: loop { - match self.curr { + match self.ch { None => { - let last_pos = self.last_pos; - panic!(self.fatal_span_(start_bpos, last_pos, "unterminated raw string")) + let pos = self.pos; + panic!(self.fatal_span_(start_bpos, pos, "unterminated raw string")) } Some('"') => { - content_end_bpos = self.last_pos; + content_end_bpos = self.pos; for _ in 0..hash_count { self.bump(); - if !self.curr_is('#') { + if !self.ch_is('#') { continue 'outer; } } @@ -1606,8 +1653,8 @@ impl<'a> StringReader<'a> { } Some(c) => { if c > '\x7F' { - let last_pos = self.last_pos; - self.err_span_char(last_pos, last_pos, "raw byte string must be ASCII", c); + let pos = self.pos; + self.err_span_char(pos, pos, "raw byte string must be ASCII", c); } } } @@ -1721,7 +1768,7 @@ mod tests { assert_eq!(tok1, tok2); assert_eq!(string_reader.next_token().tok, token::Whitespace); // the 'main' id is already read: - assert_eq!(string_reader.last_pos.clone(), BytePos(28)); + assert_eq!(string_reader.pos.clone(), BytePos(28)); // read another token: let tok3 = string_reader.next_token(); let tok4 = TokenAndSpan { @@ -1734,7 +1781,7 @@ mod tests { }; assert_eq!(tok3, tok4); // the lparen is already read: - assert_eq!(string_reader.last_pos.clone(), BytePos(29)) + assert_eq!(string_reader.pos.clone(), BytePos(29)) } // check that the given reader produces the desired stream diff --git a/src/libsyntax/parse/lexer/unicode_chars.rs b/src/libsyntax/parse/lexer/unicode_chars.rs index dab97d1d5a6..1e08b20b7e1 100644 --- a/src/libsyntax/parse/lexer/unicode_chars.rs +++ b/src/libsyntax/parse/lexer/unicode_chars.rs @@ -234,7 +234,7 @@ pub fn check_for_substitution<'a>(reader: &StringReader<'a>, .iter() .find(|&&(c, _, _)| c == ch) .map(|&(_, u_name, ascii_char)| { - let span = make_span(reader.last_pos, reader.pos); + let span = make_span(reader.pos, reader.next_pos); match ASCII_ARRAY.iter().find(|&&(c, _)| c == ascii_char) { Some(&(ascii_char, ascii_name)) => { let msg = diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index a1eceb6921c..12408c7d3c9 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -10,10 +10,11 @@ //! The main parser interface -use ast; +use ast::{self, CrateConfig}; use codemap::CodeMap; use syntax_pos::{self, Span, FileMap}; use errors::{Handler, ColorConfig, DiagnosticBuilder}; +use feature_gate::UnstableFeatures; use parse::parser::Parser; use parse::token::InternedString; use ptr::P; @@ -42,13 +43,15 @@ pub mod obsolete; /// Info about a parsing session. pub struct ParseSess { pub span_diagnostic: Handler, // better be the same as the one in the reader! + pub unstable_features: UnstableFeatures, + pub config: CrateConfig, /// Used to determine and report recursive mod inclusions included_mod_stack: RefCell<Vec<PathBuf>>, code_map: Rc<CodeMap>, } impl ParseSess { - pub fn new() -> ParseSess { + pub fn new() -> Self { let cm = Rc::new(CodeMap::new()); let handler = Handler::with_tty_emitter(ColorConfig::Auto, true, @@ -60,6 +63,8 @@ impl ParseSess { pub fn with_span_handler(handler: Handler, code_map: Rc<CodeMap>) -> ParseSess { ParseSess { span_diagnostic: handler, + unstable_features: UnstableFeatures::from_environment(), + config: Vec::new(), included_mod_stack: RefCell::new(vec![]), code_map: code_map } @@ -75,146 +80,90 @@ impl ParseSess { // uses a HOF to parse anything, and <source> includes file and // source_str. -pub fn parse_crate_from_file<'a>(input: &Path, - cfg: ast::CrateConfig, - sess: &'a ParseSess) - -> PResult<'a, ast::Crate> { - let mut parser = new_parser_from_file(sess, cfg, input); +pub fn parse_crate_from_file<'a>(input: &Path, sess: &'a ParseSess) -> PResult<'a, ast::Crate> { + let mut parser = new_parser_from_file(sess, input); parser.parse_crate_mod() } -pub fn parse_crate_attrs_from_file<'a>(input: &Path, - cfg: ast::CrateConfig, - sess: &'a ParseSess) +pub fn parse_crate_attrs_from_file<'a>(input: &Path, sess: &'a ParseSess) -> PResult<'a, Vec<ast::Attribute>> { - let mut parser = new_parser_from_file(sess, cfg, input); + let mut parser = new_parser_from_file(sess, input); parser.parse_inner_attributes() } -pub fn parse_crate_from_source_str<'a>(name: String, - source: String, - cfg: ast::CrateConfig, - sess: &'a ParseSess) +pub fn parse_crate_from_source_str<'a>(name: String, source: String, sess: &'a ParseSess) -> PResult<'a, ast::Crate> { - let mut p = new_parser_from_source_str(sess, - cfg, - name, - source); - p.parse_crate_mod() + new_parser_from_source_str(sess, name, source).parse_crate_mod() } -pub fn parse_crate_attrs_from_source_str<'a>(name: String, - source: String, - cfg: ast::CrateConfig, - sess: &'a ParseSess) +pub fn parse_crate_attrs_from_source_str<'a>(name: String, source: String, sess: &'a ParseSess) -> PResult<'a, Vec<ast::Attribute>> { - let mut p = new_parser_from_source_str(sess, - cfg, - name, - source); - p.parse_inner_attributes() + new_parser_from_source_str(sess, name, source).parse_inner_attributes() } -pub fn parse_expr_from_source_str<'a>(name: String, - source: String, - cfg: ast::CrateConfig, - sess: &'a ParseSess) +pub fn parse_expr_from_source_str<'a>(name: String, source: String, sess: &'a ParseSess) -> PResult<'a, P<ast::Expr>> { - let mut p = new_parser_from_source_str(sess, cfg, name, source); - p.parse_expr() + new_parser_from_source_str(sess, name, source).parse_expr() } /// Parses an item. /// /// Returns `Ok(Some(item))` when successful, `Ok(None)` when no item was found, and`Err` /// when a syntax error occurred. -pub fn parse_item_from_source_str<'a>(name: String, - source: String, - cfg: ast::CrateConfig, - sess: &'a ParseSess) +pub fn parse_item_from_source_str<'a>(name: String, source: String, sess: &'a ParseSess) -> PResult<'a, Option<P<ast::Item>>> { - let mut p = new_parser_from_source_str(sess, cfg, name, source); - p.parse_item() + new_parser_from_source_str(sess, name, source).parse_item() } -pub fn parse_meta_from_source_str<'a>(name: String, - source: String, - cfg: ast::CrateConfig, - sess: &'a ParseSess) +pub fn parse_meta_from_source_str<'a>(name: String, source: String, sess: &'a ParseSess) -> PResult<'a, P<ast::MetaItem>> { - let mut p = new_parser_from_source_str(sess, cfg, name, source); - p.parse_meta_item() + new_parser_from_source_str(sess, name, source).parse_meta_item() } -pub fn parse_stmt_from_source_str<'a>(name: String, - source: String, - cfg: ast::CrateConfig, - sess: &'a ParseSess) +pub fn parse_stmt_from_source_str<'a>(name: String, source: String, sess: &'a ParseSess) -> PResult<'a, Option<ast::Stmt>> { - let mut p = new_parser_from_source_str( - sess, - cfg, - name, - source - ); - p.parse_stmt() + new_parser_from_source_str(sess, name, source).parse_stmt() } // Warning: This parses with quote_depth > 0, which is not the default. -pub fn parse_tts_from_source_str<'a>(name: String, - source: String, - cfg: ast::CrateConfig, - sess: &'a ParseSess) +pub fn parse_tts_from_source_str<'a>(name: String, source: String, sess: &'a ParseSess) -> PResult<'a, Vec<tokenstream::TokenTree>> { - let mut p = new_parser_from_source_str( - sess, - cfg, - name, - source - ); + let mut p = new_parser_from_source_str(sess, name, source); p.quote_depth += 1; // right now this is re-creating the token trees from ... token trees. p.parse_all_token_trees() } // Create a new parser from a source string -pub fn new_parser_from_source_str<'a>(sess: &'a ParseSess, - cfg: ast::CrateConfig, - name: String, - source: String) +pub fn new_parser_from_source_str<'a>(sess: &'a ParseSess, name: String, source: String) -> Parser<'a> { - filemap_to_parser(sess, sess.codemap().new_filemap(name, None, source), cfg) + filemap_to_parser(sess, sess.codemap().new_filemap(name, None, source)) } /// Create a new parser, handling errors as appropriate /// if the file doesn't exist -pub fn new_parser_from_file<'a>(sess: &'a ParseSess, - cfg: ast::CrateConfig, - path: &Path) -> Parser<'a> { - filemap_to_parser(sess, file_to_filemap(sess, path, None), cfg) +pub fn new_parser_from_file<'a>(sess: &'a ParseSess, path: &Path) -> Parser<'a> { + filemap_to_parser(sess, file_to_filemap(sess, path, None)) } /// Given a session, a crate config, a path, and a span, add /// the file at the given path to the codemap, and return a parser. /// On an error, use the given span as the source of the problem. pub fn new_sub_parser_from_file<'a>(sess: &'a ParseSess, - cfg: ast::CrateConfig, path: &Path, owns_directory: bool, module_name: Option<String>, sp: Span) -> Parser<'a> { - let mut p = filemap_to_parser(sess, file_to_filemap(sess, path, Some(sp)), cfg); + let mut p = filemap_to_parser(sess, file_to_filemap(sess, path, Some(sp))); p.owns_directory = owns_directory; p.root_module_name = module_name; p } /// Given a filemap and config, return a parser -pub fn filemap_to_parser<'a>(sess: &'a ParseSess, - filemap: Rc<FileMap>, - cfg: ast::CrateConfig) -> Parser<'a> { +pub fn filemap_to_parser<'a>(sess: &'a ParseSess, filemap: Rc<FileMap>, ) -> Parser<'a> { let end_pos = filemap.end_pos; - let mut parser = tts_to_parser(sess, filemap_to_tts(sess, filemap), cfg); + let mut parser = tts_to_parser(sess, filemap_to_tts(sess, filemap)); if parser.token == token::Eof && parser.span == syntax_pos::DUMMY_SP { parser.span = syntax_pos::mk_sp(end_pos, end_pos); @@ -225,18 +174,13 @@ pub fn filemap_to_parser<'a>(sess: &'a ParseSess, // must preserve old name for now, because quote! from the *existing* // compiler expands into it -pub fn new_parser_from_tts<'a>(sess: &'a ParseSess, - cfg: ast::CrateConfig, - tts: Vec<tokenstream::TokenTree>) +pub fn new_parser_from_tts<'a>(sess: &'a ParseSess, tts: Vec<tokenstream::TokenTree>) -> Parser<'a> { - tts_to_parser(sess, tts, cfg) + tts_to_parser(sess, tts) } -pub fn new_parser_from_ts<'a>(sess: &'a ParseSess, - cfg: ast::CrateConfig, - ts: tokenstream::TokenStream) - -> Parser<'a> { - tts_to_parser(sess, ts.to_tts(), cfg) +pub fn new_parser_from_ts<'a>(sess: &'a ParseSess, ts: tokenstream::TokenStream) -> Parser<'a> { + tts_to_parser(sess, ts.to_tts()) } @@ -263,18 +207,15 @@ pub fn filemap_to_tts(sess: &ParseSess, filemap: Rc<FileMap>) -> Vec<tokenstream::TokenTree> { // it appears to me that the cfg doesn't matter here... indeed, // parsing tt's probably shouldn't require a parser at all. - let cfg = Vec::new(); let srdr = lexer::StringReader::new(&sess.span_diagnostic, filemap); - let mut p1 = Parser::new(sess, cfg, Box::new(srdr)); + let mut p1 = Parser::new(sess, Box::new(srdr)); panictry!(p1.parse_all_token_trees()) } -/// Given tts and cfg, produce a parser -pub fn tts_to_parser<'a>(sess: &'a ParseSess, - tts: Vec<tokenstream::TokenTree>, - cfg: ast::CrateConfig) -> Parser<'a> { - let trdr = lexer::new_tt_reader(&sess.span_diagnostic, None, None, tts); - let mut p = Parser::new(sess, cfg, Box::new(trdr)); +/// Given tts and the ParseSess, produce a parser +pub fn tts_to_parser<'a>(sess: &'a ParseSess, tts: Vec<tokenstream::TokenTree>) -> Parser<'a> { + let trdr = lexer::new_tt_reader(&sess.span_diagnostic, None, tts); + let mut p = Parser::new(sess, Box::new(trdr)); p.check_unknown_macro_variable(); p } @@ -286,52 +227,37 @@ pub fn tts_to_parser<'a>(sess: &'a ParseSess, pub fn char_lit(lit: &str) -> (char, isize) { use std::char; - let mut chars = lit.chars(); - match (chars.next(), chars.next()) { - (Some(c), None) if c != '\\' => return (c, 1), - (Some('\\'), Some(c)) => match c { - '"' => return ('"', 2), - 'n' => return ('\n', 2), - 'r' => return ('\r', 2), - 't' => return ('\t', 2), - '\\' => return ('\\', 2), - '\'' => return ('\'', 2), - '0' => return ('\0', 2), - _ => {} - }, - _ => panic!("lexer accepted invalid char escape `{}`", lit) - }; - - fn esc(len: usize, lit: &str) -> Option<(char, isize)> { - u32::from_str_radix(&lit[2..len], 16).ok() - .and_then(char::from_u32) - .map(|x| (x, len as isize)) + // Handle non-escaped chars first. + if lit.as_bytes()[0] != b'\\' { + // If the first byte isn't '\\' it might part of a multi-byte char, so + // get the char with chars(). + let c = lit.chars().next().unwrap(); + return (c, 1); } - let unicode_escape = || -> Option<(char, isize)> { - if lit.as_bytes()[2] == b'{' { - let idx = lit.find('}').unwrap_or_else(|| { - panic!("lexer should have rejected a bad character escape {}", lit) - }); - - let subslice = &lit[3..idx]; - u32::from_str_radix(subslice, 16).ok() - .and_then(char::from_u32) - .map(|x| (x, subslice.chars().count() as isize + 4)) - } else { - esc(6, lit) + // Handle escaped chars. + match lit.as_bytes()[1] as char { + '"' => ('"', 2), + 'n' => ('\n', 2), + 'r' => ('\r', 2), + 't' => ('\t', 2), + '\\' => ('\\', 2), + '\'' => ('\'', 2), + '0' => ('\0', 2), + 'x' => { + let v = u32::from_str_radix(&lit[2..4], 16).unwrap(); + let c = char::from_u32(v).unwrap(); + (c, 4) } - }; - - // Unicode escapes - return match lit.as_bytes()[1] as char { - 'x' | 'X' => esc(4, lit), - 'u' => unicode_escape(), - 'U' => esc(10, lit), - _ => None, - }.unwrap_or_else(|| { - panic!("lexer should have rejected a bad character escape {}", lit) - }) + 'u' => { + assert!(lit.as_bytes()[2] == b'{'); + let idx = lit.find('}').unwrap(); + let v = u32::from_str_radix(&lit[3..idx], 16).unwrap(); + let c = char::from_u32(v).unwrap(); + (c, (idx + 1) as isize) + } + _ => panic!("lexer should have rejected a bad character escape {}", lit) + } } /// Parse a string representing a string literal into its final form. Does @@ -698,12 +624,12 @@ mod tests { node: ast::ExprKind::Path(None, ast::Path { span: sp(0, 1), global: false, - segments: vec!( + segments: vec![ ast::PathSegment { identifier: str_to_ident("a"), parameters: ast::PathParameters::none(), } - ), + ], }), span: sp(0, 1), attrs: ThinVec::new(), @@ -717,7 +643,7 @@ mod tests { node: ast::ExprKind::Path(None, ast::Path { span: sp(0, 6), global: true, - segments: vec!( + segments: vec![ ast::PathSegment { identifier: str_to_ident("a"), parameters: ast::PathParameters::none(), @@ -726,7 +652,7 @@ mod tests { identifier: str_to_ident("b"), parameters: ast::PathParameters::none(), } - ) + ] }), span: sp(0, 6), attrs: ThinVec::new(), @@ -837,12 +763,12 @@ mod tests { node:ast::ExprKind::Path(None, ast::Path{ span: sp(7, 8), global: false, - segments: vec!( + segments: vec![ ast::PathSegment { identifier: str_to_ident("d"), parameters: ast::PathParameters::none(), } - ), + ], }), span:sp(7,8), attrs: ThinVec::new(), @@ -860,12 +786,12 @@ mod tests { node: ast::ExprKind::Path(None, ast::Path { span:sp(0,1), global:false, - segments: vec!( + segments: vec![ ast::PathSegment { identifier: str_to_ident("b"), parameters: ast::PathParameters::none(), } - ), + ], }), span: sp(0,1), attrs: ThinVec::new()})), @@ -902,18 +828,18 @@ mod tests { attrs:Vec::new(), id: ast::DUMMY_NODE_ID, node: ast::ItemKind::Fn(P(ast::FnDecl { - inputs: vec!(ast::Arg{ + inputs: vec![ast::Arg{ ty: P(ast::Ty{id: ast::DUMMY_NODE_ID, node: ast::TyKind::Path(None, ast::Path{ span:sp(10,13), global:false, - segments: vec!( + segments: vec![ ast::PathSegment { identifier: str_to_ident("i32"), parameters: ast::PathParameters::none(), } - ), + ], }), span:sp(10,13) }), @@ -929,7 +855,7 @@ mod tests { span: sp(6,7) }), id: ast::DUMMY_NODE_ID - }), + }], output: ast::FunctionRetTy::Default(sp(15, 15)), variadic: false }), @@ -949,14 +875,14 @@ mod tests { span: syntax_pos::DUMMY_SP, }, P(ast::Block { - stmts: vec!(ast::Stmt { + stmts: vec![ast::Stmt { node: ast::StmtKind::Semi(P(ast::Expr{ id: ast::DUMMY_NODE_ID, node: ast::ExprKind::Path(None, ast::Path{ span:sp(17,18), global:false, - segments: vec!( + segments: vec![ ast::PathSegment { identifier: str_to_ident( @@ -964,12 +890,12 @@ mod tests { parameters: ast::PathParameters::none(), } - ), + ], }), span: sp(17,18), attrs: ThinVec::new()})), id: ast::DUMMY_NODE_ID, - span: sp(17,19)}), + span: sp(17,19)}], id: ast::DUMMY_NODE_ID, rules: ast::BlockCheckMode::Default, // no idea span: sp(15,21), @@ -1069,13 +995,13 @@ mod tests { let name = "<source>".to_string(); let source = "/// doc comment\r\nfn foo() {}".to_string(); - let item = parse_item_from_source_str(name.clone(), source, Vec::new(), &sess) + let item = parse_item_from_source_str(name.clone(), source, &sess) .unwrap().unwrap(); let doc = first_attr_value_str_by_name(&item.attrs, "doc").unwrap(); assert_eq!(&doc[..], "/// doc comment"); let source = "/// doc comment\r\n/// line 2\r\nfn foo() {}".to_string(); - let item = parse_item_from_source_str(name.clone(), source, Vec::new(), &sess) + let item = parse_item_from_source_str(name.clone(), source, &sess) .unwrap().unwrap(); let docs = item.attrs.iter().filter(|a| &*a.name() == "doc") .map(|a| a.value_str().unwrap().to_string()).collect::<Vec<_>>(); @@ -1083,7 +1009,7 @@ mod tests { assert_eq!(&docs[..], b); let source = "/** doc comment\r\n * with CRLF */\r\nfn foo() {}".to_string(); - let item = parse_item_from_source_str(name, source, Vec::new(), &sess).unwrap().unwrap(); + let item = parse_item_from_source_str(name, source, &sess).unwrap().unwrap(); let doc = first_attr_value_str_by_name(&item.attrs, "doc").unwrap(); assert_eq!(&doc[..], "/** doc comment\n * with CRLF */"); } @@ -1092,7 +1018,7 @@ mod tests { fn ttdelim_span() { let sess = ParseSess::new(); let expr = parse::parse_expr_from_source_str("foo".to_string(), - "foo!( fn main() { body } )".to_string(), vec![], &sess).unwrap(); + "foo!( fn main() { body } )".to_string(), &sess).unwrap(); let tts = match expr.node { ast::ExprKind::Mac(ref mac) => mac.node.tts.clone(), diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index d0936fd981b..b80aa667be6 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -15,7 +15,7 @@ use ast::Unsafety; use ast::{Mod, Arg, Arm, Attribute, BindingMode, TraitItemKind}; use ast::Block; use ast::{BlockCheckMode, CaptureBy}; -use ast::{Constness, Crate, CrateConfig}; +use ast::{Constness, Crate}; use ast::Defaultness; use ast::EnumDef; use ast::{Expr, ExprKind, RangeLimits}; @@ -48,8 +48,7 @@ use parse::classify; use parse::common::SeqSep; use parse::lexer::{Reader, TokenAndSpan}; use parse::obsolete::ObsoleteSyntax; -use parse::token::{self, intern, MatchNt, SubstNt, SpecialVarNt, InternedString}; -use parse::token::{keywords, SpecialMacroVar}; +use parse::token::{self, intern, keywords, MatchNt, SubstNt, InternedString}; use parse::{new_sub_parser_from_file, ParseSess}; use util::parser::{AssocOp, Fixity}; use print::pprust; @@ -237,6 +236,31 @@ fn maybe_append(mut lhs: Vec<Attribute>, rhs: Option<Vec<Attribute>>) lhs } +#[derive(PartialEq)] +enum PrevTokenKind { + DocComment, + Comma, + Interpolated, + Eof, + Other, +} + +// Simple circular buffer used for keeping few next tokens. +#[derive(Default)] +struct LookaheadBuffer { + buffer: [TokenAndSpan; LOOKAHEAD_BUFFER_CAPACITY], + start: usize, + end: usize, +} + +const LOOKAHEAD_BUFFER_CAPACITY: usize = 8; + +impl LookaheadBuffer { + fn len(&self) -> usize { + (LOOKAHEAD_BUFFER_CAPACITY + self.end - self.start) % LOOKAHEAD_BUFFER_CAPACITY + } +} + /* ident is handled by common.rs */ pub struct Parser<'a> { @@ -245,16 +269,11 @@ pub struct Parser<'a> { pub token: token::Token, /// the span of the current token: pub span: Span, - /// the span of the prior token: - pub last_span: Span, - pub cfg: CrateConfig, - /// the previous token or None (only stashed sometimes). - pub last_token: Option<Box<token::Token>>, - last_token_interpolated: bool, - last_token_eof: bool, - pub buffer: [TokenAndSpan; 4], - pub buffer_start: isize, - pub buffer_end: isize, + /// the span of the previous token: + pub prev_span: Span, + /// the previous token kind + prev_token_kind: PrevTokenKind, + lookahead_buffer: LookaheadBuffer, pub tokens_consumed: usize, pub restrictions: Restrictions, pub quote_depth: usize, // not (yet) related to the quasiquoter @@ -338,11 +357,7 @@ impl From<P<Expr>> for LhsExpr { } impl<'a> Parser<'a> { - pub fn new(sess: &'a ParseSess, - cfg: ast::CrateConfig, - mut rdr: Box<Reader+'a>) - -> Parser<'a> - { + pub fn new(sess: &'a ParseSess, mut rdr: Box<Reader+'a>) -> Self { let tok0 = rdr.real_token(); let span = tok0.sp; let mut directory = match span { @@ -350,29 +365,15 @@ impl<'a> Parser<'a> { _ => PathBuf::from(sess.codemap().span_to_filename(span)), }; directory.pop(); - let placeholder = TokenAndSpan { - tok: token::Underscore, - sp: span, - }; Parser { reader: rdr, sess: sess, - cfg: cfg, token: tok0.tok, span: span, - last_span: span, - last_token: None, - last_token_interpolated: false, - last_token_eof: false, - buffer: [ - placeholder.clone(), - placeholder.clone(), - placeholder.clone(), - placeholder.clone(), - ], - buffer_start: 0, - buffer_end: 0, + prev_span: span, + prev_token_kind: PrevTokenKind::Other, + lookahead_buffer: Default::default(), tokens_consumed: 0, restrictions: Restrictions::empty(), quote_depth: 0, @@ -409,8 +410,7 @@ impl<'a> Parser<'a> { pub fn unexpected_last<T>(&self, t: &token::Token) -> PResult<'a, T> { let token_str = Parser::token_to_string(t); - let last_span = self.last_span; - Err(self.span_fatal(last_span, &format!("unexpected token: `{}`", token_str))) + Err(self.span_fatal(self.prev_span, &format!("unexpected token: `{}`", token_str))) } pub fn unexpected<T>(&mut self) -> PResult<'a, T> { @@ -500,8 +500,8 @@ impl<'a> Parser<'a> { expr: PResult<'a, P<Expr>>) -> PResult<'a, (Span, P<Expr>)> { expr.map(|e| { - if self.last_token_interpolated { - (self.last_span, e) + if self.prev_token_kind == PrevTokenKind::Interpolated { + (self.prev_span, e) } else { (e.span, e) } @@ -520,30 +520,23 @@ impl<'a> Parser<'a> { self.bug("ident interpolation not converted to real token"); } _ => { - let last_token = self.last_token.clone().map(|t| *t); - Err(match last_token { - Some(token::DocComment(_)) => self.span_fatal_help(self.last_span, + Err(if self.prev_token_kind == PrevTokenKind::DocComment { + self.span_fatal_help(self.prev_span, "found a documentation comment that doesn't document anything", "doc comments must come before what they document, maybe a comment was \ - intended with `//`?"), - _ => { + intended with `//`?") + } else { let mut err = self.fatal(&format!("expected identifier, found `{}`", self.this_token_to_string())); if self.token == token::Underscore { err.note("`_` is a wildcard pattern, not an identifier"); } err - } - }) + }) } } } - fn parse_ident_into_path(&mut self) -> PResult<'a, ast::Path> { - let ident = self.parse_ident()?; - Ok(ast::Path::from_ident(self.last_span, ident)) - } - /// Check if the next token is `tok`, and return `true` if so. /// /// This method will automatically add `tok` to `expected_tokens` if `tok` is not @@ -808,10 +801,12 @@ impl<'a> Parser<'a> { /// Eat and discard tokens until one of `kets` is encountered. Respects token trees, /// passes through any errors encountered. Used for error recovery. pub fn eat_to_tokens(&mut self, kets: &[&token::Token]) { + let handler = self.diagnostic(); + self.parse_seq_to_before_tokens(kets, SeqSep::none(), |p| p.parse_token_tree(), - |mut e| e.cancel()); + |mut e| handler.cancel(&mut e)); } /// Parse a sequence, including the closing delimiter. The function @@ -853,7 +848,7 @@ impl<'a> Parser<'a> { Fe: FnMut(DiagnosticBuilder) { let mut first: bool = true; - let mut v = vec!(); + let mut v = vec![]; while !kets.contains(&&self.token) { match sep.sep { Some(ref t) => { @@ -923,39 +918,29 @@ impl<'a> Parser<'a> { /// Advance the parser by one token pub fn bump(&mut self) { - if self.last_token_eof { + if self.prev_token_kind == PrevTokenKind::Eof { // Bumping after EOF is a bad sign, usually an infinite loop. self.bug("attempted to bump the parser past EOF (may be stuck in a loop)"); } - if self.token == token::Eof { - self.last_token_eof = true; - } + self.prev_span = self.span; - self.last_span = self.span; - // Stash token for error recovery (sometimes; clone is not necessarily cheap). - self.last_token = if self.token.is_ident() || - self.token.is_path() || - self.token.is_doc_comment() || - self.token == token::Comma { - Some(Box::new(self.token.clone())) - } else { - None + // Record last token kind for possible error recovery. + self.prev_token_kind = match self.token { + token::DocComment(..) => PrevTokenKind::DocComment, + token::Comma => PrevTokenKind::Comma, + token::Interpolated(..) => PrevTokenKind::Interpolated, + token::Eof => PrevTokenKind::Eof, + _ => PrevTokenKind::Other, }; - self.last_token_interpolated = self.token.is_interpolated(); - let next = if self.buffer_start == self.buffer_end { + + let next = if self.lookahead_buffer.start == self.lookahead_buffer.end { self.reader.real_token() } else { // Avoid token copies with `replace`. - let buffer_start = self.buffer_start as usize; - let next_index = (buffer_start + 1) & 3; - self.buffer_start = next_index as isize; - - let placeholder = TokenAndSpan { - tok: token::Underscore, - sp: self.span, - }; - mem::replace(&mut self.buffer[buffer_start], placeholder) + let old_start = self.lookahead_buffer.start; + self.lookahead_buffer.start = (old_start + 1) % LOOKAHEAD_BUFFER_CAPACITY; + mem::replace(&mut self.lookahead_buffer.buffer[old_start], Default::default()) }; self.span = next.sp; self.token = next.tok; @@ -978,32 +963,32 @@ impl<'a> Parser<'a> { next: token::Token, lo: BytePos, hi: BytePos) { - self.last_span = mk_sp(self.span.lo, lo); - // It would be incorrect to just stash current token, but fortunately - // for tokens currently using `bump_with`, last_token will be of no - // use anyway. - self.last_token = None; - self.last_token_interpolated = false; + self.prev_span = mk_sp(self.span.lo, lo); + // It would be incorrect to record the kind of the current token, but + // fortunately for tokens currently using `bump_with`, the + // prev_token_kind will be of no use anyway. + self.prev_token_kind = PrevTokenKind::Other; self.span = mk_sp(lo, hi); self.token = next; self.expected_tokens.clear(); } - pub fn buffer_length(&mut self) -> isize { - if self.buffer_start <= self.buffer_end { - return self.buffer_end - self.buffer_start; - } - return (4 - self.buffer_start) + self.buffer_end; - } - pub fn look_ahead<R, F>(&mut self, distance: usize, f: F) -> R where + pub fn look_ahead<R, F>(&mut self, dist: usize, f: F) -> R where F: FnOnce(&token::Token) -> R, { - let dist = distance as isize; - while self.buffer_length() < dist { - self.buffer[self.buffer_end as usize] = self.reader.real_token(); - self.buffer_end = (self.buffer_end + 1) & 3; + if dist == 0 { + f(&self.token) + } else if dist < LOOKAHEAD_BUFFER_CAPACITY { + while self.lookahead_buffer.len() < dist { + self.lookahead_buffer.buffer[self.lookahead_buffer.end] = self.reader.real_token(); + self.lookahead_buffer.end = + (self.lookahead_buffer.end + 1) % LOOKAHEAD_BUFFER_CAPACITY; + } + let index = (self.lookahead_buffer.start + dist - 1) % LOOKAHEAD_BUFFER_CAPACITY; + f(&self.lookahead_buffer.buffer[index].tok) + } else { + self.bug("lookahead distance is too large"); } - f(&self.buffer[((self.buffer_start + dist - 1) & 3) as usize].tok) } pub fn fatal(&self, m: &str) -> DiagnosticBuilder<'a> { self.sess.span_diagnostic.struct_span_fatal(self.span, m) @@ -1040,6 +1025,10 @@ impl<'a> Parser<'a> { self.sess.span_diagnostic.abort_if_errors(); } + fn cancel(&self, err: &mut DiagnosticBuilder) { + self.sess.span_diagnostic.cancel(err) + } + pub fn diagnostic(&self) -> &'a errors::Handler { &self.sess.span_diagnostic } @@ -1115,14 +1104,12 @@ impl<'a> Parser<'a> { let bounds = self.parse_ty_param_bounds(BoundParsingMode::Modified)?; if !bounds.iter().any(|b| if let TraitTyParamBound(..) = *b { true } else { false }) { - let last_span = self.last_span; - self.span_err(last_span, "at least one trait must be specified"); + self.span_err(self.prev_span, "at least one trait must be specified"); } Ok(ast::TyKind::ImplTrait(bounds)) } - pub fn parse_ty_path(&mut self) -> PResult<'a, TyKind> { Ok(TyKind::Path(None, self.parse_path(PathStyle::Type)?)) } @@ -1180,7 +1167,7 @@ impl<'a> Parser<'a> { let lo = self.span.lo; let (name, node) = if self.eat_keyword(keywords::Type) { - let TyParam {ident, bounds, default, ..} = self.parse_ty_param()?; + let TyParam {ident, bounds, default, ..} = self.parse_ty_param(vec![])?; self.expect(&token::Semi)?; (ident, TraitItemKind::Type(bounds, default)) } else if self.is_const_item() { @@ -1198,100 +1185,93 @@ impl<'a> Parser<'a> { None }; (ident, TraitItemKind::Const(ty, default)) - } else if !self.token.is_any_keyword() - && self.look_ahead(1, |t| *t == token::Not) - && (self.look_ahead(2, |t| *t == token::OpenDelim(token::Paren)) - || self.look_ahead(2, |t| *t == token::OpenDelim(token::Brace))) { - // trait item macro. - // code copied from parse_macro_use_or_failure... abstraction! - let lo = self.span.lo; - let pth = self.parse_ident_into_path()?; - self.expect(&token::Not)?; + } else if self.token.is_path_start() { + // trait item macro. + // code copied from parse_macro_use_or_failure... abstraction! + let lo = self.span.lo; + let pth = self.parse_path(PathStyle::Mod)?; + self.expect(&token::Not)?; - // eat a matched-delimiter token tree: - let delim = self.expect_open_delim()?; - let tts = self.parse_seq_to_end(&token::CloseDelim(delim), - SeqSep::none(), - |pp| pp.parse_token_tree())?; - let m_ = Mac_ { path: pth, tts: tts }; - let m: ast::Mac = codemap::Spanned { node: m_, - span: mk_sp(lo, - self.last_span.hi) }; - if delim != token::Brace { - self.expect(&token::Semi)? - } - (keywords::Invalid.ident(), ast::TraitItemKind::Macro(m)) - } else { - let (constness, unsafety, abi) = match self.parse_fn_front_matter() { - Ok(cua) => cua, - Err(e) => { - loop { - match self.token { - token::Eof => break, - token::CloseDelim(token::Brace) | - token::Semi => { - self.bump(); - break; - } - token::OpenDelim(token::Brace) => { - self.parse_token_tree()?; - break; - } - _ => self.bump() + // eat a matched-delimiter token tree: + let delim = self.expect_open_delim()?; + let tts = self.parse_seq_to_end(&token::CloseDelim(delim), + SeqSep::none(), + |pp| pp.parse_token_tree())?; + if delim != token::Brace { + self.expect(&token::Semi)? + } + + let mac = spanned(lo, self.prev_span.hi, Mac_ { path: pth, tts: tts }); + (keywords::Invalid.ident(), ast::TraitItemKind::Macro(mac)) + } else { + let (constness, unsafety, abi) = match self.parse_fn_front_matter() { + Ok(cua) => cua, + Err(e) => { + loop { + match self.token { + token::Eof => break, + token::CloseDelim(token::Brace) | + token::Semi => { + self.bump(); + break; + } + token::OpenDelim(token::Brace) => { + self.parse_token_tree()?; + break; } + _ => self.bump(), } - - return Err(e); } - }; - let ident = self.parse_ident()?; - let mut generics = self.parse_generics()?; + return Err(e); + } + }; - let d = self.parse_fn_decl_with_self(|p: &mut Parser<'a>|{ - // This is somewhat dubious; We don't want to allow - // argument names to be left off if there is a - // definition... - p.parse_arg_general(false) - })?; + let ident = self.parse_ident()?; + let mut generics = self.parse_generics()?; - generics.where_clause = self.parse_where_clause()?; - let sig = ast::MethodSig { - unsafety: unsafety, - constness: constness, - decl: d, - generics: generics, - abi: abi, - }; + let d = self.parse_fn_decl_with_self(|p: &mut Parser<'a>|{ + // This is somewhat dubious; We don't want to allow + // argument names to be left off if there is a + // definition... + p.parse_arg_general(false) + })?; - let body = match self.token { - token::Semi => { - self.bump(); - debug!("parse_trait_methods(): parsing required method"); - None - } - token::OpenDelim(token::Brace) => { - debug!("parse_trait_methods(): parsing provided method"); - let (inner_attrs, body) = - self.parse_inner_attrs_and_block()?; - attrs.extend(inner_attrs.iter().cloned()); - Some(body) - } + generics.where_clause = self.parse_where_clause()?; + let sig = ast::MethodSig { + unsafety: unsafety, + constness: constness, + decl: d, + generics: generics, + abi: abi, + }; - _ => { - let token_str = self.this_token_to_string(); - return Err(self.fatal(&format!("expected `;` or `{{`, found `{}`", - token_str)[..])) - } - }; - (ident, ast::TraitItemKind::Method(sig, body)) + let body = match self.token { + token::Semi => { + self.bump(); + debug!("parse_trait_methods(): parsing required method"); + None + } + token::OpenDelim(token::Brace) => { + debug!("parse_trait_methods(): parsing provided method"); + let (inner_attrs, body) = self.parse_inner_attrs_and_block()?; + attrs.extend(inner_attrs.iter().cloned()); + Some(body) + } + _ => { + let token_str = self.this_token_to_string(); + return Err(self.fatal(&format!("expected `;` or `{{`, found `{}`", token_str))); + } }; + (ident, ast::TraitItemKind::Method(sig, body)) + }; + Ok(TraitItem { id: ast::DUMMY_NODE_ID, ident: name, attrs: attrs, node: node, - span: mk_sp(lo, self.last_span.hi), + span: mk_sp(lo, self.prev_span.hi), }) } @@ -1338,13 +1318,13 @@ impl<'a> Parser<'a> { // In type grammar, `+` is treated like a binary operator, // and hence both L and R side are required. if bounds.is_empty() { - let last_span = self.last_span; - self.span_err(last_span, + let prev_span = self.prev_span; + self.span_err(prev_span, "at least one type parameter bound \ must be specified"); } - let sp = mk_sp(lo, self.last_span.hi); + let sp = mk_sp(lo, self.prev_span.hi); let sum = ast::TyKind::ObjectSum(lhs, bounds); Ok(P(Ty {id: ast::DUMMY_NODE_ID, node: sum, span: sp})) } @@ -1394,8 +1374,8 @@ impl<'a> Parser<'a> { // Parse the `; e` in `[ i32; e ]` // where `e` is a const expression let t = match self.maybe_parse_fixed_length_of_vec()? { - None => TyKind::Vec(t), - Some(suffix) => TyKind::FixedLengthVec(t, suffix) + None => TyKind::Slice(t), + Some(suffix) => TyKind::Array(t, suffix) }; self.expect(&token::CloseDelim(token::Bracket))?; t @@ -1426,9 +1406,8 @@ impl<'a> Parser<'a> { TyKind::Path(Some(qself), path) } else if self.token.is_path_start() { let path = self.parse_path(PathStyle::Type)?; - if self.check(&token::Not) { + if self.eat(&token::Not) { // MACRO INVOCATION - self.bump(); let delim = self.expect_open_delim()?; let tts = self.parse_seq_to_end(&token::CloseDelim(delim), SeqSep::none(), @@ -1447,7 +1426,7 @@ impl<'a> Parser<'a> { return Err(self.fatal(&msg)); }; - let sp = mk_sp(lo, self.last_span.hi); + let sp = mk_sp(lo, self.prev_span.hi); Ok(P(Ty {id: ast::DUMMY_NODE_ID, node: t, span: sp})) } @@ -1465,7 +1444,7 @@ impl<'a> Parser<'a> { } else if self.eat_keyword(keywords::Const) { Mutability::Immutable } else { - let span = self.last_span; + let span = self.prev_span; self.span_err(span, "expected mut or const in raw pointer type (use \ `*mut T` or `*const T` as appropriate)"); @@ -1508,7 +1487,7 @@ impl<'a> Parser<'a> { pat } else { debug!("parse_arg_general ident_to_pat"); - let sp = self.last_span; + let sp = self.prev_span; let spanned = Spanned { span: sp, node: keywords::Invalid.ident() }; P(Pat { id: ast::DUMMY_NODE_ID, @@ -1633,7 +1612,7 @@ impl<'a> Parser<'a> { let lit = self.parse_lit_token()?; lit }; - Ok(codemap::Spanned { node: lit, span: mk_sp(lo, self.last_span.hi) }) + Ok(codemap::Spanned { node: lit, span: mk_sp(lo, self.prev_span.hi) }) } /// matches '-' lit | lit @@ -1642,11 +1621,11 @@ impl<'a> Parser<'a> { let minus_present = self.eat(&token::BinOp(token::Minus)); let lo = self.span.lo; let literal = P(self.parse_lit()?); - let hi = self.last_span.hi; + let hi = self.prev_span.hi; let expr = self.mk_expr(lo, hi, ExprKind::Lit(literal), ThinVec::new()); if minus_present { - let minus_hi = self.last_span.hi; + let minus_hi = self.prev_span.hi; let unary = self.mk_unary(UnOp::Neg, expr); Ok(self.mk_expr(minus_lo, minus_hi, unary, ThinVec::new())) } else { @@ -1681,7 +1660,7 @@ impl<'a> Parser<'a> { /// `<T as U>::F::a::<S>` pub fn parse_qualified_path(&mut self, mode: PathStyle) -> PResult<'a, (QSelf, ast::Path)> { - let span = self.last_span; + let span = self.prev_span; let self_type = self.parse_ty_sum()?; let mut path = if self.eat_keyword(keywords::As) { self.parse_path(PathStyle::Type)? @@ -1714,7 +1693,7 @@ impl<'a> Parser<'a> { }; path.segments.extend(segments); - path.span.hi = self.last_span.hi; + path.span.hi = self.prev_span.hi; Ok((qself, path)) } @@ -1752,7 +1731,7 @@ impl<'a> Parser<'a> { }; // Assemble the span. - let span = mk_sp(lo, self.last_span.hi); + let span = mk_sp(lo, self.prev_span.hi); // Assemble the result. Ok(ast::Path { @@ -1772,6 +1751,17 @@ impl<'a> Parser<'a> { // First, parse an identifier. let identifier = self.parse_path_segment_ident()?; + if self.check(&token::ModSep) && self.look_ahead(1, |t| *t == token::Lt) { + self.bump(); + let prev_span = self.prev_span; + + let mut err = self.diagnostic().struct_span_err(prev_span, + "unexpected token: `::`"); + err.help( + "use `<...>` instead of `::<...>` if you meant to specify type arguments"); + err.emit(); + } + // Parse types, optionally. let parameters = if self.eat_lt() { let (lifetimes, types, bindings) = self.parse_generic_values_after_lt()?; @@ -1782,7 +1772,7 @@ impl<'a> Parser<'a> { bindings: P::from_vec(bindings), }) } else if self.eat(&token::OpenDelim(token::Paren)) { - let lo = self.last_span.lo; + let lo = self.prev_span.lo; let inputs = self.parse_seq_to_end( &token::CloseDelim(token::Paren), @@ -1795,7 +1785,7 @@ impl<'a> Parser<'a> { None }; - let hi = self.last_span.hi; + let hi = self.prev_span.hi; ast::PathParameters::Parenthesized(ast::ParenthesizedParameterData { span: mk_sp(lo, hi), @@ -1919,10 +1909,22 @@ impl<'a> Parser<'a> { /// Parses `lifetime_defs = [ lifetime_defs { ',' lifetime_defs } ]` where `lifetime_def = /// lifetime [':' lifetimes]` - pub fn parse_lifetime_defs(&mut self) -> PResult<'a, Vec<ast::LifetimeDef>> { - + /// + /// If `followed_by_ty_params` is None, then we are in a context + /// where only lifetime parameters are allowed, and thus we should + /// error if we encounter attributes after the bound lifetimes. + /// + /// If `followed_by_ty_params` is Some(r), then there may be type + /// parameter bindings after the lifetimes, so we should pass + /// along the parsed attributes to be attached to the first such + /// type parmeter. + pub fn parse_lifetime_defs(&mut self, + followed_by_ty_params: Option<&mut Vec<ast::Attribute>>) + -> PResult<'a, Vec<ast::LifetimeDef>> + { let mut res = Vec::new(); loop { + let attrs = self.parse_outer_attributes()?; match self.token { token::Lifetime(_) => { let lifetime = self.parse_lifetime()?; @@ -1932,11 +1934,20 @@ impl<'a> Parser<'a> { } else { Vec::new() }; - res.push(ast::LifetimeDef { lifetime: lifetime, + res.push(ast::LifetimeDef { attrs: attrs.into(), + lifetime: lifetime, bounds: bounds }); } _ => { + if let Some(recv) = followed_by_ty_params { + assert!(recv.is_empty()); + *recv = attrs; + } else { + let msg = "trailing attribute after lifetime parameters"; + return Err(self.fatal(msg)); + } + debug!("parse_lifetime_defs ret {:?}", res); return Ok(res); } } @@ -2001,17 +2012,30 @@ impl<'a> Parser<'a> { } } - /// Parse ident COLON expr + /// Parse ident (COLON expr)? pub fn parse_field(&mut self) -> PResult<'a, Field> { let lo = self.span.lo; - let i = self.parse_field_name()?; - let hi = self.last_span.hi; - self.expect(&token::Colon)?; - let e = self.parse_expr()?; + let hi; + + // Check if a colon exists one ahead. This means we're parsing a fieldname. + let (fieldname, expr, is_shorthand) = if self.look_ahead(1, |t| t == &token::Colon) { + let fieldname = self.parse_field_name()?; + self.bump(); + hi = self.prev_span.hi; + (fieldname, self.parse_expr()?, false) + } else { + let fieldname = self.parse_ident()?; + hi = self.prev_span.hi; + + // Mimic `x: x` for the `x` field shorthand. + let path = ast::Path::from_ident(mk_sp(lo, hi), fieldname); + (fieldname, self.mk_expr(lo, hi, ExprKind::Path(None, path), ThinVec::new()), true) + }; Ok(ast::Field { - ident: spanned(lo, hi, i), - span: mk_sp(lo, e.span.hi), - expr: e, + ident: spanned(lo, hi, fieldname), + span: mk_sp(lo, expr.span.hi), + expr: expr, + is_shorthand: is_shorthand, }) } @@ -2160,7 +2184,7 @@ impl<'a> Parser<'a> { } self.bump(); - hi = self.last_span.hi; + hi = self.prev_span.hi; return if es.len() == 1 && !trailing_comma { Ok(self.mk_expr(lo, hi, ExprKind::Paren(es.into_iter().nth(0).unwrap()), attrs)) } else { @@ -2200,16 +2224,16 @@ impl<'a> Parser<'a> { SeqSep::trailing_allowed(token::Comma), |p| Ok(p.parse_expr()?) )?; - let mut exprs = vec!(first_expr); + let mut exprs = vec![first_expr]; exprs.extend(remaining_exprs); ex = ExprKind::Vec(exprs); } else { // Vector with one element. self.expect(&token::CloseDelim(token::Bracket))?; - ex = ExprKind::Vec(vec!(first_expr)); + ex = ExprKind::Vec(vec![first_expr]); } } - hi = self.last_span.hi; + hi = self.prev_span.hi; } _ => { if self.eat_lt() { @@ -2219,18 +2243,18 @@ impl<'a> Parser<'a> { return Ok(self.mk_expr(lo, hi, ExprKind::Path(Some(qself), path), attrs)); } if self.eat_keyword(keywords::Move) { - let lo = self.last_span.lo; + let lo = self.prev_span.lo; return self.parse_lambda_expr(lo, CaptureBy::Value, attrs); } if self.eat_keyword(keywords::If) { return self.parse_if_expr(attrs); } if self.eat_keyword(keywords::For) { - let lo = self.last_span.lo; + let lo = self.prev_span.lo; return self.parse_for_expr(None, lo, attrs); } if self.eat_keyword(keywords::While) { - let lo = self.last_span.lo; + let lo = self.prev_span.lo; return self.parse_while_expr(None, lo, attrs); } if self.token.is_lifetime() { @@ -2251,7 +2275,7 @@ impl<'a> Parser<'a> { return Err(self.fatal("expected `while`, `for`, or `loop` after a label")) } if self.eat_keyword(keywords::Loop) { - let lo = self.last_span.lo; + let lo = self.prev_span.lo; return self.parse_loop_expr(None, lo, attrs); } if self.eat_keyword(keywords::Continue) { @@ -2265,7 +2289,7 @@ impl<'a> Parser<'a> { } else { ExprKind::Continue(None) }; - let hi = self.last_span.hi; + let hi = self.prev_span.hi; return Ok(self.mk_expr(lo, hi, ex, attrs)); } if self.eat_keyword(keywords::Match) { @@ -2295,7 +2319,7 @@ impl<'a> Parser<'a> { } else { ex = ExprKind::Break(None); } - hi = self.last_span.hi; + hi = self.prev_span.hi; } else if self.token.is_keyword(keywords::Let) { // Catch this syntax error here, instead of in `check_strict_keywords`, so // that we can explicitly mention that let is not to be used as an expression @@ -2306,21 +2330,14 @@ impl<'a> Parser<'a> { let pth = self.parse_path(PathStyle::Expr)?; // `!`, as an operator, is prefix, so we know this isn't that - if self.check(&token::Not) { + if self.eat(&token::Not) { // MACRO INVOCATION expression - self.bump(); - let delim = self.expect_open_delim()?; - let tts = self.parse_seq_to_end( - &token::CloseDelim(delim), - SeqSep::none(), - |p| p.parse_token_tree())?; - let hi = self.last_span.hi; - - return Ok(self.mk_mac_expr(lo, - hi, - Mac_ { path: pth, tts: tts }, - attrs)); + let tts = self.parse_seq_to_end(&token::CloseDelim(delim), + SeqSep::none(), + |p| p.parse_token_tree())?; + let hi = self.prev_span.hi; + return Ok(self.mk_mac_expr(lo, hi, Mac_ { path: pth, tts: tts }, attrs)); } if self.check(&token::OpenDelim(token::Brace)) { // This is a struct literal, unless we're prohibited @@ -2329,51 +2346,7 @@ impl<'a> Parser<'a> { Restrictions::RESTRICTION_NO_STRUCT_LITERAL ); if !prohibited { - // It's a struct literal. - self.bump(); - let mut fields = Vec::new(); - let mut base = None; - - attrs.extend(self.parse_inner_attributes()?); - - while self.token != token::CloseDelim(token::Brace) { - if self.eat(&token::DotDot) { - match self.parse_expr() { - Ok(e) => { - base = Some(e); - } - Err(mut e) => { - e.emit(); - self.recover_stmt(); - } - } - break; - } - - match self.parse_field() { - Ok(f) => fields.push(f), - Err(mut e) => { - e.emit(); - self.recover_stmt(); - break; - } - } - - match self.expect_one_of(&[token::Comma], - &[token::CloseDelim(token::Brace)]) { - Ok(()) => {} - Err(mut e) => { - e.emit(); - self.recover_stmt(); - break; - } - } - } - - hi = self.span.hi; - self.expect(&token::CloseDelim(token::Brace))?; - ex = ExprKind::Struct(pth, fields, base); - return Ok(self.mk_expr(lo, hi, ex, attrs)); + return self.parse_struct_expr(lo, pth, attrs); } } @@ -2386,7 +2359,7 @@ impl<'a> Parser<'a> { ex = ExprKind::Lit(P(lit)); } Err(mut err) => { - err.cancel(); + self.cancel(&mut err); let msg = format!("expected expression, found {}", self.this_token_descr()); return Err(self.fatal(&msg)); @@ -2399,6 +2372,53 @@ impl<'a> Parser<'a> { return Ok(self.mk_expr(lo, hi, ex, attrs)); } + fn parse_struct_expr(&mut self, lo: BytePos, pth: ast::Path, mut attrs: ThinVec<Attribute>) + -> PResult<'a, P<Expr>> { + self.bump(); + let mut fields = Vec::new(); + let mut base = None; + + attrs.extend(self.parse_inner_attributes()?); + + while self.token != token::CloseDelim(token::Brace) { + if self.eat(&token::DotDot) { + match self.parse_expr() { + Ok(e) => { + base = Some(e); + } + Err(mut e) => { + e.emit(); + self.recover_stmt(); + } + } + break; + } + + match self.parse_field() { + Ok(f) => fields.push(f), + Err(mut e) => { + e.emit(); + self.recover_stmt(); + break; + } + } + + match self.expect_one_of(&[token::Comma], + &[token::CloseDelim(token::Brace)]) { + Ok(()) => {} + Err(mut e) => { + e.emit(); + self.recover_stmt(); + break; + } + } + } + + let hi = self.span.hi; + self.expect(&token::CloseDelim(token::Brace))?; + return Ok(self.mk_expr(lo, hi, ExprKind::Struct(pth, fields, base), attrs)); + } + fn parse_or_use_outer_attributes(&mut self, already_parsed_attrs: Option<ThinVec<Attribute>>) -> PResult<'a, ThinVec<Attribute>> { @@ -2481,8 +2501,8 @@ impl<'a> Parser<'a> { }; if !bindings.is_empty() { - let last_span = self.last_span; - self.span_err(last_span, "type bindings are only permitted on trait paths"); + let prev_span = self.prev_span; + self.span_err(prev_span, "type bindings are only permitted on trait paths"); } Ok(match self.token { @@ -2494,7 +2514,7 @@ impl<'a> Parser<'a> { SeqSep::trailing_allowed(token::Comma), |p| Ok(p.parse_expr()?) )?; - let hi = self.last_span.hi; + let hi = self.prev_span.hi; es.insert(0, self_value); let id = spanned(ident_span.lo, ident_span.hi, ident); @@ -2504,8 +2524,8 @@ impl<'a> Parser<'a> { // Field access. _ => { if !tys.is_empty() { - let last_span = self.last_span; - self.span_err(last_span, + let prev_span = self.prev_span; + self.span_err(prev_span, "field expressions may not \ have type parameters"); } @@ -2523,7 +2543,7 @@ impl<'a> Parser<'a> { loop { // expr? while self.eat(&token::Question) { - let hi = self.last_span.hi; + let hi = self.prev_span.hi; e = self.mk_expr(lo, hi, ExprKind::Try(e), ThinVec::new()); } @@ -2531,7 +2551,7 @@ impl<'a> Parser<'a> { if self.eat(&token::Dot) { match self.token { token::Ident(i) => { - let dot_pos = self.last_span.hi; + let dot_pos = self.prev_span.hi; hi = self.span.hi; self.bump(); @@ -2543,7 +2563,7 @@ impl<'a> Parser<'a> { // A tuple index may not have a suffix self.expect_no_suffix(sp, "tuple index", suf); - let dot = self.last_span.hi; + let dot = self.prev_span.hi; hi = self.span.hi; self.bump(); @@ -2555,16 +2575,16 @@ impl<'a> Parser<'a> { e = self.mk_expr(lo, hi, field, ThinVec::new()); } None => { - let last_span = self.last_span; - self.span_err(last_span, "invalid tuple or tuple struct index"); + let prev_span = self.prev_span; + self.span_err(prev_span, "invalid tuple or tuple struct index"); } } } token::Literal(token::Float(n), _suf) => { self.bump(); - let last_span = self.last_span; + let prev_span = self.prev_span; let fstr = n.as_str(); - let mut err = self.diagnostic().struct_span_err(last_span, + let mut err = self.diagnostic().struct_span_err(prev_span, &format!("unexpected token: `{}`", n.as_str())); if fstr.chars().all(|x| "0123456789.".contains(x)) { let float = match fstr.parse::<f64>().ok() { @@ -2583,7 +2603,7 @@ impl<'a> Parser<'a> { let actual = self.this_token_to_string(); self.span_err(self.span, &format!("unexpected token: `{}`", actual)); - let dot_pos = self.last_span.hi; + let dot_pos = self.prev_span.hi; e = self.parse_dot_suffix(keywords::Invalid.ident(), mk_sp(dot_pos, dot_pos), e, lo)?; @@ -2601,7 +2621,7 @@ impl<'a> Parser<'a> { SeqSep::trailing_allowed(token::Comma), |p| Ok(p.parse_expr()?) )?; - hi = self.last_span.hi; + hi = self.prev_span.hi; let nd = self.mk_call(e, es); e = self.mk_expr(lo, hi, nd, ThinVec::new()); @@ -2647,8 +2667,12 @@ impl<'a> Parser<'a> { num_captures: name_num }))); } else if self.token.is_keyword(keywords::Crate) { + let ident = match self.token { + token::Ident(id) => ast::Ident { name: token::intern("$crate"), ..id }, + _ => unreachable!(), + }; self.bump(); - return Ok(TokenTree::Token(sp, SpecialVarNt(SpecialMacroVar::CrateMacroVar))); + return Ok(TokenTree::Token(sp, token::Ident(ident))); } else { sp = mk_sp(sp.lo, self.span.hi); self.parse_ident().unwrap_or_else(|mut e| { @@ -2944,8 +2968,8 @@ impl<'a> Parser<'a> { self.expected_tokens.push(TokenType::Operator); while let Some(op) = AssocOp::from_token(&self.token) { - let lhs_span = if self.last_token_interpolated { - self.last_span + let lhs_span = if self.prev_token_kind == PrevTokenKind::Interpolated { + self.prev_span } else { lhs.span }; @@ -3138,7 +3162,7 @@ impl<'a> Parser<'a> { if self.check_keyword(keywords::Let) { return self.parse_if_let_expr(attrs); } - let lo = self.last_span.lo; + let lo = self.prev_span.lo; let cond = self.parse_expr_res(Restrictions::RESTRICTION_NO_STRUCT_LITERAL, None)?; let thn = self.parse_block()?; let mut els: Option<P<Expr>> = None; @@ -3154,7 +3178,7 @@ impl<'a> Parser<'a> { /// Parse an 'if let' expression ('if' token already eaten) pub fn parse_if_let_expr(&mut self, attrs: ThinVec<Attribute>) -> PResult<'a, P<Expr>> { - let lo = self.last_span.lo; + let lo = self.prev_span.lo; self.expect_keyword(keywords::Let)?; let pat = self.parse_pat()?; self.expect(&token::Eq)?; @@ -3177,7 +3201,7 @@ impl<'a> Parser<'a> { -> PResult<'a, P<Expr>> { let decl = self.parse_fn_block_decl()?; - let decl_hi = self.last_span.hi; + let decl_hi = self.prev_span.hi; let body = match decl.output { FunctionRetTy::Default(_) => { // If no explicit return type is given, parse any @@ -3230,7 +3254,7 @@ impl<'a> Parser<'a> { let (iattrs, loop_block) = self.parse_inner_attrs_and_block()?; attrs.extend(iattrs); - let hi = self.last_span.hi; + let hi = self.prev_span.hi; Ok(self.mk_expr(span_lo, hi, ExprKind::ForLoop(pat, expr, loop_block, opt_ident), @@ -3278,8 +3302,8 @@ impl<'a> Parser<'a> { // `match` token already eaten fn parse_match_expr(&mut self, mut attrs: ThinVec<Attribute>) -> PResult<'a, P<Expr>> { - let match_span = self.last_span; - let lo = self.last_span.lo; + let match_span = self.prev_span; + let lo = self.prev_span.lo; let discriminant = self.parse_expr_res(Restrictions::RESTRICTION_NO_STRUCT_LITERAL, None)?; if let Err(mut e) = self.expect(&token::OpenDelim(token::Brace)) { @@ -3401,7 +3425,7 @@ impl<'a> Parser<'a> { } } else if ddpos.is_some() && self.eat(&token::DotDot) { // Emit a friendly error, ignore `..` and continue parsing - self.span_err(self.last_span, "`..` can only be used once per \ + self.span_err(self.prev_span, "`..` can only be used once per \ tuple or tuple struct pattern"); } else { fields.push(self.parse_pat()?); @@ -3512,7 +3536,7 @@ impl<'a> Parser<'a> { let is_ref = self.eat_keyword(keywords::Ref); let is_mut = self.eat_keyword(keywords::Mut); let fieldname = self.parse_ident()?; - hi = self.last_span.hi; + hi = self.prev_span.hi; let bind_type = match (is_ref, is_mut) { (true, true) => BindingMode::ByRef(Mutability::Mutable), @@ -3520,7 +3544,7 @@ impl<'a> Parser<'a> { (false, true) => BindingMode::ByValue(Mutability::Mutable), (false, false) => BindingMode::ByValue(Mutability::Immutable), }; - let fieldpath = codemap::Spanned{span:self.last_span, node:fieldname}; + let fieldpath = codemap::Spanned{span:self.prev_span, node:fieldname}; let fieldpat = P(ast::Pat{ id: ast::DUMMY_NODE_ID, node: PatKind::Ident(bind_type, fieldpath, None), @@ -3559,7 +3583,7 @@ impl<'a> Parser<'a> { // Parse an unqualified path (None, self.parse_path(PathStyle::Expr)?) }; - let hi = self.last_span.hi; + let hi = self.prev_span.hi; Ok(self.mk_expr(lo, hi, ExprKind::Path(qself, path), ThinVec::new())) } else { self.parse_pat_literal_maybe_minus() @@ -3573,39 +3597,37 @@ impl<'a> Parser<'a> { let lo = self.span.lo; let pat; match self.token { - token::Underscore => { - // Parse _ - self.bump(); - pat = PatKind::Wild; - } - token::BinOp(token::And) | token::AndAnd => { - // Parse &pat / &mut pat - self.expect_and()?; - let mutbl = self.parse_mutability()?; - if let token::Lifetime(ident) = self.token { - return Err(self.fatal(&format!("unexpected lifetime `{}` in pattern", ident))); + token::Underscore => { + // Parse _ + self.bump(); + pat = PatKind::Wild; + } + token::BinOp(token::And) | token::AndAnd => { + // Parse &pat / &mut pat + self.expect_and()?; + let mutbl = self.parse_mutability()?; + if let token::Lifetime(ident) = self.token { + return Err(self.fatal(&format!("unexpected lifetime `{}` in pattern", ident))); + } + let subpat = self.parse_pat()?; + pat = PatKind::Ref(subpat, mutbl); + } + token::OpenDelim(token::Paren) => { + // Parse (pat,pat,pat,...) as tuple pattern + self.bump(); + let (fields, ddpos) = self.parse_pat_tuple_elements(true)?; + self.expect(&token::CloseDelim(token::Paren))?; + pat = PatKind::Tuple(fields, ddpos); + } + token::OpenDelim(token::Bracket) => { + // Parse [pat,pat,...] as slice pattern + self.bump(); + let (before, slice, after) = self.parse_pat_vec_elements()?; + self.expect(&token::CloseDelim(token::Bracket))?; + pat = PatKind::Slice(before, slice, after); } - - let subpat = self.parse_pat()?; - pat = PatKind::Ref(subpat, mutbl); - } - token::OpenDelim(token::Paren) => { - // Parse (pat,pat,pat,...) as tuple pattern - self.bump(); - let (fields, ddpos) = self.parse_pat_tuple_elements(true)?; - self.expect(&token::CloseDelim(token::Paren))?; - pat = PatKind::Tuple(fields, ddpos); - } - token::OpenDelim(token::Bracket) => { - // Parse [pat,pat,...] as slice pattern - self.bump(); - let (before, slice, after) = self.parse_pat_vec_elements()?; - self.expect(&token::CloseDelim(token::Bracket))?; - pat = PatKind::Vec(before, slice, after); - } - _ => { // At this point, token != _, &, &&, (, [ - if self.eat_keyword(keywords::Mut) { + _ => if self.eat_keyword(keywords::Mut) { // Parse mut ident @ pat pat = self.parse_pat_ident(BindingMode::ByValue(Mutability::Mutable))?; } else if self.eat_keyword(keywords::Ref) { @@ -3616,53 +3638,49 @@ impl<'a> Parser<'a> { // Parse box pat let subpat = self.parse_pat()?; pat = PatKind::Box(subpat); + } else if self.token.is_ident() && !self.token.is_any_keyword() && + self.look_ahead(1, |t| match *t { + token::OpenDelim(token::Paren) | token::OpenDelim(token::Brace) | + token::DotDotDot | token::ModSep | token::Not => false, + _ => true, + }) { + // Parse ident @ pat + // This can give false positives and parse nullary enums, + // they are dealt with later in resolve + let binding_mode = BindingMode::ByValue(Mutability::Immutable); + pat = self.parse_pat_ident(binding_mode)?; } else if self.token.is_path_start() { // Parse pattern starting with a path - if self.token.is_ident() && self.look_ahead(1, |t| *t != token::DotDotDot && - *t != token::OpenDelim(token::Brace) && - *t != token::OpenDelim(token::Paren) && - *t != token::ModSep) { - // Plain idents have some extra abilities here compared to general paths - if self.look_ahead(1, |t| *t == token::Not) { + let (qself, path) = if self.eat_lt() { + // Parse a qualified path + let (qself, path) = self.parse_qualified_path(PathStyle::Expr)?; + (Some(qself), path) + } else { + // Parse an unqualified path + (None, self.parse_path(PathStyle::Expr)?) + }; + match self.token { + token::Not if qself.is_none() => { // Parse macro invocation - let path = self.parse_ident_into_path()?; self.bump(); let delim = self.expect_open_delim()?; - let tts = self.parse_seq_to_end( - &token::CloseDelim(delim), - SeqSep::none(), |p| p.parse_token_tree())?; - let mac = Mac_ { path: path, tts: tts }; - pat = PatKind::Mac(codemap::Spanned {node: mac, - span: mk_sp(lo, self.last_span.hi)}); - } else { - // Parse ident @ pat - // This can give false positives and parse nullary enums, - // they are dealt with later in resolve - let binding_mode = BindingMode::ByValue(Mutability::Immutable); - pat = self.parse_pat_ident(binding_mode)?; + let tts = self.parse_seq_to_end(&token::CloseDelim(delim), + SeqSep::none(), + |p| p.parse_token_tree())?; + let mac = spanned(lo, self.prev_span.hi, Mac_ { path: path, tts: tts }); + pat = PatKind::Mac(mac); } - } else { - let (qself, path) = if self.eat_lt() { - // Parse a qualified path - let (qself, path) = - self.parse_qualified_path(PathStyle::Expr)?; - (Some(qself), path) - } else { - // Parse an unqualified path - (None, self.parse_path(PathStyle::Expr)?) - }; - match self.token { - token::DotDotDot => { + token::DotDotDot => { // Parse range - let hi = self.last_span.hi; + let hi = self.prev_span.hi; let begin = self.mk_expr(lo, hi, ExprKind::Path(qself, path), ThinVec::new()); self.bump(); let end = self.parse_pat_range_end()?; pat = PatKind::Range(begin, end); - } - token::OpenDelim(token::Brace) => { - if qself.is_some() { + } + token::OpenDelim(token::Brace) => { + if qself.is_some() { return Err(self.fatal("unexpected `{` after qualified path")); } // Parse struct pattern @@ -3674,8 +3692,8 @@ impl<'a> Parser<'a> { }); self.bump(); pat = PatKind::Struct(path, fields, etc); - } - token::OpenDelim(token::Paren) => { + } + token::OpenDelim(token::Paren) => { if qself.is_some() { return Err(self.fatal("unexpected `(` after qualified path")); } @@ -3684,11 +3702,8 @@ impl<'a> Parser<'a> { let (fields, ddpos) = self.parse_pat_tuple_elements(false)?; self.expect(&token::CloseDelim(token::Paren))?; pat = PatKind::TupleStruct(path, fields, ddpos) - } - _ => { - pat = PatKind::Path(qself, path); - } } + _ => pat = PatKind::Path(qself, path), } } else { // Try to parse everything else as literal with optional minus @@ -3702,16 +3717,15 @@ impl<'a> Parser<'a> { } } Err(mut err) => { - err.cancel(); + self.cancel(&mut err); let msg = format!("expected pattern, found {}", self.this_token_descr()); return Err(self.fatal(&msg)); } } } - } } - let hi = self.last_span.hi; + let hi = self.prev_span.hi; Ok(P(ast::Pat { id: ast::DUMMY_NODE_ID, node: pat, @@ -3726,8 +3740,8 @@ impl<'a> Parser<'a> { binding_mode: ast::BindingMode) -> PResult<'a, PatKind> { let ident = self.parse_ident()?; - let last_span = self.last_span; - let name = codemap::Spanned{span: last_span, node: ident}; + let prev_span = self.prev_span; + let name = codemap::Spanned{span: prev_span, node: ident}; let sub = if self.eat(&token::At) { Some(self.parse_pat()?) } else { @@ -3741,9 +3755,8 @@ impl<'a> Parser<'a> { // binding mode then we do not end up here, because the lookahead // will direct us over to parse_enum_variant() if self.token == token::OpenDelim(token::Paren) { - let last_span = self.last_span; return Err(self.span_fatal( - last_span, + self.prev_span, "expected identifier, found enum pattern")) } @@ -3765,7 +3778,7 @@ impl<'a> Parser<'a> { pat: pat, init: init, id: ast::DUMMY_NODE_ID, - span: mk_sp(lo, self.last_span.hi), + span: mk_sp(lo, self.prev_span.hi), attrs: attrs, })) } @@ -3780,7 +3793,7 @@ impl<'a> Parser<'a> { self.expect(&token::Colon)?; let ty = self.parse_ty_sum()?; Ok(StructField { - span: mk_sp(lo, self.last_span.hi), + span: mk_sp(lo, self.prev_span.hi), ident: Some(name), vis: vis, id: ast::DUMMY_NODE_ID, @@ -3798,7 +3811,7 @@ impl<'a> Parser<'a> { _ => "expected item after attributes", }; - self.span_err(self.last_span, message); + self.span_err(self.prev_span, message); } /// Parse a statement. This stops just before trailing semicolons on everything but items. @@ -3868,15 +3881,22 @@ impl<'a> Parser<'a> { } } - fn parse_stmt_(&mut self, macro_expanded: bool) -> Option<Stmt> { - self.parse_stmt_without_recovery(macro_expanded).unwrap_or_else(|mut e| { + fn parse_stmt_(&mut self, macro_legacy_warnings: bool) -> Option<Stmt> { + self.parse_stmt_without_recovery(macro_legacy_warnings).unwrap_or_else(|mut e| { e.emit(); self.recover_stmt_(SemiColonMode::Break); None }) } - fn parse_stmt_without_recovery(&mut self, macro_expanded: bool) -> PResult<'a, Option<Stmt>> { + fn is_union_item(&mut self) -> bool { + self.token.is_keyword(keywords::Union) && + self.look_ahead(1, |t| t.is_ident() && !t.is_any_keyword()) + } + + fn parse_stmt_without_recovery(&mut self, + macro_legacy_warnings: bool) + -> PResult<'a, Option<Stmt>> { maybe_whole!(Some deref self, NtStmt); let attrs = self.parse_outer_attributes()?; @@ -3886,18 +3906,35 @@ impl<'a> Parser<'a> { Stmt { id: ast::DUMMY_NODE_ID, node: StmtKind::Local(self.parse_local(attrs.into())?), - span: mk_sp(lo, self.last_span.hi), + span: mk_sp(lo, self.prev_span.hi), } - } else if self.token.is_ident() - && !self.token.is_any_keyword() - && self.look_ahead(1, |t| *t == token::Not) { - // it's a macro invocation: + // Starts like a simple path, but not a union item. + } else if self.token.is_path_start() && + !self.token.is_qpath_start() && + !self.is_union_item() { + let pth = self.parse_path(PathStyle::Expr)?; - // Potential trouble: if we allow macros with paths instead of - // idents, we'd need to look ahead past the whole path here... - let pth = self.parse_ident_into_path()?; - self.bump(); + if !self.eat(&token::Not) { + let expr = if self.check(&token::OpenDelim(token::Brace)) { + self.parse_struct_expr(lo, pth, ThinVec::new())? + } else { + let hi = self.prev_span.hi; + self.mk_expr(lo, hi, ExprKind::Path(None, pth), ThinVec::new()) + }; + let expr = self.with_res(Restrictions::RESTRICTION_STMT_EXPR, |this| { + let expr = this.parse_dot_or_call_expr_with(expr, lo, attrs.into())?; + this.parse_assoc_expr_with(0, LhsExpr::AlreadyParsed(expr)) + })?; + + return Ok(Some(Stmt { + id: ast::DUMMY_NODE_ID, + node: StmtKind::Expr(expr), + span: mk_sp(lo, self.prev_span.hi), + })); + } + + // it's a macro invocation let id = match self.token { token::OpenDelim(_) => keywords::Invalid.ident(), // no special identifier _ => self.parse_ident()?, @@ -3929,7 +3966,7 @@ impl<'a> Parser<'a> { SeqSep::none(), |p| p.parse_token_tree() )?; - let hi = self.last_span.hi; + let hi = self.prev_span.hi; let style = if delim == token::Brace { MacStmtStyle::Braces @@ -3946,7 +3983,7 @@ impl<'a> Parser<'a> { // We used to incorrectly stop parsing macro-expanded statements here. // If the next token will be an error anyway but could have parsed with the // earlier behavior, stop parsing here and emit a warning to avoid breakage. - else if macro_expanded && self.token.can_begin_expr() && match self.token { + else if macro_legacy_warnings && self.token.can_begin_expr() && match self.token { // These can continue an expression, so we can't stop parsing and warn. token::OpenDelim(token::Paren) | token::OpenDelim(token::Bracket) | token::BinOp(token::Minus) | token::BinOp(token::Star) | @@ -3974,8 +4011,7 @@ impl<'a> Parser<'a> { // Require a semicolon or braces. if style != MacStmtStyle::Braces { if !self.eat(&token::Semi) { - let last_span = self.last_span; - self.span_err(last_span, + self.span_err(self.prev_span, "macros that expand to items must \ either be surrounded with braces or \ followed by a semicolon"); @@ -4006,13 +4042,13 @@ impl<'a> Parser<'a> { None => { let unused_attrs = |attrs: &[_], s: &mut Self| { if attrs.len() > 0 { - let last_token = s.last_token.clone().map(|t| *t); - match last_token { - Some(token::DocComment(_)) => s.span_err_help(s.last_span, + if s.prev_token_kind == PrevTokenKind::DocComment { + s.span_err_help(s.prev_span, "found a documentation comment that doesn't document anything", "doc comments must come before what they document, maybe a \ - comment was intended with `//`?"), - _ => s.span_err(s.span, "expected statement after outer attribute"), + comment was intended with `//`?"); + } else { + s.span_err(s.span, "expected statement after outer attribute"); } } }; @@ -4070,13 +4106,13 @@ impl<'a> Parser<'a> { let mut stmt_span = stmt.span; // expand the span to include the semicolon, if it exists if self.eat(&token::Semi) { - stmt_span.hi = self.last_span.hi; + stmt_span.hi = self.prev_span.hi; } e.span_help(stmt_span, "try placing this code inside a block"); } Err(mut e) => { self.recover_stmt_(SemiColonMode::Break); - e.cancel(); + self.cancel(&mut e); } _ => () } @@ -4116,13 +4152,13 @@ impl<'a> Parser<'a> { stmts: stmts, id: ast::DUMMY_NODE_ID, rules: s, - span: mk_sp(lo, self.last_span.hi), + span: mk_sp(lo, self.prev_span.hi), })) } /// Parse a statement, including the trailing semicolon. - pub fn parse_full_stmt(&mut self, macro_expanded: bool) -> PResult<'a, Option<Stmt>> { - let mut stmt = match self.parse_stmt_(macro_expanded) { + pub fn parse_full_stmt(&mut self, macro_legacy_warnings: bool) -> PResult<'a, Option<Stmt>> { + let mut stmt = match self.parse_stmt_(macro_legacy_warnings) { Some(stmt) => stmt, None => return Ok(None), }; @@ -4142,7 +4178,7 @@ impl<'a> Parser<'a> { } StmtKind::Local(..) => { // We used to incorrectly allow a macro-expanded let statement to lack a semicolon. - if macro_expanded && self.token != token::Semi { + if macro_legacy_warnings && self.token != token::Semi { self.warn_missing_semicolon(); } else { self.expect_one_of(&[token::Semi], &[])?; @@ -4155,7 +4191,7 @@ impl<'a> Parser<'a> { stmt = stmt.add_trailing_semicolon(); } - stmt.span.hi = self.last_span.hi; + stmt.span.hi = self.prev_span.hi; Ok(Some(stmt)) } @@ -4188,7 +4224,7 @@ impl<'a> Parser<'a> { mode: BoundParsingMode) -> PResult<'a, TyParamBounds> { - let mut result = vec!(); + let mut result = vec![]; loop { let question_span = self.span; let ate_question = self.eat(&token::Question); @@ -4232,7 +4268,7 @@ impl<'a> Parser<'a> { } /// Matches typaram = IDENT (`?` unbound)? optbounds ( EQ ty )? - fn parse_ty_param(&mut self) -> PResult<'a, TyParam> { + fn parse_ty_param(&mut self, preceding_attrs: Vec<ast::Attribute>) -> PResult<'a, TyParam> { let span = self.span; let ident = self.parse_ident()?; @@ -4246,6 +4282,7 @@ impl<'a> Parser<'a> { }; Ok(TyParam { + attrs: preceding_attrs.into(), ident: ident, id: ast::DUMMY_NODE_ID, bounds: bounds, @@ -4266,20 +4303,42 @@ impl<'a> Parser<'a> { let span_lo = self.span.lo; if self.eat(&token::Lt) { - let lifetime_defs = self.parse_lifetime_defs()?; + // Upon encountering attribute in generics list, we do not + // know if it is attached to lifetime or to type param. + // + // Solution: 1. eagerly parse attributes in tandem with + // lifetime defs, 2. store last set of parsed (and unused) + // attributes in `attrs`, and 3. pass in those attributes + // when parsing formal type param after lifetime defs. + let mut attrs = vec![]; + let lifetime_defs = self.parse_lifetime_defs(Some(&mut attrs))?; let mut seen_default = false; + let mut post_lifetime_attrs = Some(attrs); let ty_params = self.parse_seq_to_gt(Some(token::Comma), |p| { p.forbid_lifetime()?; - let ty_param = p.parse_ty_param()?; + // Move out of `post_lifetime_attrs` if present. O/w + // not first type param: parse attributes anew. + let attrs = match post_lifetime_attrs.as_mut() { + None => p.parse_outer_attributes()?, + Some(attrs) => mem::replace(attrs, vec![]), + }; + post_lifetime_attrs = None; + let ty_param = p.parse_ty_param(attrs)?; if ty_param.default.is_some() { seen_default = true; } else if seen_default { - let last_span = p.last_span; - p.span_err(last_span, + let prev_span = p.prev_span; + p.span_err(prev_span, "type parameters with a default must be trailing"); } Ok(ty_param) })?; + if let Some(attrs) = post_lifetime_attrs { + if !attrs.is_empty() { + self.span_err(attrs[0].span, + "trailing attribute after lifetime parameters"); + } + } Ok(ast::Generics { lifetimes: lifetime_defs, ty_params: ty_params, @@ -4287,7 +4346,7 @@ impl<'a> Parser<'a> { id: ast::DUMMY_NODE_ID, predicates: Vec::new(), }, - span: mk_sp(span_lo, self.last_span.hi), + span: mk_sp(span_lo, self.prev_span.hi), }) } else { Ok(ast::Generics::default()) @@ -4302,9 +4361,7 @@ impl<'a> Parser<'a> { let missing_comma = !lifetimes.is_empty() && !self.token.is_like_gt() && - self.last_token - .as_ref().map_or(true, - |x| &**x != &token::Comma); + self.prev_token_kind != PrevTokenKind::Comma; if missing_comma { @@ -4317,7 +4374,7 @@ impl<'a> Parser<'a> { let span_hi = match self.parse_ty() { Ok(..) => self.span.hi, Err(ref mut err) => { - err.cancel(); + self.cancel(err); span_hi } }; @@ -4411,7 +4468,7 @@ impl<'a> Parser<'a> { let bounds = self.parse_lifetimes(token::BinOp(token::Plus))?; - let hi = self.last_span.hi; + let hi = self.prev_span.hi; let span = mk_sp(lo, hi); where_clause.predicates.push(ast::WherePredicate::RegionPredicate( @@ -4429,7 +4486,7 @@ impl<'a> Parser<'a> { let bound_lifetimes = if self.eat_keyword(keywords::For) { // Higher ranked constraint. self.expect(&token::Lt)?; - let lifetime_defs = self.parse_lifetime_defs()?; + let lifetime_defs = self.parse_lifetime_defs(None)?; self.expect_gt()?; lifetime_defs } else { @@ -4440,7 +4497,7 @@ impl<'a> Parser<'a> { if self.eat(&token::Colon) { let bounds = self.parse_ty_param_bounds(BoundParsingMode::Bare)?; - let hi = self.last_span.hi; + let hi = self.prev_span.hi; let span = mk_sp(lo, hi); if bounds.is_empty() { @@ -4460,7 +4517,7 @@ impl<'a> Parser<'a> { parsed_something = true; } else if self.eat(&token::Eq) { // let ty = try!(self.parse_ty()); - let hi = self.last_span.hi; + let hi = self.prev_span.hi; let span = mk_sp(lo, hi); // where_clause.predicates.push( // ast::WherePredicate::EqPredicate(ast::WhereEqPredicate { @@ -4475,8 +4532,8 @@ impl<'a> Parser<'a> { "equality constraints are not yet supported \ in where clauses (#20041)"); } else { - let last_span = self.last_span; - self.span_err(last_span, + let prev_span = self.prev_span; + self.span_err(prev_span, "unexpected token in `where` clause"); } } @@ -4488,8 +4545,8 @@ impl<'a> Parser<'a> { } if !parsed_something { - let last_span = self.last_span; - self.span_err(last_span, + let prev_span = self.prev_span; + self.span_err(prev_span, "a `where` clause must have at least one predicate \ in it"); } @@ -4562,9 +4619,13 @@ impl<'a> Parser<'a> { fn parse_self_arg(&mut self) -> PResult<'a, Option<Arg>> { let expect_ident = |this: &mut Self| match this.token { // Preserve hygienic context. - token::Ident(ident) => { this.bump(); codemap::respan(this.last_span, ident) } + token::Ident(ident) => { this.bump(); codemap::respan(this.prev_span, ident) } _ => unreachable!() }; + let isolated_self = |this: &mut Self, n| { + this.look_ahead(n, |t| t.is_keyword(keywords::SelfValue)) && + this.look_ahead(n + 1, |t| t != &token::ModSep) + }; // Parse optional self parameter of a method. // Only a limited set of initial token sequences is considered self parameters, anything @@ -4577,22 +4638,22 @@ impl<'a> Parser<'a> { // &'lt self // &'lt mut self // ¬_self - if self.look_ahead(1, |t| t.is_keyword(keywords::SelfValue)) { + if isolated_self(self, 1) { self.bump(); (SelfKind::Region(None, Mutability::Immutable), expect_ident(self)) } else if self.look_ahead(1, |t| t.is_keyword(keywords::Mut)) && - self.look_ahead(2, |t| t.is_keyword(keywords::SelfValue)) { + isolated_self(self, 2) { self.bump(); self.bump(); (SelfKind::Region(None, Mutability::Mutable), expect_ident(self)) } else if self.look_ahead(1, |t| t.is_lifetime()) && - self.look_ahead(2, |t| t.is_keyword(keywords::SelfValue)) { + isolated_self(self, 2) { self.bump(); let lt = self.parse_lifetime()?; (SelfKind::Region(Some(lt), Mutability::Immutable), expect_ident(self)) } else if self.look_ahead(1, |t| t.is_lifetime()) && self.look_ahead(2, |t| t.is_keyword(keywords::Mut)) && - self.look_ahead(3, |t| t.is_keyword(keywords::SelfValue)) { + isolated_self(self, 3) { self.bump(); let lt = self.parse_lifetime()?; self.bump(); @@ -4607,12 +4668,12 @@ impl<'a> Parser<'a> { // *mut self // *not_self // Emit special error for `self` cases. - if self.look_ahead(1, |t| t.is_keyword(keywords::SelfValue)) { + if isolated_self(self, 1) { self.bump(); self.span_err(self.span, "cannot pass `self` by raw pointer"); (SelfKind::Value(Mutability::Immutable), expect_ident(self)) } else if self.look_ahead(1, |t| t.is_mutability()) && - self.look_ahead(2, |t| t.is_keyword(keywords::SelfValue)) { + isolated_self(self, 2) { self.bump(); self.bump(); self.span_err(self.span, "cannot pass `self` by raw pointer"); @@ -4622,7 +4683,7 @@ impl<'a> Parser<'a> { } } token::Ident(..) => { - if self.token.is_keyword(keywords::SelfValue) { + if isolated_self(self, 0) { // self // self: TYPE let eself_ident = expect_ident(self); @@ -4633,7 +4694,7 @@ impl<'a> Parser<'a> { (SelfKind::Value(Mutability::Immutable), eself_ident) } } else if self.token.is_keyword(keywords::Mut) && - self.look_ahead(1, |t| t.is_keyword(keywords::SelfValue)) { + isolated_self(self, 1) { // mut self // mut self: TYPE self.bump(); @@ -4651,7 +4712,7 @@ impl<'a> Parser<'a> { _ => return Ok(None), }; - let eself = codemap::respan(mk_sp(eself_lo, self.last_span.hi), eself); + let eself = codemap::respan(mk_sp(eself_lo, self.prev_span.hi), eself); Ok(Some(Arg::from_self(eself, eself_ident))) } @@ -4769,7 +4830,7 @@ impl<'a> Parser<'a> { ast::Unsafety, abi::Abi)> { let is_const_fn = self.eat_keyword(keywords::Const); - let const_span = self.last_span; + let const_span = self.prev_span; let unsafety = self.parse_unsafety()?; let (constness, unsafety, abi) = if is_const_fn { (respan(const_span, Constness::Const), unsafety, Abi::Rust) @@ -4779,7 +4840,7 @@ impl<'a> Parser<'a> { } else { Abi::Rust }; - (respan(self.last_span, Constness::NotConst), unsafety, abi) + (respan(self.prev_span, Constness::NotConst), unsafety, abi) }; self.expect_keyword(keywords::Fn)?; Ok((constness, unsafety, abi)) @@ -4816,7 +4877,7 @@ impl<'a> Parser<'a> { Ok(ImplItem { id: ast::DUMMY_NODE_ID, - span: mk_sp(lo, self.last_span.hi), + span: mk_sp(lo, self.prev_span.hi), ident: name, vis: vis, defaultness: defaultness, @@ -4853,17 +4914,14 @@ impl<'a> Parser<'a> { fn parse_impl_method(&mut self, vis: &Visibility) -> PResult<'a, (Ident, Vec<ast::Attribute>, ast::ImplItemKind)> { // code copied from parse_macro_use_or_failure... abstraction! - if !self.token.is_any_keyword() - && self.look_ahead(1, |t| *t == token::Not) - && (self.look_ahead(2, |t| *t == token::OpenDelim(token::Paren)) - || self.look_ahead(2, |t| *t == token::OpenDelim(token::Brace))) { + if self.token.is_path_start() { // method macro. - let last_span = self.last_span; - self.complain_if_pub_macro(&vis, last_span); + let prev_span = self.prev_span; + self.complain_if_pub_macro(&vis, prev_span); let lo = self.span.lo; - let pth = self.parse_ident_into_path()?; + let pth = self.parse_path(PathStyle::Mod)?; self.expect(&token::Not)?; // eat a matched-delimiter token tree: @@ -4871,14 +4929,12 @@ impl<'a> Parser<'a> { let tts = self.parse_seq_to_end(&token::CloseDelim(delim), SeqSep::none(), |p| p.parse_token_tree())?; - let m_ = Mac_ { path: pth, tts: tts }; - let m: ast::Mac = codemap::Spanned { node: m_, - span: mk_sp(lo, - self.last_span.hi) }; if delim != token::Brace { self.expect(&token::Semi)? } - Ok((keywords::Invalid.ident(), vec![], ast::ImplItemKind::Macro(m))) + + let mac = spanned(lo, self.prev_span.hi, Mac_ { path: pth, tts: tts }); + Ok((keywords::Invalid.ident(), vec![], ast::ImplItemKind::Macro(mac))) } else { let (constness, unsafety, abi) = self.parse_fn_front_matter()?; let ident = self.parse_ident()?; @@ -5002,7 +5058,7 @@ impl<'a> Parser<'a> { fn parse_late_bound_lifetime_defs(&mut self) -> PResult<'a, Vec<ast::LifetimeDef>> { if self.eat_keyword(keywords::For) { self.expect(&token::Lt)?; - let lifetime_defs = self.parse_lifetime_defs()?; + let lifetime_defs = self.parse_lifetime_defs(None)?; self.expect_gt()?; Ok(lifetime_defs) } else { @@ -5018,7 +5074,7 @@ impl<'a> Parser<'a> { Ok(ast::PolyTraitRef { bound_lifetimes: lifetime_defs, trait_ref: self.parse_trait_ref()?, - span: mk_sp(lo, self.last_span.hi), + span: mk_sp(lo, self.prev_span.hi), }) } @@ -5094,7 +5150,11 @@ impl<'a> Parser<'a> { let mut fields = Vec::new(); if self.eat(&token::OpenDelim(token::Brace)) { while self.token != token::CloseDelim(token::Brace) { - fields.push(self.parse_struct_decl_field()?); + fields.push(self.parse_struct_decl_field().map_err(|e| { + self.recover_stmt(); + self.eat(&token::CloseDelim(token::Brace)); + e + })?); } self.bump(); @@ -5184,7 +5244,7 @@ impl<'a> Parser<'a> { // If `allow_path` is false, just parse the `pub` in `pub(path)` (but still parse `pub(crate)`) fn parse_visibility(&mut self, allow_path: bool) -> PResult<'a, Visibility> { let pub_crate = |this: &mut Self| { - let span = this.last_span; + let span = this.prev_span; this.expect(&token::CloseDelim(token::Paren))?; Ok(Visibility::Crate(span)) }; @@ -5235,7 +5295,7 @@ impl<'a> Parser<'a> { let hi = if self.span == syntax_pos::DUMMY_SP { inner_lo } else { - self.last_span.hi + self.prev_span.hi }; Ok(ast::Mod { @@ -5260,39 +5320,51 @@ impl<'a> Parser<'a> { /// Parse a `mod <foo> { ... }` or `mod <foo>;` item fn parse_item_mod(&mut self, outer_attrs: &[Attribute]) -> PResult<'a, ItemInfo> { - let outer_attrs = ::config::StripUnconfigured { - config: &self.cfg, - sess: self.sess, - should_test: false, // irrelevant - features: None, // don't perform gated feature checking - }.process_cfg_attrs(outer_attrs.to_owned()); + let (in_cfg, outer_attrs) = { + let mut strip_unconfigured = ::config::StripUnconfigured { + sess: self.sess, + should_test: false, // irrelevant + features: None, // don't perform gated feature checking + }; + let outer_attrs = strip_unconfigured.process_cfg_attrs(outer_attrs.to_owned()); + (strip_unconfigured.in_cfg(&outer_attrs), outer_attrs) + }; let id_span = self.span; let id = self.parse_ident()?; if self.check(&token::Semi) { self.bump(); - // This mod is in an external file. Let's go get it! - let (m, attrs) = self.eval_src_mod(id, &outer_attrs, id_span)?; - Ok((id, m, Some(attrs))) + if in_cfg { + // This mod is in an external file. Let's go get it! + let (m, attrs) = self.eval_src_mod(id, &outer_attrs, id_span)?; + Ok((id, m, Some(attrs))) + } else { + let placeholder = ast::Mod { inner: syntax_pos::DUMMY_SP, items: Vec::new() }; + Ok((id, ItemKind::Mod(placeholder), None)) + } } else { let directory = self.directory.clone(); - self.push_directory(id, &outer_attrs); + let restrictions = self.push_directory(id, &outer_attrs); self.expect(&token::OpenDelim(token::Brace))?; let mod_inner_lo = self.span.lo; let attrs = self.parse_inner_attributes()?; - let m = self.parse_mod_items(&token::CloseDelim(token::Brace), mod_inner_lo)?; + let m = self.with_res(restrictions, |this| { + this.parse_mod_items(&token::CloseDelim(token::Brace), mod_inner_lo) + })?; self.directory = directory; Ok((id, ItemKind::Mod(m), Some(attrs))) } } - fn push_directory(&mut self, id: Ident, attrs: &[Attribute]) { - let default_path = self.id_to_interned_str(id); - let file_path = match ::attr::first_attr_value_str_by_name(attrs, "path") { - Some(d) => d, - None => default_path, - }; - self.directory.push(&*file_path) + fn push_directory(&mut self, id: Ident, attrs: &[Attribute]) -> Restrictions { + if let Some(path) = ::attr::first_attr_value_str_by_name(attrs, "path") { + self.directory.push(&*path); + self.restrictions - Restrictions::NO_NONINLINE_MOD + } else { + let default_path = self.id_to_interned_str(id); + self.directory.push(&*default_path); + self.restrictions + } } pub fn submod_path_from_attr(attrs: &[ast::Attribute], dir_path: &Path) -> Option<PathBuf> { @@ -5417,12 +5489,7 @@ impl<'a> Parser<'a> { included_mod_stack.push(path.clone()); drop(included_mod_stack); - let mut p0 = new_sub_parser_from_file(self.sess, - self.cfg.clone(), - &path, - owns_directory, - Some(name), - id_sp); + let mut p0 = new_sub_parser_from_file(self.sess, &path, owns_directory, Some(name), id_sp); let mod_inner_lo = p0.span.lo; let mod_attrs = p0.parse_inner_attributes()?; let m0 = p0.parse_mod_items(&token::Eof, mod_inner_lo)?; @@ -5491,9 +5558,9 @@ impl<'a> Parser<'a> { }; self.expect(&token::Semi)?; - let last_span = self.last_span; + let prev_span = self.prev_span; Ok(self.mk_item(lo, - last_span.hi, + prev_span.hi, ident, ItemKind::ExternCrate(maybe_path), visibility, @@ -5528,13 +5595,13 @@ impl<'a> Parser<'a> { } self.expect(&token::CloseDelim(token::Brace))?; - let last_span = self.last_span; + let prev_span = self.prev_span; let m = ast::ForeignMod { abi: abi, items: foreign_items }; Ok(self.mk_item(lo, - last_span.hi, + prev_span.hi, keywords::Invalid.ident(), ItemKind::ForeignMod(m), visibility, @@ -5587,7 +5654,7 @@ impl<'a> Parser<'a> { data: struct_def, disr_expr: disr_expr, }; - variants.push(spanned(vlo, self.last_span.hi, vr)); + variants.push(spanned(vlo, self.prev_span.hi, vr)); if !self.eat(&token::Comma) { break; } } @@ -5609,7 +5676,11 @@ impl<'a> Parser<'a> { generics.where_clause = self.parse_where_clause()?; self.expect(&token::OpenDelim(token::Brace))?; - let enum_definition = self.parse_enum_def(&generics)?; + let enum_definition = self.parse_enum_def(&generics).map_err(|e| { + self.recover_stmt(); + self.eat(&token::CloseDelim(token::Brace)); + e + })?; Ok((id, ItemKind::Enum(enum_definition, generics), None)) } @@ -5624,9 +5695,9 @@ impl<'a> Parser<'a> { match abi::lookup(&s.as_str()) { Some(abi) => Ok(Some(abi)), None => { - let last_span = self.last_span; + let prev_span = self.prev_span; self.span_err( - last_span, + prev_span, &format!("invalid ABI: expected one of [{}], \ found `{}`", abi::all_names().join(", "), @@ -5668,9 +5739,9 @@ impl<'a> Parser<'a> { let item_ = ItemKind::Use(self.parse_view_path()?); self.expect(&token::Semi)?; - let last_span = self.last_span; + let prev_span = self.prev_span; let item = self.mk_item(lo, - last_span.hi, + prev_span.hi, keywords::Invalid.ident(), item_, visibility, @@ -5687,15 +5758,15 @@ impl<'a> Parser<'a> { if self.eat_keyword(keywords::Fn) { // EXTERN FUNCTION ITEM - let fn_span = self.last_span; + let fn_span = self.prev_span; let abi = opt_abi.unwrap_or(Abi::C); let (ident, item_, extra_attrs) = self.parse_item_fn(Unsafety::Normal, respan(fn_span, Constness::NotConst), abi)?; - let last_span = self.last_span; + let prev_span = self.prev_span; let item = self.mk_item(lo, - last_span.hi, + prev_span.hi, ident, item_, visibility, @@ -5716,9 +5787,9 @@ impl<'a> Parser<'a> { Mutability::Immutable }; let (ident, item_, extra_attrs) = self.parse_item_const(Some(m))?; - let last_span = self.last_span; + let prev_span = self.prev_span; let item = self.mk_item(lo, - last_span.hi, + prev_span.hi, ident, item_, visibility, @@ -5726,7 +5797,7 @@ impl<'a> Parser<'a> { return Ok(Some(item)); } if self.eat_keyword(keywords::Const) { - let const_span = self.last_span; + let const_span = self.prev_span; if self.check_keyword(keywords::Fn) || (self.check_keyword(keywords::Unsafe) && self.look_ahead(1, |t| t.is_keyword(keywords::Fn))) { @@ -5741,9 +5812,9 @@ impl<'a> Parser<'a> { self.parse_item_fn(unsafety, respan(const_span, Constness::Const), Abi::Rust)?; - let last_span = self.last_span; + let prev_span = self.prev_span; let item = self.mk_item(lo, - last_span.hi, + prev_span.hi, ident, item_, visibility, @@ -5753,15 +5824,15 @@ impl<'a> Parser<'a> { // CONST ITEM if self.eat_keyword(keywords::Mut) { - let last_span = self.last_span; - self.diagnostic().struct_span_err(last_span, "const globals cannot be mutable") + let prev_span = self.prev_span; + self.diagnostic().struct_span_err(prev_span, "const globals cannot be mutable") .help("did you mean to declare a static?") .emit(); } let (ident, item_, extra_attrs) = self.parse_item_const(None)?; - let last_span = self.last_span; + let prev_span = self.prev_span; let item = self.mk_item(lo, - last_span.hi, + prev_span.hi, ident, item_, visibility, @@ -5776,9 +5847,9 @@ impl<'a> Parser<'a> { self.expect_keyword(keywords::Trait)?; let (ident, item_, extra_attrs) = self.parse_item_trait(ast::Unsafety::Unsafe)?; - let last_span = self.last_span; + let prev_span = self.prev_span; let item = self.mk_item(lo, - last_span.hi, + prev_span.hi, ident, item_, visibility, @@ -5792,9 +5863,9 @@ impl<'a> Parser<'a> { self.expect_keyword(keywords::Unsafe)?; self.expect_keyword(keywords::Impl)?; let (ident, item_, extra_attrs) = self.parse_item_impl(ast::Unsafety::Unsafe)?; - let last_span = self.last_span; + let prev_span = self.prev_span; let item = self.mk_item(lo, - last_span.hi, + prev_span.hi, ident, item_, visibility, @@ -5804,14 +5875,14 @@ impl<'a> Parser<'a> { if self.check_keyword(keywords::Fn) { // FUNCTION ITEM self.bump(); - let fn_span = self.last_span; + let fn_span = self.prev_span; let (ident, item_, extra_attrs) = self.parse_item_fn(Unsafety::Normal, respan(fn_span, Constness::NotConst), Abi::Rust)?; - let last_span = self.last_span; + let prev_span = self.prev_span; let item = self.mk_item(lo, - last_span.hi, + prev_span.hi, ident, item_, visibility, @@ -5828,14 +5899,14 @@ impl<'a> Parser<'a> { Abi::Rust }; self.expect_keyword(keywords::Fn)?; - let fn_span = self.last_span; + let fn_span = self.prev_span; let (ident, item_, extra_attrs) = self.parse_item_fn(Unsafety::Unsafe, respan(fn_span, Constness::NotConst), abi)?; - let last_span = self.last_span; + let prev_span = self.prev_span; let item = self.mk_item(lo, - last_span.hi, + prev_span.hi, ident, item_, visibility, @@ -5846,9 +5917,9 @@ impl<'a> Parser<'a> { // MODULE ITEM let (ident, item_, extra_attrs) = self.parse_item_mod(&attrs[..])?; - let last_span = self.last_span; + let prev_span = self.prev_span; let item = self.mk_item(lo, - last_span.hi, + prev_span.hi, ident, item_, visibility, @@ -5858,9 +5929,9 @@ impl<'a> Parser<'a> { if self.eat_keyword(keywords::Type) { // TYPE ITEM let (ident, item_, extra_attrs) = self.parse_item_type()?; - let last_span = self.last_span; + let prev_span = self.prev_span; let item = self.mk_item(lo, - last_span.hi, + prev_span.hi, ident, item_, visibility, @@ -5870,9 +5941,9 @@ impl<'a> Parser<'a> { if self.eat_keyword(keywords::Enum) { // ENUM ITEM let (ident, item_, extra_attrs) = self.parse_item_enum()?; - let last_span = self.last_span; + let prev_span = self.prev_span; let item = self.mk_item(lo, - last_span.hi, + prev_span.hi, ident, item_, visibility, @@ -5883,9 +5954,9 @@ impl<'a> Parser<'a> { // TRAIT ITEM let (ident, item_, extra_attrs) = self.parse_item_trait(ast::Unsafety::Normal)?; - let last_span = self.last_span; + let prev_span = self.prev_span; let item = self.mk_item(lo, - last_span.hi, + prev_span.hi, ident, item_, visibility, @@ -5895,9 +5966,9 @@ impl<'a> Parser<'a> { if self.eat_keyword(keywords::Impl) { // IMPL ITEM let (ident, item_, extra_attrs) = self.parse_item_impl(ast::Unsafety::Normal)?; - let last_span = self.last_span; + let prev_span = self.prev_span; let item = self.mk_item(lo, - last_span.hi, + prev_span.hi, ident, item_, visibility, @@ -5907,23 +5978,22 @@ impl<'a> Parser<'a> { if self.eat_keyword(keywords::Struct) { // STRUCT ITEM let (ident, item_, extra_attrs) = self.parse_item_struct()?; - let last_span = self.last_span; + let prev_span = self.prev_span; let item = self.mk_item(lo, - last_span.hi, + prev_span.hi, ident, item_, visibility, maybe_append(attrs, extra_attrs)); return Ok(Some(item)); } - if self.check_keyword(keywords::Union) && - self.look_ahead(1, |t| t.is_ident() && !t.is_any_keyword()) { + if self.is_union_item() { // UNION ITEM self.bump(); let (ident, item_, extra_attrs) = self.parse_item_union()?; - let last_span = self.last_span; + let prev_span = self.prev_span; let item = self.mk_item(lo, - last_span.hi, + prev_span.hi, ident, item_, visibility, @@ -5966,20 +6036,16 @@ impl<'a> Parser<'a> { lo: BytePos, visibility: Visibility ) -> PResult<'a, Option<P<Item>>> { - if macros_allowed && !self.token.is_any_keyword() - && self.look_ahead(1, |t| *t == token::Not) - && (self.look_ahead(2, |t| t.is_ident()) - || self.look_ahead(2, |t| *t == token::OpenDelim(token::Paren)) - || self.look_ahead(2, |t| *t == token::OpenDelim(token::Brace))) { + if macros_allowed && self.token.is_path_start() { // MACRO INVOCATION ITEM - let last_span = self.last_span; - self.complain_if_pub_macro(&visibility, last_span); + let prev_span = self.prev_span; + self.complain_if_pub_macro(&visibility, prev_span); let mac_lo = self.span.lo; // item macro. - let pth = self.parse_ident_into_path()?; + let pth = self.parse_path(PathStyle::Mod)?; self.expect(&token::Not)?; // a 'special' identifier (like what `macro_rules!` uses) @@ -5995,30 +6061,19 @@ impl<'a> Parser<'a> { let tts = self.parse_seq_to_end(&token::CloseDelim(delim), SeqSep::none(), |p| p.parse_token_tree())?; - // single-variant-enum... : - let m = Mac_ { path: pth, tts: tts }; - let m: ast::Mac = codemap::Spanned { node: m, - span: mk_sp(mac_lo, - self.last_span.hi) }; - if delim != token::Brace { if !self.eat(&token::Semi) { - let last_span = self.last_span; - self.span_err(last_span, + let prev_span = self.prev_span; + self.span_err(prev_span, "macros that expand to items must either \ be surrounded with braces or followed by \ a semicolon"); } } - let item_ = ItemKind::Mac(m); - let last_span = self.last_span; - let item = self.mk_item(lo, - last_span.hi, - id, - item_, - visibility, - attrs); + let hi = self.prev_span.hi; + let mac = spanned(mac_lo, hi, Mac_ { path: pth, tts: tts }); + let item = self.mk_item(lo, hi, id, ItemKind::Mac(mac), visibility, attrs); return Ok(Some(item)); } @@ -6026,8 +6081,8 @@ impl<'a> Parser<'a> { match visibility { Visibility::Inherited => {} _ => { - let last_span = self.last_span; - return Err(self.span_fatal(last_span, "unmatched visibility `pub`")); + let prev_span = self.prev_span; + return Err(self.span_fatal(prev_span, "unmatched visibility `pub`")); } } @@ -6058,7 +6113,7 @@ impl<'a> Parser<'a> { rename: rename, id: ast::DUMMY_NODE_ID }; - let hi = this.last_span.hi; + let hi = this.prev_span.hi; Ok(spanned(lo, hi, node)) }) } @@ -6078,15 +6133,20 @@ impl<'a> Parser<'a> { /// MOD_SEP? LBRACE item_seq RBRACE fn parse_view_path(&mut self) -> PResult<'a, P<ViewPath>> { let lo = self.span.lo; - if self.check(&token::OpenDelim(token::Brace)) || self.is_import_coupler() { - // `{foo, bar}` or `::{foo, bar}` + if self.check(&token::OpenDelim(token::Brace)) || self.check(&token::BinOp(token::Star)) || + self.is_import_coupler() { + // `{foo, bar}`, `::{foo, bar}`, `*`, or `::*`. let prefix = ast::Path { global: self.eat(&token::ModSep), segments: Vec::new(), span: mk_sp(lo, self.span.hi), }; - let items = self.parse_path_list_items()?; - Ok(P(spanned(lo, self.span.hi, ViewPathList(prefix, items)))) + let view_path_kind = if self.eat(&token::BinOp(token::Star)) { + ViewPathGlob(prefix) + } else { + ViewPathList(prefix, self.parse_path_list_items()?) + }; + Ok(P(spanned(lo, self.span.hi, view_path_kind))) } else { let prefix = self.parse_path(PathStyle::Mod)?; if self.is_import_coupler() { @@ -6103,7 +6163,7 @@ impl<'a> Parser<'a> { // `foo::bar` or `foo::bar as baz` let rename = self.parse_rename()?. unwrap_or(prefix.segments.last().unwrap().identifier); - Ok(P(spanned(lo, self.last_span.hi, ViewPathSimple(rename, prefix)))) + Ok(P(spanned(lo, self.prev_span.hi, ViewPathSimple(rename, prefix)))) } } } @@ -6123,7 +6183,6 @@ impl<'a> Parser<'a> { Ok(ast::Crate { attrs: self.parse_inner_attributes()?, module: self.parse_mod_items(&token::Eof, lo)?, - config: self.cfg.clone(), span: mk_sp(lo, self.span.lo), exported_macros: Vec::new(), }) @@ -6151,7 +6210,7 @@ impl<'a> Parser<'a> { pub fn parse_str(&mut self) -> PResult<'a, (InternedString, StrStyle)> { match self.parse_optional_str() { Some((s, style, suf)) => { - let sp = self.last_span; + let sp = self.prev_span; self.expect_no_suffix(sp, "string literal", suf); Ok((s, style)) } diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index ff01d375815..4d0da660302 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -14,7 +14,7 @@ pub use self::DelimToken::*; pub use self::Lit::*; pub use self::Token::*; -use ast::{self, BinOpKind}; +use ast::{self}; use ptr::P; use util::interner::Interner; use tokenstream; @@ -53,21 +53,6 @@ pub enum DelimToken { } #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug, Copy)] -pub enum SpecialMacroVar { - /// `$crate` will be filled in with the name of the crate a macro was - /// imported from, if any. - CrateMacroVar, -} - -impl SpecialMacroVar { - pub fn as_str(self) -> &'static str { - match self { - SpecialMacroVar::CrateMacroVar => "crate", - } - } -} - -#[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug, Copy)] pub enum Lit { Byte(ast::Name), Char(ast::Name), @@ -148,8 +133,6 @@ pub enum Token { // In right-hand-sides of MBE macros: /// A syntactic variable that will be filled in by macro expansion. SubstNt(ast::Ident), - /// A macro variable with special meaning. - SpecialVarNt(SpecialMacroVar), // Junk. These carry no data because we don't really care about the data // they *would* carry, and don't really want to allocate a new ident for @@ -176,10 +159,8 @@ impl Token { /// Returns `true` if the token can appear at the start of an expression. pub fn can_begin_expr(&self) -> bool { match *self { - OpenDelim(_) => true, + OpenDelim(..) => true, Ident(..) => true, - Underscore => true, - Tilde => true, Literal(..) => true, Not => true, BinOp(Minus) => true, @@ -189,6 +170,7 @@ impl Token { OrOr => true, // in lambda syntax AndAnd => true, // double borrow DotDot | DotDotDot => true, // range notation + Lt | BinOp(Shl) => true, // associated path ModSep => true, Interpolated(NtExpr(..)) => true, Interpolated(NtIdent(..)) => true, @@ -253,34 +235,13 @@ impl Token { self.is_keyword(keywords::Const) } - pub fn is_path_start(&self) -> bool { - self == &ModSep || self == &Lt || self.is_path() || - self.is_path_segment_keyword() || self.is_ident() && !self.is_any_keyword() + pub fn is_qpath_start(&self) -> bool { + self == &Lt || self == &BinOp(Shl) } - /// Maps a token to its corresponding binary operator. - pub fn to_binop(&self) -> Option<BinOpKind> { - match *self { - BinOp(Star) => Some(BinOpKind::Mul), - BinOp(Slash) => Some(BinOpKind::Div), - BinOp(Percent) => Some(BinOpKind::Rem), - BinOp(Plus) => Some(BinOpKind::Add), - BinOp(Minus) => Some(BinOpKind::Sub), - BinOp(Shl) => Some(BinOpKind::Shl), - BinOp(Shr) => Some(BinOpKind::Shr), - BinOp(And) => Some(BinOpKind::BitAnd), - BinOp(Caret) => Some(BinOpKind::BitXor), - BinOp(Or) => Some(BinOpKind::BitOr), - Lt => Some(BinOpKind::Lt), - Le => Some(BinOpKind::Le), - Ge => Some(BinOpKind::Ge), - Gt => Some(BinOpKind::Gt), - EqEq => Some(BinOpKind::Eq), - Ne => Some(BinOpKind::Ne), - AndAnd => Some(BinOpKind::And), - OrOr => Some(BinOpKind::Or), - _ => None, - } + pub fn is_path_start(&self) -> bool { + self == &ModSep || self.is_qpath_start() || self.is_path() || + self.is_path_segment_keyword() || self.is_ident() && !self.is_any_keyword() } /// Returns `true` if the token is a given keyword, `kw`. @@ -503,27 +464,20 @@ pub fn clear_ident_interner() { /// somehow. #[derive(Clone, PartialEq, Hash, PartialOrd, Eq, Ord)] pub struct InternedString { - string: Rc<String>, + string: Rc<str>, } impl InternedString { #[inline] pub fn new(string: &'static str) -> InternedString { InternedString { - string: Rc::new(string.to_owned()), - } - } - - #[inline] - fn new_from_rc_str(string: Rc<String>) -> InternedString { - InternedString { - string: string, + string: Rc::__from_str(string), } } #[inline] pub fn new_from_name(name: ast::Name) -> InternedString { - with_ident_interner(|interner| InternedString::new_from_rc_str(interner.get(name))) + with_ident_interner(|interner| InternedString { string: interner.get(name) }) } } @@ -591,7 +545,7 @@ impl PartialEq<InternedString> for str { impl Decodable for InternedString { fn decode<D: Decoder>(d: &mut D) -> Result<InternedString, D::Error> { - Ok(intern(d.read_str()?.as_ref()).as_str()) + Ok(intern(&d.read_str()?).as_str()) } } diff --git a/src/libsyntax/print/pp.rs b/src/libsyntax/print/pp.rs index 32b66da4d96..792239e7219 100644 --- a/src/libsyntax/print/pp.rs +++ b/src/libsyntax/print/pp.rs @@ -125,9 +125,8 @@ impl fmt::Display for Token { } } -fn buf_str(toks: &[Token], szs: &[isize], left: usize, right: usize, lim: usize) -> String { - let n = toks.len(); - assert_eq!(n, szs.len()); +fn buf_str(buf: &[BufEntry], left: usize, right: usize, lim: usize) -> String { + let n = buf.len(); let mut i = left; let mut l = lim; let mut s = String::from("["); @@ -136,7 +135,7 @@ fn buf_str(toks: &[Token], szs: &[isize], left: usize, right: usize, lim: usize) if i != left { s.push_str(", "); } - s.push_str(&format!("{}={}", szs[i], &toks[i])); + s.push_str(&format!("{}={}", buf[i].size, &buf[i].token)); i += 1; i %= n; } @@ -159,13 +158,9 @@ pub struct PrintStackElem { const SIZE_INFINITY: isize = 0xffff; pub fn mk_printer<'a>(out: Box<io::Write+'a>, linewidth: usize) -> Printer<'a> { - // Yes 55, it makes the ring buffers big enough to never - // fall behind. + // Yes 55, it makes the ring buffers big enough to never fall behind. let n: usize = 55 * linewidth; debug!("mk_printer {}", linewidth); - let token = vec![Token::Eof; n]; - let size = vec![0; n]; - let scan_stack = VecDeque::with_capacity(n); Printer { out: out, buf_len: n, @@ -173,11 +168,10 @@ pub fn mk_printer<'a>(out: Box<io::Write+'a>, linewidth: usize) -> Printer<'a> { space: linewidth as isize, left: 0, right: 0, - token: token, - size: size, + buf: vec![BufEntry { token: Token::Eof, size: 0 }; n], left_total: 0, right_total: 0, - scan_stack: scan_stack, + scan_stack: VecDeque::new(), print_stack: Vec::new(), pending_indentation: 0 } @@ -269,10 +263,8 @@ pub struct Printer<'a> { left: usize, /// Index of right side of input stream right: usize, - /// Ring-buffer stream goes through - token: Vec<Token> , - /// Ring-buffer of calculated sizes - size: Vec<isize> , + /// Ring-buffer of tokens and calculated sizes + buf: Vec<BufEntry>, /// Running size of stream "...left" left_total: isize, /// Running size of stream "...right" @@ -283,20 +275,26 @@ pub struct Printer<'a> { /// Begin (if there is any) on top of it. Stuff is flushed off the /// bottom as it becomes irrelevant due to the primary ring-buffer /// advancing. - scan_stack: VecDeque<usize> , + scan_stack: VecDeque<usize>, /// Stack of blocks-in-progress being flushed by print print_stack: Vec<PrintStackElem> , /// Buffered indentation to avoid writing trailing whitespace pending_indentation: isize, } +#[derive(Clone)] +struct BufEntry { + token: Token, + size: isize, +} + impl<'a> Printer<'a> { pub fn last_token(&mut self) -> Token { - self.token[self.right].clone() + self.buf[self.right].token.clone() } // be very careful with this! pub fn replace_last_token(&mut self, t: Token) { - self.token[self.right] = t; + self.buf[self.right].token = t; } pub fn pretty_print(&mut self, token: Token) -> io::Result<()> { debug!("pp Vec<{},{}>", self.left, self.right); @@ -318,8 +316,7 @@ impl<'a> Printer<'a> { } else { self.advance_right(); } debug!("pp Begin({})/buffer Vec<{},{}>", b.offset, self.left, self.right); - self.token[self.right] = token; - self.size[self.right] = -self.right_total; + self.buf[self.right] = BufEntry { token: token, size: -self.right_total }; let right = self.right; self.scan_push(right); Ok(()) @@ -331,8 +328,7 @@ impl<'a> Printer<'a> { } else { debug!("pp End/buffer Vec<{},{}>", self.left, self.right); self.advance_right(); - self.token[self.right] = token; - self.size[self.right] = -1; + self.buf[self.right] = BufEntry { token: token, size: -1 }; let right = self.right; self.scan_push(right); Ok(()) @@ -350,8 +346,7 @@ impl<'a> Printer<'a> { self.check_stack(0); let right = self.right; self.scan_push(right); - self.token[self.right] = token; - self.size[self.right] = -self.right_total; + self.buf[self.right] = BufEntry { token: token, size: -self.right_total }; self.right_total += b.blank_space; Ok(()) } @@ -364,8 +359,7 @@ impl<'a> Printer<'a> { debug!("pp String('{}')/buffer Vec<{},{}>", s, self.left, self.right); self.advance_right(); - self.token[self.right] = Token::String(s, len); - self.size[self.right] = len; + self.buf[self.right] = BufEntry { token: Token::String(s, len), size: len }; self.right_total += len; self.check_stream() } @@ -381,7 +375,7 @@ impl<'a> Printer<'a> { if Some(&self.left) == self.scan_stack.back() { debug!("setting {} to infinity and popping", self.left); let scanned = self.scan_pop_bottom(); - self.size[scanned] = SIZE_INFINITY; + self.buf[scanned].size = SIZE_INFINITY; } self.advance_left()?; if self.left != self.right { @@ -410,12 +404,12 @@ impl<'a> Printer<'a> { } pub fn advance_left(&mut self) -> io::Result<()> { debug!("advance_left Vec<{},{}>, sizeof({})={}", self.left, self.right, - self.left, self.size[self.left]); + self.left, self.buf[self.left].size); - let mut left_size = self.size[self.left]; + let mut left_size = self.buf[self.left].size; while left_size >= 0 { - let left = self.token[self.left].clone(); + let left = self.buf[self.left].token.clone(); let len = match left { Token::Break(b) => b.blank_space, @@ -437,7 +431,7 @@ impl<'a> Printer<'a> { self.left += 1; self.left %= self.buf_len; - left_size = self.size[self.left]; + left_size = self.buf[self.left].size; } Ok(()) @@ -445,23 +439,23 @@ impl<'a> Printer<'a> { pub fn check_stack(&mut self, k: isize) { if !self.scan_stack.is_empty() { let x = self.scan_top(); - match self.token[x] { + match self.buf[x].token { Token::Begin(_) => { if k > 0 { let popped = self.scan_pop(); - self.size[popped] = self.size[x] + self.right_total; + self.buf[popped].size = self.buf[x].size + self.right_total; self.check_stack(k - 1); } } Token::End => { // paper says + not =, but that makes no sense. let popped = self.scan_pop(); - self.size[popped] = 1; + self.buf[popped].size = 1; self.check_stack(k + 1); } _ => { let popped = self.scan_pop(); - self.size[popped] = self.size[x] + self.right_total; + self.buf[popped].size = self.buf[x].size + self.right_total; if k > 0 { self.check_stack(k); } @@ -499,8 +493,7 @@ impl<'a> Printer<'a> { pub fn print(&mut self, token: Token, l: isize) -> io::Result<()> { debug!("print {} {} (remaining line space={})", token, l, self.space); - debug!("{}", buf_str(&self.token, - &self.size, + debug!("{}", buf_str(&self.buf, self.left, self.right, 6)); diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs index 8563d27908d..b0bd6446743 100644 --- a/src/libsyntax/print/pprust.rs +++ b/src/libsyntax/print/pprust.rs @@ -112,7 +112,7 @@ pub fn print_crate<'a>(cm: &'a CodeMap, out, ann, is_expanded); - if is_expanded && !std_inject::no_std(krate) { + if is_expanded && !std_inject::injected_crate_name(krate).is_none() { // We need to print `#![no_std]` (and its feature gate) so that // compiling pretty-printed source won't inject libstd again. // However we don't want these attributes in the AST because @@ -285,8 +285,6 @@ pub fn token_to_string(tok: &Token) -> String { token::Comment => "/* */".to_string(), token::Shebang(s) => format!("/* shebang: {}*/", s), - token::SpecialVarNt(var) => format!("${}", var.as_str()), - token::Interpolated(ref nt) => match *nt { token::NtExpr(ref e) => expr_to_string(&e), token::NtMeta(ref e) => meta_item_to_string(&e), @@ -545,15 +543,12 @@ pub trait PrintState<'a> { } fn maybe_print_comment(&mut self, pos: BytePos) -> io::Result<()> { - loop { - match self.next_comment() { - Some(ref cmnt) => { - if (*cmnt).pos < pos { - try!(self.print_comment(cmnt)); - self.cur_cmnt_and_lit().cur_cmnt += 1; - } else { break; } - } - _ => break + while let Some(ref cmnt) = self.next_comment() { + if cmnt.pos < pos { + try!(self.print_comment(cmnt)); + self.cur_cmnt_and_lit().cur_cmnt += 1; + } else { + break } } Ok(()) @@ -581,7 +576,9 @@ pub trait PrintState<'a> { Ok(()) } comments::Trailing => { - try!(word(self.writer(), " ")); + if !self.is_bol() { + try!(word(self.writer(), " ")); + } if cmnt.lines.len() == 1 { try!(word(self.writer(), &cmnt.lines[0])); hardbreak(self.writer()) @@ -972,7 +969,7 @@ impl<'a> State<'a> { try!(self.maybe_print_comment(ty.span.lo)); try!(self.ibox(0)); match ty.node { - ast::TyKind::Vec(ref ty) => { + ast::TyKind::Slice(ref ty) => { try!(word(&mut self.s, "[")); try!(self.print_type(&ty)); try!(word(&mut self.s, "]")); @@ -1039,7 +1036,7 @@ impl<'a> State<'a> { ast::TyKind::ImplTrait(ref bounds) => { try!(self.print_bounds("impl ", &bounds[..])); } - ast::TyKind::FixedLengthVec(ref ty, ref v) => { + ast::TyKind::Array(ref ty, ref v) => { try!(word(&mut self.s, "[")); try!(self.print_type(&ty)); try!(word(&mut self.s, "; ")); @@ -1361,6 +1358,7 @@ impl<'a> State<'a> { if comma { try!(self.word_space(",")) } + try!(self.print_outer_attributes_inline(&lifetime_def.attrs)); try!(self.print_lifetime_bounds(&lifetime_def.lifetime, &lifetime_def.bounds)); comma = true; } @@ -1715,6 +1713,7 @@ impl<'a> State<'a> { for (i, st) in blk.stmts.iter().enumerate() { match st.node { ast::StmtKind::Expr(ref expr) if i == blk.stmts.len() - 1 => { + try!(self.maybe_print_comment(st.span.lo)); try!(self.space_if_not_bol()); try!(self.print_expr_outer_attr_style(&expr, false)); try!(self.maybe_print_trailing_comment(expr.span, Some(blk.span.hi))); @@ -1894,8 +1893,10 @@ impl<'a> State<'a> { &fields[..], |s, field| { try!(s.ibox(INDENT_UNIT)); - try!(s.print_ident(field.ident.node)); - try!(s.word_space(":")); + if !field.is_shorthand { + try!(s.print_ident(field.ident.node)); + try!(s.word_space(":")); + } try!(s.print_expr(&field.expr)); s.end() }, @@ -2276,7 +2277,7 @@ impl<'a> State<'a> { Ok(()) })); - let mut options = vec!(); + let mut options = vec![]; if a.volatile { options.push("volatile"); } @@ -2450,13 +2451,10 @@ impl<'a> State<'a> { |s, ty| s.print_type(&ty))); try!(word(&mut self.s, ")")); - match data.output { - None => { } - Some(ref ty) => { - try!(self.space_if_not_bol()); - try!(self.word_space("->")); - try!(self.print_type(&ty)); - } + if let Some(ref ty) = data.output { + try!(self.space_if_not_bol()); + try!(self.word_space("->")); + try!(self.print_type(&ty)); } } } @@ -2573,7 +2571,7 @@ impl<'a> State<'a> { try!(word(&mut self.s, "...")); try!(self.print_expr(&end)); } - PatKind::Vec(ref before, ref slice, ref after) => { + PatKind::Slice(ref before, ref slice, ref after) => { try!(word(&mut self.s, "[")); try!(self.commasep(Inconsistent, &before[..], @@ -2604,6 +2602,7 @@ impl<'a> State<'a> { } try!(self.cbox(INDENT_UNIT)); try!(self.ibox(0)); + try!(self.maybe_print_comment(arm.pats[0].span.lo)); try!(self.print_outer_attributes(&arm.attrs)); let mut first = true; for p in &arm.pats { @@ -2803,6 +2802,7 @@ impl<'a> State<'a> { try!(self.commasep(Inconsistent, &ints[..], |s, &idx| { if idx < generics.lifetimes.len() { let lifetime_def = &generics.lifetimes[idx]; + try!(s.print_outer_attributes_inline(&lifetime_def.attrs)); s.print_lifetime_bounds(&lifetime_def.lifetime, &lifetime_def.bounds) } else { let idx = idx - generics.lifetimes.len(); @@ -2816,6 +2816,7 @@ impl<'a> State<'a> { } pub fn print_ty_param(&mut self, param: &ast::TyParam) -> io::Result<()> { + try!(self.print_outer_attributes_inline(¶m.attrs)); try!(self.print_ident(param.ident)); try!(self.print_bounds(":", ¶m.bounds)); match param.default { @@ -3007,15 +3008,11 @@ impl<'a> State<'a> { _ => return Ok(()) }; if let Some(ref cmnt) = self.next_comment() { - if (*cmnt).style != comments::Trailing { return Ok(()) } + if cmnt.style != comments::Trailing { return Ok(()) } let span_line = cm.lookup_char_pos(span.hi); - let comment_line = cm.lookup_char_pos((*cmnt).pos); - let mut next = (*cmnt).pos + BytePos(1); - if let Some(p) = next_pos { - next = p; - } - if span.hi < (*cmnt).pos && (*cmnt).pos < next && - span_line.line == comment_line.line { + let comment_line = cm.lookup_char_pos(cmnt.pos); + let next = next_pos.unwrap_or(cmnt.pos + BytePos(1)); + if span.hi < cmnt.pos && cmnt.pos < next && span_line.line == comment_line.line { self.print_comment(cmnt)?; self.cur_cmnt_and_lit.cur_cmnt += 1; } diff --git a/src/libsyntax/std_inject.rs b/src/libsyntax/std_inject.rs index d1454ab06cb..1b63a2b7076 100644 --- a/src/libsyntax/std_inject.rs +++ b/src/libsyntax/std_inject.rs @@ -34,23 +34,25 @@ fn ignored_span(sess: &ParseSess, sp: Span) -> Span { return sp; } -pub fn no_core(krate: &ast::Crate) -> bool { - attr::contains_name(&krate.attrs, "no_core") -} - -pub fn no_std(krate: &ast::Crate) -> bool { - attr::contains_name(&krate.attrs, "no_std") || no_core(krate) +pub fn injected_crate_name(krate: &ast::Crate) -> Option<&'static str> { + if attr::contains_name(&krate.attrs, "no_core") { + None + } else if attr::contains_name(&krate.attrs, "no_std") { + Some("core") + } else { + Some("std") + } } pub fn maybe_inject_crates_ref(sess: &ParseSess, mut krate: ast::Crate, alt_std_name: Option<String>) -> ast::Crate { - if no_core(&krate) { - return krate; - } + let name = match injected_crate_name(&krate) { + Some(name) => name, + None => return krate, + }; - let name = if no_std(&krate) { "core" } else { "std" }; let crate_name = token::intern(&alt_std_name.unwrap_or(name.to_string())); krate.module.items.insert(0, P(ast::Item { diff --git a/src/libsyntax/test.rs b/src/libsyntax/test.rs index 6327e8f71bc..618878c1f79 100644 --- a/src/libsyntax/test.rs +++ b/src/libsyntax/test.rs @@ -19,7 +19,7 @@ use std::iter; use std::slice; use std::mem; use std::vec; -use attr; +use attr::{self, HasAttrs}; use syntax_pos::{self, DUMMY_SP, NO_EXPANSION, Span, FileMap, BytePos}; use std::rc::Rc; @@ -119,7 +119,7 @@ impl<'a> fold::Folder for TestHarnessGenerator<'a> { } debug!("current path: {}", path_name_i(&self.cx.path)); - let i = if is_test_fn(&self.cx, &i) || is_bench_fn(&self.cx, &i) { + if is_test_fn(&self.cx, &i) || is_bench_fn(&self.cx, &i) { match i.node { ast::ItemKind::Fn(_, ast::Unsafety::Unsafe, _, _, _, _) => { let diag = self.cx.span_diagnostic; @@ -136,54 +136,37 @@ impl<'a> fold::Folder for TestHarnessGenerator<'a> { }; self.cx.testfns.push(test); self.tests.push(i.ident); - // debug!("have {} test/bench functions", - // cx.testfns.len()); - - // Make all tests public so we can call them from outside - // the module (note that the tests are re-exported and must - // be made public themselves to avoid privacy errors). - i.map(|mut i| { - i.vis = ast::Visibility::Public; - i - }) } } - } else { - i - }; + } + let mut item = i.unwrap(); // We don't want to recurse into anything other than mods, since // mods or tests inside of functions will break things - let res = match i.node { - ast::ItemKind::Mod(..) => fold::noop_fold_item(i, self), - _ => SmallVector::one(i), - }; + if let ast::ItemKind::Mod(module) = item.node { + let tests = mem::replace(&mut self.tests, Vec::new()); + let tested_submods = mem::replace(&mut self.tested_submods, Vec::new()); + let mut mod_folded = fold::noop_fold_mod(module, self); + let tests = mem::replace(&mut self.tests, tests); + let tested_submods = mem::replace(&mut self.tested_submods, tested_submods); + + if !tests.is_empty() || !tested_submods.is_empty() { + let (it, sym) = mk_reexport_mod(&mut self.cx, item.id, tests, tested_submods); + mod_folded.items.push(it); + + if !self.cx.path.is_empty() { + self.tested_submods.push((self.cx.path[self.cx.path.len()-1], sym)); + } else { + debug!("pushing nothing, sym: {:?}", sym); + self.cx.toplevel_reexport = Some(sym); + } + } + item.node = ast::ItemKind::Mod(mod_folded); + } if ident.name != keywords::Invalid.name() { self.cx.path.pop(); } - res - } - - fn fold_mod(&mut self, m: ast::Mod) -> ast::Mod { - let tests = mem::replace(&mut self.tests, Vec::new()); - let tested_submods = mem::replace(&mut self.tested_submods, Vec::new()); - let mut mod_folded = fold::noop_fold_mod(m, self); - let tests = mem::replace(&mut self.tests, tests); - let tested_submods = mem::replace(&mut self.tested_submods, tested_submods); - - if !tests.is_empty() || !tested_submods.is_empty() { - let (it, sym) = mk_reexport_mod(&mut self.cx, tests, tested_submods); - mod_folded.items.push(it); - - if !self.cx.path.is_empty() { - self.tested_submods.push((self.cx.path[self.cx.path.len()-1], sym)); - } else { - debug!("pushing nothing, sym: {:?}", sym); - self.cx.toplevel_reexport = Some(sym); - } - } - - mod_folded + SmallVector::one(P(item)) } fn fold_mac(&mut self, mac: ast::Mac) -> ast::Mac { mac } @@ -239,16 +222,24 @@ impl fold::Folder for EntryPointCleaner { fn fold_mac(&mut self, mac: ast::Mac) -> ast::Mac { mac } } -fn mk_reexport_mod(cx: &mut TestCtxt, tests: Vec<ast::Ident>, +fn mk_reexport_mod(cx: &mut TestCtxt, parent: ast::NodeId, tests: Vec<ast::Ident>, tested_submods: Vec<(ast::Ident, ast::Ident)>) -> (P<ast::Item>, ast::Ident) { let super_ = token::str_to_ident("super"); + // Generate imports with `#[allow(private_in_public)]` to work around issue #36768. + let allow_private_in_public = cx.ext_cx.attribute(DUMMY_SP, cx.ext_cx.meta_list( + DUMMY_SP, + InternedString::new("allow"), + vec![cx.ext_cx.meta_list_item_word(DUMMY_SP, InternedString::new("private_in_public"))], + )); let items = tests.into_iter().map(|r| { cx.ext_cx.item_use_simple(DUMMY_SP, ast::Visibility::Public, cx.ext_cx.path(DUMMY_SP, vec![super_, r])) + .map_attrs(|_| vec![allow_private_in_public.clone()]) }).chain(tested_submods.into_iter().map(|(r, sym)| { let path = cx.ext_cx.path(DUMMY_SP, vec![super_, r, sym]); cx.ext_cx.item_use_simple_(DUMMY_SP, ast::Visibility::Public, r, path) + .map_attrs(|_| vec![allow_private_in_public.clone()]) })).collect(); let reexport_mod = ast::Mod { @@ -257,6 +248,8 @@ fn mk_reexport_mod(cx: &mut TestCtxt, tests: Vec<ast::Ident>, }; let sym = token::gensym_ident("__test_reexports"); + let parent = if parent == ast::DUMMY_NODE_ID { ast::CRATE_NODE_ID } else { parent }; + cx.ext_cx.current_expansion.mark = cx.ext_cx.resolver.get_module_scope(parent); let it = cx.ext_cx.monotonic_expander().fold_item(P(ast::Item { ident: sym.clone(), attrs: Vec::new(), @@ -281,7 +274,7 @@ fn generate_test_harness(sess: &ParseSess, let mut cx: TestCtxt = TestCtxt { sess: sess, span_diagnostic: sd, - ext_cx: ExtCtxt::new(sess, vec![], ExpansionConfig::default("test".to_string()), resolver), + ext_cx: ExtCtxt::new(sess, ExpansionConfig::default("test".to_string()), resolver), path: Vec::new(), testfns: Vec::new(), reexport_test_harness_main: reexport_test_harness_main, @@ -437,7 +430,7 @@ fn mk_std(cx: &TestCtxt) -> P<ast::Item> { let (vi, vis, ident) = if cx.is_test_crate { (ast::ItemKind::Use( P(nospan(ast::ViewPathSimple(id_test, - path_node(vec!(id_test)))))), + path_node(vec![id_test]))))), ast::Visibility::Public, keywords::Invalid.ident()) } else { (ast::ItemKind::ExternCrate(None), ast::Visibility::Inherited, id_test) @@ -579,7 +572,7 @@ fn mk_tests(cx: &TestCtxt) -> P<ast::Item> { let static_lt = ecx.lifetime(sp, keywords::StaticLifetime.name()); // &'static [self::test::TestDescAndFn] let static_type = ecx.ty_rptr(sp, - ecx.ty(sp, ast::TyKind::Vec(struct_type)), + ecx.ty(sp, ast::TyKind::Slice(struct_type)), Some(static_lt), ast::Mutability::Immutable); // static TESTS: $static_type = &[...]; diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index 7b1df6f0e97..9e644e59e86 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -33,6 +33,7 @@ use parse::lexer::comments::{doc_comment_style, strip_doc_comment_decoration}; use parse::lexer; use parse; use parse::token::{self, Token, Lit, Nonterminal}; +use print::pprust; use std::fmt; use std::iter::*; @@ -133,7 +134,6 @@ impl TokenTree { AttrStyle::Inner => 3, } } - TokenTree::Token(_, token::SpecialVarNt(..)) => 2, TokenTree::Token(_, token::MatchNt(..)) => 3, TokenTree::Token(_, token::Interpolated(Nonterminal::NtTT(..))) => 1, TokenTree::Delimited(_, ref delimed) => delimed.tts.len() + 2, @@ -187,11 +187,6 @@ impl TokenTree { } delimed.tts[index - 1].clone() } - (&TokenTree::Token(sp, token::SpecialVarNt(var)), _) => { - let v = [TokenTree::Token(sp, token::Dollar), - TokenTree::Token(sp, token::Ident(token::str_to_ident(var.as_str())))]; - v[index].clone() - } (&TokenTree::Token(sp, token::MatchNt(name, kind)), _) => { let v = [TokenTree::Token(sp, token::SubstNt(name)), TokenTree::Token(sp, token::Colon), @@ -223,10 +218,9 @@ impl TokenTree { // `None` is because we're not interpolating let arg_rdr = lexer::new_tt_reader_with_doc_flag(&cx.parse_sess().span_diagnostic, None, - None, tts.iter().cloned().collect(), true); - macro_parser::parse(cx.parse_sess(), cx.cfg(), arg_rdr, mtch) + macro_parser::parse(cx.parse_sess(), arg_rdr, mtch) } /// Check if this TokenTree is equal to the other, regardless of span information. @@ -781,6 +775,12 @@ impl TokenStream { } } +impl fmt::Display for TokenStream { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str(&pprust::tts_to_string(&self.to_tts())) + } +} + // FIXME Reimplement this iterator to hold onto a slice iterator for a leaf, getting the // next leaf's iterator when the current one is exhausted. pub struct Iter<'a> { diff --git a/src/libsyntax/util/interner.rs b/src/libsyntax/util/interner.rs index 6bb409715aa..f56c6cedcd1 100644 --- a/src/libsyntax/util/interner.rs +++ b/src/libsyntax/util/interner.rs @@ -14,23 +14,13 @@ use ast::Name; -use std::borrow::Borrow; use std::collections::HashMap; use std::rc::Rc; -#[derive(PartialEq, Eq, Hash)] -struct RcStr(Rc<String>); - -impl Borrow<str> for RcStr { - fn borrow(&self) -> &str { - &self.0 - } -} - #[derive(Default)] pub struct Interner { - names: HashMap<RcStr, Name>, - strings: Vec<Rc<String>>, + names: HashMap<Rc<str>, Name>, + strings: Vec<Rc<str>>, } /// When traits can extend traits, we should extend index<Name,T> to get [] @@ -47,22 +37,22 @@ impl Interner { this } - pub fn intern<T: Borrow<str> + Into<String>>(&mut self, string: T) -> Name { - if let Some(&name) = self.names.get(string.borrow()) { + pub fn intern(&mut self, string: &str) -> Name { + if let Some(&name) = self.names.get(string) { return name; } let name = Name(self.strings.len() as u32); - let string = Rc::new(string.into()); + let string = Rc::__from_str(string); self.strings.push(string.clone()); - self.names.insert(RcStr(string), name); + self.names.insert(string, name); name } pub fn gensym(&mut self, string: &str) -> Name { let gensym = Name(self.strings.len() as u32); // leave out of `names` to avoid colliding - self.strings.push(Rc::new(string.to_owned())); + self.strings.push(Rc::__from_str(string)); gensym } @@ -75,7 +65,7 @@ impl Interner { gensym } - pub fn get(&self, name: Name) -> Rc<String> { + pub fn get(&self, name: Name) -> Rc<str> { self.strings[name.0 as usize].clone() } @@ -109,13 +99,13 @@ mod tests { assert_eq!(i.gensym("dog"), Name(4)); // gensym tests again with gensym_copy: assert_eq!(i.gensym_copy(Name(2)), Name(5)); - assert_eq!(*i.get(Name(5)), "zebra"); + assert_eq!(&*i.get(Name(5)), "zebra"); assert_eq!(i.gensym_copy(Name(2)), Name(6)); - assert_eq!(*i.get(Name(6)), "zebra"); - assert_eq!(*i.get(Name(0)), "dog"); - assert_eq!(*i.get(Name(1)), "cat"); - assert_eq!(*i.get(Name(2)), "zebra"); - assert_eq!(*i.get(Name(3)), "zebra"); - assert_eq!(*i.get(Name(4)), "dog"); + assert_eq!(&*i.get(Name(6)), "zebra"); + assert_eq!(&*i.get(Name(0)), "dog"); + assert_eq!(&*i.get(Name(1)), "cat"); + assert_eq!(&*i.get(Name(2)), "zebra"); + assert_eq!(&*i.get(Name(3)), "zebra"); + assert_eq!(&*i.get(Name(4)), "dog"); } } diff --git a/src/libsyntax/util/parser_testing.rs b/src/libsyntax/util/parser_testing.rs index f59428bf536..76d3f2a063c 100644 --- a/src/libsyntax/util/parser_testing.rs +++ b/src/libsyntax/util/parser_testing.rs @@ -25,10 +25,7 @@ pub fn string_to_tts(source_str: String) -> Vec<tokenstream::TokenTree> { /// Map string to parser (via tts) pub fn string_to_parser<'a>(ps: &'a ParseSess, source_str: String) -> Parser<'a> { - new_parser_from_source_str(ps, - Vec::new(), - "bogofile".to_string(), - source_str) + new_parser_from_source_str(ps, "bogofile".to_string(), source_str) } fn with_error_checking_parse<'a, T, F>(s: String, ps: &'a ParseSess, f: F) -> T where diff --git a/src/libsyntax/util/small_vector.rs b/src/libsyntax/util/small_vector.rs index 373dfc4ddfa..9be7dbd6817 100644 --- a/src/libsyntax/util/small_vector.rs +++ b/src/libsyntax/util/small_vector.rs @@ -11,6 +11,7 @@ use self::SmallVectorRepr::*; use self::IntoIterRepr::*; +use core::ops; use std::iter::{IntoIterator, FromIterator}; use std::mem; use std::slice; @@ -19,10 +20,12 @@ use std::vec; use util::move_map::MoveMap; /// A vector type optimized for cases where the size is almost always 0 or 1 +#[derive(Clone)] pub struct SmallVector<T> { repr: SmallVectorRepr<T>, } +#[derive(Clone)] enum SmallVectorRepr<T> { Zero, One(T), @@ -75,16 +78,11 @@ impl<T> SmallVector<T> { } pub fn as_slice(&self) -> &[T] { - match self.repr { - Zero => { - let result: &[T] = &[]; - result - } - One(ref v) => { - unsafe { slice::from_raw_parts(v, 1) } - } - Many(ref vs) => vs - } + self + } + + pub fn as_mut_slice(&mut self) -> &mut [T] { + self } pub fn pop(&mut self) -> Option<T> { @@ -107,7 +105,7 @@ impl<T> SmallVector<T> { One(..) => { let one = mem::replace(&mut self.repr, Zero); match one { - One(v1) => mem::replace(&mut self.repr, Many(vec!(v1, v))), + One(v1) => mem::replace(&mut self.repr, Many(vec![v1, v])), _ => unreachable!() }; } @@ -163,6 +161,38 @@ impl<T> SmallVector<T> { } } +impl<T> ops::Deref for SmallVector<T> { + type Target = [T]; + + fn deref(&self) -> &[T] { + match self.repr { + Zero => { + let result: &[T] = &[]; + result + } + One(ref v) => { + unsafe { slice::from_raw_parts(v, 1) } + } + Many(ref vs) => vs + } + } +} + +impl<T> ops::DerefMut for SmallVector<T> { + fn deref_mut(&mut self) -> &mut [T] { + match self.repr { + Zero => { + let result: &mut [T] = &mut []; + result + } + One(ref mut v) => { + unsafe { slice::from_raw_parts_mut(v, 1) } + } + Many(ref mut vs) => vs + } + } +} + impl<T> IntoIterator for SmallVector<T> { type Item = T; type IntoIter = IntoIter<T>; @@ -284,12 +314,12 @@ mod tests { #[test] #[should_panic] fn test_expect_one_many() { - SmallVector::many(vec!(1, 2)).expect_one(""); + SmallVector::many(vec![1, 2]).expect_one(""); } #[test] fn test_expect_one_one() { assert_eq!(1, SmallVector::one(1).expect_one("")); - assert_eq!(1, SmallVector::many(vec!(1)).expect_one("")); + assert_eq!(1, SmallVector::many(vec![1]).expect_one("")); } } diff --git a/src/libsyntax/visit.rs b/src/libsyntax/visit.rs index efd9b027504..7fb3e5c6bee 100644 --- a/src/libsyntax/visit.rs +++ b/src/libsyntax/visit.rs @@ -201,6 +201,7 @@ pub fn walk_lifetime<V: Visitor>(visitor: &mut V, lifetime: &Lifetime) { pub fn walk_lifetime_def<V: Visitor>(visitor: &mut V, lifetime_def: &LifetimeDef) { visitor.visit_lifetime(&lifetime_def.lifetime); walk_list!(visitor, visit_lifetime, &lifetime_def.bounds); + walk_list!(visitor, visit_attribute, &*lifetime_def.attrs); } pub fn walk_poly_trait_ref<V>(visitor: &mut V, trait_ref: &PolyTraitRef, _: &TraitBoundModifier) @@ -313,7 +314,7 @@ pub fn walk_variant<V>(visitor: &mut V, variant: &Variant, generics: &Generics, pub fn walk_ty<V: Visitor>(visitor: &mut V, typ: &Ty) { match typ.node { - TyKind::Vec(ref ty) | TyKind::Paren(ref ty) => { + TyKind::Slice(ref ty) | TyKind::Paren(ref ty) => { visitor.visit_ty(ty) } TyKind::Ptr(ref mutable_type) => { @@ -341,7 +342,7 @@ pub fn walk_ty<V: Visitor>(visitor: &mut V, typ: &Ty) { visitor.visit_ty(ty); walk_list!(visitor, visit_ty_param_bound, bounds); } - TyKind::FixedLengthVec(ref ty, ref expression) => { + TyKind::Array(ref ty, ref expression) => { visitor.visit_ty(ty); visitor.visit_expr(expression) } @@ -434,7 +435,7 @@ pub fn walk_pat<V: Visitor>(visitor: &mut V, pattern: &Pat) { visitor.visit_expr(upper_bound) } PatKind::Wild => (), - PatKind::Vec(ref prepatterns, ref slice_pattern, ref postpatterns) => { + PatKind::Slice(ref prepatterns, ref slice_pattern, ref postpatterns) => { walk_list!(visitor, visit_pat, prepatterns); walk_list!(visitor, visit_pat, slice_pattern); walk_list!(visitor, visit_pat, postpatterns); @@ -474,6 +475,7 @@ pub fn walk_generics<V: Visitor>(visitor: &mut V, generics: &Generics) { visitor.visit_ident(param.span, param.ident); walk_list!(visitor, visit_ty_param_bound, ¶m.bounds); walk_list!(visitor, visit_ty, ¶m.default); + walk_list!(visitor, visit_attribute, &*param.attrs); } walk_list!(visitor, visit_lifetime_def, &generics.lifetimes); for predicate in &generics.where_clause.predicates { @@ -532,8 +534,8 @@ pub fn walk_fn_kind<V: Visitor>(visitor: &mut V, function_kind: FnKind) { pub fn walk_fn<V>(visitor: &mut V, kind: FnKind, declaration: &FnDecl, body: &Block, _span: Span) where V: Visitor, { - walk_fn_decl(visitor, declaration); walk_fn_kind(visitor, kind); + walk_fn_decl(visitor, declaration); visitor.visit_block(body) } @@ -652,13 +654,13 @@ pub fn walk_expr<V: Visitor>(visitor: &mut V, expression: &Expr) { walk_list!(visitor, visit_expr, subexpressions); } ExprKind::Call(ref callee_expression, ref arguments) => { + visitor.visit_expr(callee_expression); walk_list!(visitor, visit_expr, arguments); - visitor.visit_expr(callee_expression) } ExprKind::MethodCall(ref ident, ref types, ref arguments) => { visitor.visit_ident(ident.span, ident.node); - walk_list!(visitor, visit_expr, arguments); walk_list!(visitor, visit_ty, types); + walk_list!(visitor, visit_expr, arguments); } ExprKind::Binary(_, ref left_expression, ref right_expression) => { visitor.visit_expr(left_expression); @@ -717,12 +719,12 @@ pub fn walk_expr<V: Visitor>(visitor: &mut V, expression: &Expr) { } ExprKind::Block(ref block) => visitor.visit_block(block), ExprKind::Assign(ref left_hand_expression, ref right_hand_expression) => { + visitor.visit_expr(left_hand_expression); visitor.visit_expr(right_hand_expression); - visitor.visit_expr(left_hand_expression) } ExprKind::AssignOp(_, ref left_expression, ref right_expression) => { + visitor.visit_expr(left_expression); visitor.visit_expr(right_expression); - visitor.visit_expr(left_expression) } ExprKind::Field(ref subexpression, ref ident) => { visitor.visit_expr(subexpression); |