diff options
Diffstat (limited to 'src/libsyntax')
46 files changed, 1536 insertions, 9843 deletions
diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs index 023952042e6..51a62cd0658 100644 --- a/src/libsyntax/ast.rs +++ b/src/libsyntax/ast.rs @@ -2,31 +2,29 @@ pub use GenericArgs::*; pub use UnsafeSource::*; -pub use crate::symbol::{Ident, Symbol as Name}; pub use crate::util::parser::ExprPrecedence; -use crate::ext::hygiene::ExpnId; use crate::parse::token::{self, DelimToken}; -use crate::print::pprust; use crate::ptr::P; use crate::source_map::{dummy_spanned, respan, Spanned}; -use crate::symbol::{kw, sym, Symbol}; use crate::tokenstream::TokenStream; -use crate::ThinVec; + +use rustc_target::spec::abi::Abi; +pub use rustc_target::abi::FloatTy; + +use syntax_pos::{Span, DUMMY_SP, ExpnId}; +use syntax_pos::symbol::{kw, sym, Symbol}; +pub use syntax_pos::symbol::{Ident, Symbol as Name}; use rustc_index::vec::Idx; #[cfg(target_arch = "x86_64")] use rustc_data_structures::static_assert_size; -use rustc_target::spec::abi::Abi; -use syntax_pos::{Span, DUMMY_SP}; - use rustc_data_structures::fx::FxHashSet; use rustc_data_structures::sync::Lrc; +use rustc_data_structures::thin_vec::ThinVec; use rustc_serialize::{self, Decoder, Encoder}; use std::fmt; -pub use rustc_target::abi::FloatTy; - #[cfg(test)] mod tests; @@ -70,7 +68,7 @@ impl fmt::Display for Lifetime { /// along with a bunch of supporting information. /// /// E.g., `std::cmp::PartialEq`. -#[derive(Clone, RustcEncodable, RustcDecodable)] +#[derive(Clone, RustcEncodable, RustcDecodable, Debug)] pub struct Path { pub span: Span, /// The segments in the path: the things separated by `::`. @@ -86,18 +84,6 @@ impl PartialEq<Symbol> for Path { } } -impl fmt::Debug for Path { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "path({})", pprust::path_to_string(self)) - } -} - -impl fmt::Display for Path { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", pprust::path_to_string(self)) - } -} - impl Path { // Convert a span and an identifier to the corresponding // one-segment path. @@ -507,19 +493,13 @@ pub struct Block { pub span: Span, } -#[derive(Clone, RustcEncodable, RustcDecodable)] +#[derive(Clone, RustcEncodable, RustcDecodable, Debug)] pub struct Pat { pub id: NodeId, pub kind: PatKind, pub span: Span, } -impl fmt::Debug for Pat { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "pat({}: {})", self.id, pprust::pat_to_string(self)) - } -} - impl Pat { /// Attempt reparsing the pattern as a type. /// This is intended for use by diagnostics. @@ -831,7 +811,7 @@ impl UnOp { } /// A statement -#[derive(Clone, RustcEncodable, RustcDecodable)] +#[derive(Clone, RustcEncodable, RustcDecodable, Debug)] pub struct Stmt { pub id: NodeId, pub kind: StmtKind, @@ -865,18 +845,7 @@ impl Stmt { } } -impl fmt::Debug for Stmt { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!( - f, - "stmt({}: {})", - self.id.to_string(), - pprust::stmt_to_string(self) - ) - } -} - -#[derive(Clone, RustcEncodable, RustcDecodable)] +#[derive(Clone, RustcEncodable, RustcDecodable, Debug)] pub enum StmtKind { /// A local (let) binding. Local(P<Local>), @@ -973,7 +942,7 @@ pub struct AnonConst { } /// An expression. -#[derive(Clone, RustcEncodable, RustcDecodable)] +#[derive(Clone, RustcEncodable, RustcDecodable, Debug)] pub struct Expr { pub id: NodeId, pub kind: ExprKind, @@ -1100,12 +1069,6 @@ impl Expr { } } -impl fmt::Debug for Expr { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "expr({}: {})", self.id, pprust::expr_to_string(self)) - } -} - /// Limit types of a range (inclusive or exclusive) #[derive(Copy, Clone, PartialEq, RustcEncodable, RustcDecodable, Debug)] pub enum RangeLimits { @@ -1342,6 +1305,7 @@ impl MacroDef { } } +// Clippy uses Hash and PartialEq #[derive(Clone, RustcEncodable, RustcDecodable, Debug, Copy, Hash, PartialEq)] pub enum StrStyle { /// A regular string, like `"foo"`. @@ -1364,6 +1328,7 @@ pub struct Lit { pub span: Span, } +// Clippy uses Hash and PartialEq #[derive(Clone, RustcEncodable, RustcDecodable, Debug, Copy, Hash, PartialEq)] pub enum LitIntType { Signed(IntTy), @@ -1374,6 +1339,7 @@ pub enum LitIntType { /// Literal kind. /// /// E.g., `"foo"`, `42`, `12.34`, or `bool`. +// Clippy uses Hash and PartialEq #[derive(Clone, RustcEncodable, RustcDecodable, Debug, Hash, PartialEq)] pub enum LitKind { /// A string literal (`"foo"`). @@ -1660,19 +1626,13 @@ pub enum AssocTyConstraintKind { }, } -#[derive(Clone, RustcEncodable, RustcDecodable)] +#[derive(Clone, RustcEncodable, RustcDecodable, Debug)] pub struct Ty { pub id: NodeId, pub kind: TyKind, pub span: Span, } -impl fmt::Debug for Ty { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "type({})", pprust::ty_to_string(self)) - } -} - #[derive(Clone, RustcEncodable, RustcDecodable, Debug)] pub struct BareFnTy { pub unsafety: Unsafety, diff --git a/src/libsyntax/attr/builtin.rs b/src/libsyntax/attr/builtin.rs index 2a8e6b2cc95..84c86c9651f 100644 --- a/src/libsyntax/attr/builtin.rs +++ b/src/libsyntax/attr/builtin.rs @@ -2,9 +2,9 @@ use crate::ast::{self, Attribute, MetaItem, NestedMetaItem}; use crate::early_buffered_lints::BufferedEarlyLintId; -use crate::ext::base::ExtCtxt; use crate::feature_gate::{Features, GatedCfg}; -use crate::parse::ParseSess; +use crate::print::pprust; +use crate::sess::ParseSess; use errors::{Applicability, Handler}; use syntax_pos::hygiene::Transparency; @@ -31,6 +31,10 @@ pub struct AttributeTemplate { } impl AttributeTemplate { + pub fn only_word() -> Self { + Self { word: true, list: None, name_value_str: None } + } + /// Checks that the given meta-item is compatible with this template. fn compatible(&self, meta_item_kind: &ast::MetaItemKind) -> bool { match meta_item_kind { @@ -80,7 +84,7 @@ fn handle_errors(sess: &ParseSess, span: Span, error: AttrError) { } } -#[derive(Copy, Clone, Hash, PartialEq, RustcEncodable, RustcDecodable)] +#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable)] pub enum InlineAttr { None, Hint, @@ -88,7 +92,7 @@ pub enum InlineAttr { Never, } -#[derive(Copy, Clone, Hash, PartialEq, RustcEncodable, RustcDecodable)] +#[derive(Clone, RustcEncodable, RustcDecodable)] pub enum OptimizeAttr { None, Speed, @@ -243,7 +247,11 @@ fn find_stability_generic<'a, I>(sess: &ParseSess, let meta = meta.as_ref().unwrap(); let get = |meta: &MetaItem, item: &mut Option<Symbol>| { if item.is_some() { - handle_errors(sess, meta.span, AttrError::MultipleItem(meta.path.to_string())); + handle_errors( + sess, + meta.span, + AttrError::MultipleItem(pprust::path_to_string(&meta.path)), + ); return false } if let Some(v) = meta.value_str() { @@ -271,7 +279,10 @@ fn find_stability_generic<'a, I>(sess: &ParseSess, handle_errors( sess, mi.span, - AttrError::UnknownMetaItem(mi.path.to_string(), expected), + AttrError::UnknownMetaItem( + pprust::path_to_string(&mi.path), + expected, + ), ); continue 'outer } @@ -362,7 +373,7 @@ fn find_stability_generic<'a, I>(sess: &ParseSess, sess, meta.span(), AttrError::UnknownMetaItem( - mi.path.to_string(), + pprust::path_to_string(&mi.path), &["feature", "reason", "issue", "soft"] ), ); @@ -434,7 +445,8 @@ fn find_stability_generic<'a, I>(sess: &ParseSess, sess, meta.span(), AttrError::UnknownMetaItem( - mi.path.to_string(), &["since", "note"], + pprust::path_to_string(&mi.path), + &["since", "note"], ), ); continue 'outer @@ -597,8 +609,11 @@ pub fn eval_condition<F>(cfg: &ast::MetaItem, sess: &ParseSess, eval: &mut F) !eval_condition(mis[0].meta_item().unwrap(), sess, eval) }, _ => { - span_err!(sess.span_diagnostic, cfg.span, E0537, - "invalid predicate `{}`", cfg.path); + span_err!( + sess.span_diagnostic, cfg.span, E0537, + "invalid predicate `{}`", + pprust::path_to_string(&cfg.path) + ); false } } @@ -609,8 +624,7 @@ pub fn eval_condition<F>(cfg: &ast::MetaItem, sess: &ParseSess, eval: &mut F) } } - -#[derive(RustcEncodable, RustcDecodable, PartialEq, PartialOrd, Clone, Debug, Eq, Hash)] +#[derive(RustcEncodable, RustcDecodable, Clone)] pub struct Deprecation { pub since: Option<Symbol>, pub note: Option<Symbol>, @@ -653,7 +667,9 @@ fn find_deprecation_generic<'a, I>(sess: &ParseSess, let get = |meta: &MetaItem, item: &mut Option<Symbol>| { if item.is_some() { handle_errors( - sess, meta.span, AttrError::MultipleItem(meta.path.to_string()) + sess, + meta.span, + AttrError::MultipleItem(pprust::path_to_string(&meta.path)), ); return false } @@ -691,8 +707,10 @@ fn find_deprecation_generic<'a, I>(sess: &ParseSess, handle_errors( sess, meta.span(), - AttrError::UnknownMetaItem(mi.path.to_string(), - &["since", "note"]), + AttrError::UnknownMetaItem( + pprust::path_to_string(&mi.path), + &["since", "note"], + ), ); continue 'outer } @@ -730,7 +748,7 @@ pub enum ReprAttr { ReprAlign(u32), } -#[derive(Eq, Hash, PartialEq, Debug, RustcEncodable, RustcDecodable, Copy, Clone)] +#[derive(Eq, PartialEq, Debug, RustcEncodable, RustcDecodable, Copy, Clone)] pub enum IntType { SignedInt(ast::IntTy), UnsignedInt(ast::UintTy) @@ -921,14 +939,7 @@ pub fn find_transparency( (transparency.map_or(fallback, |t| t.0), error) } -pub fn check_builtin_macro_attribute(ecx: &ExtCtxt<'_>, meta_item: &MetaItem, name: Symbol) { - // All the built-in macro attributes are "words" at the moment. - let template = AttributeTemplate { word: true, list: None, name_value_str: None }; - let attr = ecx.attribute(meta_item.clone()); - check_builtin_attribute(ecx.parse_sess, &attr, name, template); -} - -crate fn check_builtin_attribute( +pub fn check_builtin_attribute( sess: &ParseSess, attr: &ast::Attribute, name: Symbol, template: AttributeTemplate ) { // Some special attributes like `cfg` must be checked diff --git a/src/libsyntax/attr/mod.rs b/src/libsyntax/attr/mod.rs index 7bef693a5be..4aec5040881 100644 --- a/src/libsyntax/attr/mod.rs +++ b/src/libsyntax/attr/mod.rs @@ -16,12 +16,13 @@ use crate::mut_visit::visit_clobber; use crate::source_map::{BytePos, Spanned, DUMMY_SP}; use crate::parse::lexer::comments::{doc_comment_style, strip_doc_comment_decoration}; use crate::parse::parser::Parser; -use crate::parse::{ParseSess, PResult}; +use crate::parse::PResult; use crate::parse::token::{self, Token}; use crate::ptr::P; +use crate::sess::ParseSess; use crate::symbol::{sym, Symbol}; use crate::ThinVec; -use crate::tokenstream::{TokenStream, TokenTree, DelimSpan}; +use crate::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndJoint}; use crate::GLOBALS; use log::debug; @@ -279,7 +280,7 @@ impl Attribute { self.item.meta(self.span) } - pub fn parse<'a, T, F>(&self, sess: &'a ParseSess, mut f: F) -> PResult<'a, T> + crate fn parse<'a, T, F>(&self, sess: &'a ParseSess, mut f: F) -> PResult<'a, T> where F: FnMut(&mut Parser<'a>) -> PResult<'a, T>, { let mut parser = Parser::new( @@ -297,24 +298,11 @@ impl Attribute { Ok(result) } - pub fn parse_list<'a, T, F>(&self, sess: &'a ParseSess, mut f: F) -> PResult<'a, Vec<T>> - where F: FnMut(&mut Parser<'a>) -> PResult<'a, T>, - { + pub fn parse_derive_paths<'a>(&self, sess: &'a ParseSess) -> PResult<'a, Vec<Path>> { if self.tokens.is_empty() { return Ok(Vec::new()); } - self.parse(sess, |parser| { - parser.expect(&token::OpenDelim(token::Paren))?; - let mut list = Vec::new(); - while !parser.eat(&token::CloseDelim(token::Paren)) { - list.push(f(parser)?); - if !parser.eat(&token::Comma) { - parser.expect(&token::CloseDelim(token::Paren))?; - break - } - } - Ok(list) - }) + self.parse(sess, |p| p.parse_derive_paths()) } pub fn parse_meta<'a>(&self, sess: &'a ParseSess) -> PResult<'a, MetaItem> { @@ -475,7 +463,7 @@ pub fn first_attr_value_str_by_name(attrs: &[Attribute], name: Symbol) -> Option } impl MetaItem { - fn tokens(&self) -> TokenStream { + fn token_trees_and_joints(&self) -> Vec<TreeAndJoint> { let mut idents = vec![]; let mut last_pos = BytePos(0 as u32); for (i, segment) in self.path.segments.iter().enumerate() { @@ -489,8 +477,8 @@ impl MetaItem { idents.push(TokenTree::Token(Token::from_ast_ident(segment.ident)).into()); last_pos = segment.ident.span.hi(); } - self.kind.tokens(self.span).append_to_tree_and_joint_vec(&mut idents); - TokenStream::new(idents) + idents.extend(self.kind.token_trees_and_joints(self.span)); + idents } fn from_tokens<I>(tokens: &mut iter::Peekable<I>) -> Option<MetaItem> @@ -549,13 +537,14 @@ impl MetaItem { } impl MetaItemKind { - pub fn tokens(&self, span: Span) -> TokenStream { + pub fn token_trees_and_joints(&self, span: Span) -> Vec<TreeAndJoint> { match *self { - MetaItemKind::Word => TokenStream::empty(), + MetaItemKind::Word => vec![], MetaItemKind::NameValue(ref lit) => { - let mut vec = vec![TokenTree::token(token::Eq, span).into()]; - lit.tokens().append_to_tree_and_joint_vec(&mut vec); - TokenStream::new(vec) + vec![ + TokenTree::token(token::Eq, span).into(), + lit.token_tree().into(), + ] } MetaItemKind::List(ref list) => { let mut tokens = Vec::new(); @@ -563,17 +552,26 @@ impl MetaItemKind { if i > 0 { tokens.push(TokenTree::token(token::Comma, span).into()); } - item.tokens().append_to_tree_and_joint_vec(&mut tokens); + tokens.extend(item.token_trees_and_joints()) } - TokenTree::Delimited( - DelimSpan::from_single(span), - token::Paren, - TokenStream::new(tokens).into(), - ).into() + vec![ + TokenTree::Delimited( + DelimSpan::from_single(span), + token::Paren, + TokenStream::new(tokens).into(), + ).into() + ] } } } + // Premature conversions of `TokenTree`s to `TokenStream`s can hurt + // performance. Do not use this function if `token_trees_and_joints()` can + // be used instead. + pub fn tokens(&self, span: Span) -> TokenStream { + TokenStream::new(self.token_trees_and_joints(span)) + } + fn from_tokens<I>(tokens: &mut iter::Peekable<I>) -> Option<MetaItemKind> where I: Iterator<Item = TokenTree>, { @@ -615,10 +613,10 @@ impl NestedMetaItem { } } - fn tokens(&self) -> TokenStream { + fn token_trees_and_joints(&self) -> Vec<TreeAndJoint> { match *self { - NestedMetaItem::MetaItem(ref item) => item.tokens(), - NestedMetaItem::Literal(ref lit) => lit.tokens(), + NestedMetaItem::MetaItem(ref item) => item.token_trees_and_joints(), + NestedMetaItem::Literal(ref lit) => vec![lit.token_tree().into()], } } diff --git a/src/libsyntax/config.rs b/src/libsyntax/config.rs index 2923cc86ba0..54dc95291d6 100644 --- a/src/libsyntax/config.rs +++ b/src/libsyntax/config.rs @@ -10,8 +10,8 @@ use crate::attr; use crate::ast; use crate::edition::Edition; use crate::mut_visit::*; -use crate::parse::{token, ParseSess}; use crate::ptr::P; +use crate::sess::ParseSess; use crate::symbol::sym; use crate::util::map_in_place::MapInPlace; @@ -56,6 +56,7 @@ pub fn features(mut krate: ast::Crate, sess: &ParseSess, edition: Edition, (krate, features) } +#[macro_export] macro_rules! configure { ($this:ident, $node:ident) => { match $this.configure($node) { @@ -111,25 +112,7 @@ impl<'a> StripUnconfigured<'a> { return vec![]; } - let (cfg_predicate, expanded_attrs) = match attr.parse(self.sess, |parser| { - parser.expect(&token::OpenDelim(token::Paren))?; - - let cfg_predicate = parser.parse_meta_item()?; - parser.expect(&token::Comma)?; - - // Presumably, the majority of the time there will only be one attr. - let mut expanded_attrs = Vec::with_capacity(1); - - while !parser.check(&token::CloseDelim(token::Paren)) { - let lo = parser.token.span.lo(); - let item = parser.parse_attr_item()?; - expanded_attrs.push((item, parser.prev_span.with_lo(lo))); - parser.expect_one_of(&[token::Comma], &[token::CloseDelim(token::Paren)])?; - } - - parser.expect(&token::CloseDelim(token::Paren))?; - Ok((cfg_predicate, expanded_attrs)) - }) { + let (cfg_predicate, expanded_attrs) = match attr.parse(self.sess, |p| p.parse_cfg_attr()) { Ok(result) => result, Err(mut e) => { e.emit(); diff --git a/src/libsyntax/error_codes.rs b/src/libsyntax/error_codes.rs index fc3f095856a..17ea4767520 100644 --- a/src/libsyntax/error_codes.rs +++ b/src/libsyntax/error_codes.rs @@ -295,6 +295,33 @@ named `file_that_doesnt_exist.rs` or `file_that_doesnt_exist/mod.rs` in the same directory. "##, +E0584: r##" +A doc comment that is not attached to anything has been encountered. + +Erroneous code example: + +```compile_fail,E0584 +trait Island { + fn lost(); + + /// I'm lost! +} +``` + +A little reminder: a doc comment has to be placed before the item it's supposed +to document. So if you want to document the `Island` trait, you need to put a +doc comment before it, not inside it. Same goes for the `lost` method: the doc +comment needs to be before it: + +``` +/// I'm THE island! +trait Island { + /// I'm lost! + fn lost(); +} +``` +"##, + E0585: r##" A documentation comment that doesn't document anything was found. @@ -494,7 +521,6 @@ features in the `-Z allow_features` flag. E0549, E0553, // multiple rustc_const_unstable attributes // E0555, // replaced with a generic attribute input check - E0584, // file for module `..` found at both .. and .. E0629, // missing 'feature' (rustc_const_unstable) // rustc_const_unstable attribute must be paired with stable/unstable // attribute diff --git a/src/libsyntax/ext/allocator.rs b/src/libsyntax/ext/allocator.rs deleted file mode 100644 index 99aeb5414c5..00000000000 --- a/src/libsyntax/ext/allocator.rs +++ /dev/null @@ -1,75 +0,0 @@ -use crate::{ast, attr, visit}; -use crate::symbol::{sym, Symbol}; -use syntax_pos::Span; - -#[derive(Clone, Copy)] -pub enum AllocatorKind { - Global, - DefaultLib, - DefaultExe, -} - -impl AllocatorKind { - pub fn fn_name(&self, base: &str) -> String { - match *self { - AllocatorKind::Global => format!("__rg_{}", base), - AllocatorKind::DefaultLib => format!("__rdl_{}", base), - AllocatorKind::DefaultExe => format!("__rde_{}", base), - } - } -} - -pub enum AllocatorTy { - Layout, - Ptr, - ResultPtr, - Unit, - Usize, -} - -pub struct AllocatorMethod { - pub name: &'static str, - pub inputs: &'static [AllocatorTy], - pub output: AllocatorTy, -} - -pub static ALLOCATOR_METHODS: &[AllocatorMethod] = &[ - AllocatorMethod { - name: "alloc", - inputs: &[AllocatorTy::Layout], - output: AllocatorTy::ResultPtr, - }, - AllocatorMethod { - name: "dealloc", - inputs: &[AllocatorTy::Ptr, AllocatorTy::Layout], - output: AllocatorTy::Unit, - }, - AllocatorMethod { - name: "realloc", - inputs: &[AllocatorTy::Ptr, AllocatorTy::Layout, AllocatorTy::Usize], - output: AllocatorTy::ResultPtr, - }, - AllocatorMethod { - name: "alloc_zeroed", - inputs: &[AllocatorTy::Layout], - output: AllocatorTy::ResultPtr, - }, -]; - -pub fn global_allocator_spans(krate: &ast::Crate) -> Vec<Span> { - struct Finder { name: Symbol, spans: Vec<Span> } - impl<'ast> visit::Visitor<'ast> for Finder { - fn visit_item(&mut self, item: &'ast ast::Item) { - if item.ident.name == self.name && - attr::contains_name(&item.attrs, sym::rustc_std_internal_symbol) { - self.spans.push(item.span); - } - visit::walk_item(self, item) - } - } - - let name = Symbol::intern(&AllocatorKind::Global.fn_name("alloc")); - let mut f = Finder { name, spans: Vec::new() }; - visit::walk_crate(&mut f, krate); - f.spans -} diff --git a/src/libsyntax/ext/base.rs b/src/libsyntax/ext/base.rs deleted file mode 100644 index 583fb3f7701..00000000000 --- a/src/libsyntax/ext/base.rs +++ /dev/null @@ -1,1187 +0,0 @@ -use crate::ast::{self, NodeId, Attribute, Name, PatKind}; -use crate::attr::{self, HasAttrs, Stability, Deprecation}; -use crate::source_map::SourceMap; -use crate::edition::Edition; -use crate::ext::expand::{self, AstFragment, Invocation}; -use crate::ext::hygiene::ExpnId; -use crate::mut_visit::{self, MutVisitor}; -use crate::parse::{self, parser, ParseSess, DirectoryOwnership}; -use crate::parse::token; -use crate::ptr::P; -use crate::symbol::{kw, sym, Ident, Symbol}; -use crate::{ThinVec, MACRO_ARGUMENTS}; -use crate::tokenstream::{self, TokenStream}; -use crate::visit::Visitor; - -use errors::{DiagnosticBuilder, DiagnosticId}; -use smallvec::{smallvec, SmallVec}; -use syntax_pos::{FileName, Span, MultiSpan, DUMMY_SP}; -use syntax_pos::hygiene::{AstPass, ExpnData, ExpnKind}; - -use rustc_data_structures::fx::FxHashMap; -use rustc_data_structures::sync::{self, Lrc}; -use std::iter; -use std::path::PathBuf; -use std::rc::Rc; -use std::default::Default; - -pub use syntax_pos::hygiene::MacroKind; - -#[derive(Debug,Clone)] -pub enum Annotatable { - Item(P<ast::Item>), - TraitItem(P<ast::TraitItem>), - ImplItem(P<ast::ImplItem>), - ForeignItem(P<ast::ForeignItem>), - Stmt(P<ast::Stmt>), - Expr(P<ast::Expr>), - Arm(ast::Arm), - Field(ast::Field), - FieldPat(ast::FieldPat), - GenericParam(ast::GenericParam), - Param(ast::Param), - StructField(ast::StructField), - Variant(ast::Variant), -} - -impl HasAttrs for Annotatable { - fn attrs(&self) -> &[Attribute] { - match *self { - Annotatable::Item(ref item) => &item.attrs, - Annotatable::TraitItem(ref trait_item) => &trait_item.attrs, - Annotatable::ImplItem(ref impl_item) => &impl_item.attrs, - Annotatable::ForeignItem(ref foreign_item) => &foreign_item.attrs, - Annotatable::Stmt(ref stmt) => stmt.attrs(), - Annotatable::Expr(ref expr) => &expr.attrs, - Annotatable::Arm(ref arm) => &arm.attrs, - Annotatable::Field(ref field) => &field.attrs, - Annotatable::FieldPat(ref fp) => &fp.attrs, - Annotatable::GenericParam(ref gp) => &gp.attrs, - Annotatable::Param(ref p) => &p.attrs, - Annotatable::StructField(ref sf) => &sf.attrs, - Annotatable::Variant(ref v) => &v.attrs(), - } - } - - fn visit_attrs<F: FnOnce(&mut Vec<Attribute>)>(&mut self, f: F) { - match self { - Annotatable::Item(item) => item.visit_attrs(f), - Annotatable::TraitItem(trait_item) => trait_item.visit_attrs(f), - Annotatable::ImplItem(impl_item) => impl_item.visit_attrs(f), - Annotatable::ForeignItem(foreign_item) => foreign_item.visit_attrs(f), - Annotatable::Stmt(stmt) => stmt.visit_attrs(f), - Annotatable::Expr(expr) => expr.visit_attrs(f), - Annotatable::Arm(arm) => arm.visit_attrs(f), - Annotatable::Field(field) => field.visit_attrs(f), - Annotatable::FieldPat(fp) => fp.visit_attrs(f), - Annotatable::GenericParam(gp) => gp.visit_attrs(f), - Annotatable::Param(p) => p.visit_attrs(f), - Annotatable::StructField(sf) => sf.visit_attrs(f), - Annotatable::Variant(v) => v.visit_attrs(f), - } - } -} - -impl Annotatable { - pub fn span(&self) -> Span { - match *self { - Annotatable::Item(ref item) => item.span, - Annotatable::TraitItem(ref trait_item) => trait_item.span, - Annotatable::ImplItem(ref impl_item) => impl_item.span, - Annotatable::ForeignItem(ref foreign_item) => foreign_item.span, - Annotatable::Stmt(ref stmt) => stmt.span, - Annotatable::Expr(ref expr) => expr.span, - Annotatable::Arm(ref arm) => arm.span, - Annotatable::Field(ref field) => field.span, - Annotatable::FieldPat(ref fp) => fp.pat.span, - Annotatable::GenericParam(ref gp) => gp.ident.span, - Annotatable::Param(ref p) => p.span, - Annotatable::StructField(ref sf) => sf.span, - Annotatable::Variant(ref v) => v.span, - } - } - - pub fn visit_with<'a, V: Visitor<'a>>(&'a self, visitor: &mut V) { - match self { - Annotatable::Item(item) => visitor.visit_item(item), - Annotatable::TraitItem(trait_item) => visitor.visit_trait_item(trait_item), - Annotatable::ImplItem(impl_item) => visitor.visit_impl_item(impl_item), - Annotatable::ForeignItem(foreign_item) => visitor.visit_foreign_item(foreign_item), - Annotatable::Stmt(stmt) => visitor.visit_stmt(stmt), - Annotatable::Expr(expr) => visitor.visit_expr(expr), - Annotatable::Arm(arm) => visitor.visit_arm(arm), - Annotatable::Field(field) => visitor.visit_field(field), - Annotatable::FieldPat(fp) => visitor.visit_field_pattern(fp), - Annotatable::GenericParam(gp) => visitor.visit_generic_param(gp), - Annotatable::Param(p) => visitor.visit_param(p), - Annotatable::StructField(sf) =>visitor.visit_struct_field(sf), - Annotatable::Variant(v) => visitor.visit_variant(v), - } - } - - pub fn expect_item(self) -> P<ast::Item> { - match self { - Annotatable::Item(i) => i, - _ => panic!("expected Item") - } - } - - pub fn map_item_or<F, G>(self, mut f: F, mut or: G) -> Annotatable - where F: FnMut(P<ast::Item>) -> P<ast::Item>, - G: FnMut(Annotatable) -> Annotatable - { - match self { - Annotatable::Item(i) => Annotatable::Item(f(i)), - _ => or(self) - } - } - - pub fn expect_trait_item(self) -> ast::TraitItem { - match self { - Annotatable::TraitItem(i) => i.into_inner(), - _ => panic!("expected Item") - } - } - - pub fn expect_impl_item(self) -> ast::ImplItem { - match self { - Annotatable::ImplItem(i) => i.into_inner(), - _ => panic!("expected Item") - } - } - - pub fn expect_foreign_item(self) -> ast::ForeignItem { - match self { - Annotatable::ForeignItem(i) => i.into_inner(), - _ => panic!("expected foreign item") - } - } - - pub fn expect_stmt(self) -> ast::Stmt { - match self { - Annotatable::Stmt(stmt) => stmt.into_inner(), - _ => panic!("expected statement"), - } - } - - pub fn expect_expr(self) -> P<ast::Expr> { - match self { - Annotatable::Expr(expr) => expr, - _ => panic!("expected expression"), - } - } - - pub fn expect_arm(self) -> ast::Arm { - match self { - Annotatable::Arm(arm) => arm, - _ => panic!("expected match arm") - } - } - - pub fn expect_field(self) -> ast::Field { - match self { - Annotatable::Field(field) => field, - _ => panic!("expected field") - } - } - - pub fn expect_field_pattern(self) -> ast::FieldPat { - match self { - Annotatable::FieldPat(fp) => fp, - _ => panic!("expected field pattern") - } - } - - pub fn expect_generic_param(self) -> ast::GenericParam { - match self { - Annotatable::GenericParam(gp) => gp, - _ => panic!("expected generic parameter") - } - } - - pub fn expect_param(self) -> ast::Param { - match self { - Annotatable::Param(param) => param, - _ => panic!("expected parameter") - } - } - - pub fn expect_struct_field(self) -> ast::StructField { - match self { - Annotatable::StructField(sf) => sf, - _ => panic!("expected struct field") - } - } - - pub fn expect_variant(self) -> ast::Variant { - match self { - Annotatable::Variant(v) => v, - _ => panic!("expected variant") - } - } - - pub fn derive_allowed(&self) -> bool { - match *self { - Annotatable::Item(ref item) => match item.kind { - ast::ItemKind::Struct(..) | - ast::ItemKind::Enum(..) | - ast::ItemKind::Union(..) => true, - _ => false, - }, - _ => false, - } - } -} - -// `meta_item` is the annotation, and `item` is the item being modified. -// FIXME Decorators should follow the same pattern too. -pub trait MultiItemModifier { - fn expand(&self, - ecx: &mut ExtCtxt<'_>, - span: Span, - meta_item: &ast::MetaItem, - item: Annotatable) - -> Vec<Annotatable>; -} - -impl<F, T> MultiItemModifier for F - where F: Fn(&mut ExtCtxt<'_>, Span, &ast::MetaItem, Annotatable) -> T, - T: Into<Vec<Annotatable>>, -{ - fn expand(&self, - ecx: &mut ExtCtxt<'_>, - span: Span, - meta_item: &ast::MetaItem, - item: Annotatable) - -> Vec<Annotatable> { - (*self)(ecx, span, meta_item, item).into() - } -} - -impl Into<Vec<Annotatable>> for Annotatable { - fn into(self) -> Vec<Annotatable> { - vec![self] - } -} - -pub trait ProcMacro { - fn expand<'cx>(&self, - ecx: &'cx mut ExtCtxt<'_>, - span: Span, - ts: TokenStream) - -> TokenStream; -} - -impl<F> ProcMacro for F - where F: Fn(TokenStream) -> TokenStream -{ - fn expand<'cx>(&self, - _ecx: &'cx mut ExtCtxt<'_>, - _span: Span, - ts: TokenStream) - -> TokenStream { - // FIXME setup implicit context in TLS before calling self. - (*self)(ts) - } -} - -pub trait AttrProcMacro { - fn expand<'cx>(&self, - ecx: &'cx mut ExtCtxt<'_>, - span: Span, - annotation: TokenStream, - annotated: TokenStream) - -> TokenStream; -} - -impl<F> AttrProcMacro for F - where F: Fn(TokenStream, TokenStream) -> TokenStream -{ - fn expand<'cx>(&self, - _ecx: &'cx mut ExtCtxt<'_>, - _span: Span, - annotation: TokenStream, - annotated: TokenStream) - -> TokenStream { - // FIXME setup implicit context in TLS before calling self. - (*self)(annotation, annotated) - } -} - -/// Represents a thing that maps token trees to Macro Results -pub trait TTMacroExpander { - fn expand<'cx>( - &self, - ecx: &'cx mut ExtCtxt<'_>, - span: Span, - input: TokenStream, - ) -> Box<dyn MacResult+'cx>; -} - -pub type MacroExpanderFn = - for<'cx> fn(&'cx mut ExtCtxt<'_>, Span, TokenStream) - -> Box<dyn MacResult+'cx>; - -impl<F> TTMacroExpander for F - where F: for<'cx> Fn(&'cx mut ExtCtxt<'_>, Span, TokenStream) - -> Box<dyn MacResult+'cx> -{ - fn expand<'cx>( - &self, - ecx: &'cx mut ExtCtxt<'_>, - span: Span, - mut input: TokenStream, - ) -> Box<dyn MacResult+'cx> { - struct AvoidInterpolatedIdents; - - impl MutVisitor for AvoidInterpolatedIdents { - fn visit_tt(&mut self, tt: &mut tokenstream::TokenTree) { - if let tokenstream::TokenTree::Token(token) = tt { - if let token::Interpolated(nt) = &token.kind { - if let token::NtIdent(ident, is_raw) = **nt { - *tt = tokenstream::TokenTree::token( - token::Ident(ident.name, is_raw), ident.span - ); - } - } - } - mut_visit::noop_visit_tt(tt, self) - } - - fn visit_mac(&mut self, mac: &mut ast::Mac) { - mut_visit::noop_visit_mac(mac, self) - } - } - AvoidInterpolatedIdents.visit_tts(&mut input); - (*self)(ecx, span, input) - } -} - -// Use a macro because forwarding to a simple function has type system issues -macro_rules! make_stmts_default { - ($me:expr) => { - $me.make_expr().map(|e| smallvec![ast::Stmt { - id: ast::DUMMY_NODE_ID, - span: e.span, - kind: ast::StmtKind::Expr(e), - }]) - } -} - -/// The result of a macro expansion. The return values of the various -/// methods are spliced into the AST at the callsite of the macro. -pub trait MacResult { - /// Creates an expression. - fn make_expr(self: Box<Self>) -> Option<P<ast::Expr>> { - None - } - /// Creates zero or more items. - fn make_items(self: Box<Self>) -> Option<SmallVec<[P<ast::Item>; 1]>> { - None - } - - /// Creates zero or more impl items. - fn make_impl_items(self: Box<Self>) -> Option<SmallVec<[ast::ImplItem; 1]>> { - None - } - - /// Creates zero or more trait items. - fn make_trait_items(self: Box<Self>) -> Option<SmallVec<[ast::TraitItem; 1]>> { - None - } - - /// Creates zero or more items in an `extern {}` block - fn make_foreign_items(self: Box<Self>) -> Option<SmallVec<[ast::ForeignItem; 1]>> { None } - - /// Creates a pattern. - fn make_pat(self: Box<Self>) -> Option<P<ast::Pat>> { - None - } - - /// Creates zero or more statements. - /// - /// By default this attempts to create an expression statement, - /// returning None if that fails. - fn make_stmts(self: Box<Self>) -> Option<SmallVec<[ast::Stmt; 1]>> { - make_stmts_default!(self) - } - - fn make_ty(self: Box<Self>) -> Option<P<ast::Ty>> { - None - } - - fn make_arms(self: Box<Self>) -> Option<SmallVec<[ast::Arm; 1]>> { - None - } - - fn make_fields(self: Box<Self>) -> Option<SmallVec<[ast::Field; 1]>> { - None - } - - fn make_field_patterns(self: Box<Self>) -> Option<SmallVec<[ast::FieldPat; 1]>> { - None - } - - fn make_generic_params(self: Box<Self>) -> Option<SmallVec<[ast::GenericParam; 1]>> { - None - } - - fn make_params(self: Box<Self>) -> Option<SmallVec<[ast::Param; 1]>> { - None - } - - fn make_struct_fields(self: Box<Self>) -> Option<SmallVec<[ast::StructField; 1]>> { - None - } - - fn make_variants(self: Box<Self>) -> Option<SmallVec<[ast::Variant; 1]>> { - None - } -} - -macro_rules! make_MacEager { - ( $( $fld:ident: $t:ty, )* ) => { - /// `MacResult` implementation for the common case where you've already - /// built each form of AST that you might return. - #[derive(Default)] - pub struct MacEager { - $( - pub $fld: Option<$t>, - )* - } - - impl MacEager { - $( - pub fn $fld(v: $t) -> Box<dyn MacResult> { - Box::new(MacEager { - $fld: Some(v), - ..Default::default() - }) - } - )* - } - } -} - -make_MacEager! { - expr: P<ast::Expr>, - pat: P<ast::Pat>, - items: SmallVec<[P<ast::Item>; 1]>, - impl_items: SmallVec<[ast::ImplItem; 1]>, - trait_items: SmallVec<[ast::TraitItem; 1]>, - foreign_items: SmallVec<[ast::ForeignItem; 1]>, - stmts: SmallVec<[ast::Stmt; 1]>, - ty: P<ast::Ty>, -} - -impl MacResult for MacEager { - fn make_expr(self: Box<Self>) -> Option<P<ast::Expr>> { - self.expr - } - - fn make_items(self: Box<Self>) -> Option<SmallVec<[P<ast::Item>; 1]>> { - self.items - } - - fn make_impl_items(self: Box<Self>) -> Option<SmallVec<[ast::ImplItem; 1]>> { - self.impl_items - } - - fn make_trait_items(self: Box<Self>) -> Option<SmallVec<[ast::TraitItem; 1]>> { - self.trait_items - } - - fn make_foreign_items(self: Box<Self>) -> Option<SmallVec<[ast::ForeignItem; 1]>> { - self.foreign_items - } - - fn make_stmts(self: Box<Self>) -> Option<SmallVec<[ast::Stmt; 1]>> { - match self.stmts.as_ref().map_or(0, |s| s.len()) { - 0 => make_stmts_default!(self), - _ => self.stmts, - } - } - - fn make_pat(self: Box<Self>) -> Option<P<ast::Pat>> { - if let Some(p) = self.pat { - return Some(p); - } - if let Some(e) = self.expr { - if let ast::ExprKind::Lit(_) = e.kind { - return Some(P(ast::Pat { - id: ast::DUMMY_NODE_ID, - span: e.span, - kind: PatKind::Lit(e), - })); - } - } - None - } - - fn make_ty(self: Box<Self>) -> Option<P<ast::Ty>> { - self.ty - } -} - -/// Fill-in macro expansion result, to allow compilation to continue -/// after hitting errors. -#[derive(Copy, Clone)] -pub struct DummyResult { - is_error: bool, - span: Span, -} - -impl DummyResult { - /// Creates a default MacResult that can be anything. - /// - /// Use this as a return value after hitting any errors and - /// calling `span_err`. - pub fn any(span: Span) -> Box<dyn MacResult+'static> { - Box::new(DummyResult { is_error: true, span }) - } - - /// Same as `any`, but must be a valid fragment, not error. - pub fn any_valid(span: Span) -> Box<dyn MacResult+'static> { - Box::new(DummyResult { is_error: false, span }) - } - - /// A plain dummy expression. - pub fn raw_expr(sp: Span, is_error: bool) -> P<ast::Expr> { - P(ast::Expr { - id: ast::DUMMY_NODE_ID, - kind: if is_error { ast::ExprKind::Err } else { ast::ExprKind::Tup(Vec::new()) }, - span: sp, - attrs: ThinVec::new(), - }) - } - - /// A plain dummy pattern. - pub fn raw_pat(sp: Span) -> ast::Pat { - ast::Pat { - id: ast::DUMMY_NODE_ID, - kind: PatKind::Wild, - span: sp, - } - } - - /// A plain dummy type. - pub fn raw_ty(sp: Span, is_error: bool) -> P<ast::Ty> { - P(ast::Ty { - id: ast::DUMMY_NODE_ID, - kind: if is_error { ast::TyKind::Err } else { ast::TyKind::Tup(Vec::new()) }, - span: sp - }) - } -} - -impl MacResult for DummyResult { - fn make_expr(self: Box<DummyResult>) -> Option<P<ast::Expr>> { - Some(DummyResult::raw_expr(self.span, self.is_error)) - } - - fn make_pat(self: Box<DummyResult>) -> Option<P<ast::Pat>> { - Some(P(DummyResult::raw_pat(self.span))) - } - - fn make_items(self: Box<DummyResult>) -> Option<SmallVec<[P<ast::Item>; 1]>> { - Some(SmallVec::new()) - } - - fn make_impl_items(self: Box<DummyResult>) -> Option<SmallVec<[ast::ImplItem; 1]>> { - Some(SmallVec::new()) - } - - fn make_trait_items(self: Box<DummyResult>) -> Option<SmallVec<[ast::TraitItem; 1]>> { - Some(SmallVec::new()) - } - - fn make_foreign_items(self: Box<Self>) -> Option<SmallVec<[ast::ForeignItem; 1]>> { - Some(SmallVec::new()) - } - - fn make_stmts(self: Box<DummyResult>) -> Option<SmallVec<[ast::Stmt; 1]>> { - Some(smallvec![ast::Stmt { - id: ast::DUMMY_NODE_ID, - kind: ast::StmtKind::Expr(DummyResult::raw_expr(self.span, self.is_error)), - span: self.span, - }]) - } - - fn make_ty(self: Box<DummyResult>) -> Option<P<ast::Ty>> { - Some(DummyResult::raw_ty(self.span, self.is_error)) - } - - fn make_arms(self: Box<DummyResult>) -> Option<SmallVec<[ast::Arm; 1]>> { - Some(SmallVec::new()) - } - - fn make_fields(self: Box<DummyResult>) -> Option<SmallVec<[ast::Field; 1]>> { - Some(SmallVec::new()) - } - - fn make_field_patterns(self: Box<DummyResult>) -> Option<SmallVec<[ast::FieldPat; 1]>> { - Some(SmallVec::new()) - } - - fn make_generic_params(self: Box<DummyResult>) -> Option<SmallVec<[ast::GenericParam; 1]>> { - Some(SmallVec::new()) - } - - fn make_params(self: Box<DummyResult>) -> Option<SmallVec<[ast::Param; 1]>> { - Some(SmallVec::new()) - } - - fn make_struct_fields(self: Box<DummyResult>) -> Option<SmallVec<[ast::StructField; 1]>> { - Some(SmallVec::new()) - } - - fn make_variants(self: Box<DummyResult>) -> Option<SmallVec<[ast::Variant; 1]>> { - Some(SmallVec::new()) - } -} - -/// A syntax extension kind. -pub enum SyntaxExtensionKind { - /// A token-based function-like macro. - Bang( - /// An expander with signature TokenStream -> TokenStream. - Box<dyn ProcMacro + sync::Sync + sync::Send>, - ), - - /// An AST-based function-like macro. - LegacyBang( - /// An expander with signature TokenStream -> AST. - Box<dyn TTMacroExpander + sync::Sync + sync::Send>, - ), - - /// A token-based attribute macro. - Attr( - /// An expander with signature (TokenStream, TokenStream) -> TokenStream. - /// The first TokenSteam is the attribute itself, the second is the annotated item. - /// The produced TokenSteam replaces the input TokenSteam. - Box<dyn AttrProcMacro + sync::Sync + sync::Send>, - ), - - /// An AST-based attribute macro. - LegacyAttr( - /// An expander with signature (AST, AST) -> AST. - /// The first AST fragment is the attribute itself, the second is the annotated item. - /// The produced AST fragment replaces the input AST fragment. - Box<dyn MultiItemModifier + sync::Sync + sync::Send>, - ), - - /// A trivial attribute "macro" that does nothing, - /// only keeps the attribute and marks it as inert, - /// thus making it ineligible for further expansion. - NonMacroAttr { - /// Suppresses the `unused_attributes` lint for this attribute. - mark_used: bool, - }, - - /// A token-based derive macro. - Derive( - /// An expander with signature TokenStream -> TokenStream (not yet). - /// The produced TokenSteam is appended to the input TokenSteam. - Box<dyn MultiItemModifier + sync::Sync + sync::Send>, - ), - - /// An AST-based derive macro. - LegacyDerive( - /// An expander with signature AST -> AST. - /// The produced AST fragment is appended to the input AST fragment. - Box<dyn MultiItemModifier + sync::Sync + sync::Send>, - ), -} - -/// A struct representing a macro definition in "lowered" form ready for expansion. -pub struct SyntaxExtension { - /// A syntax extension kind. - pub kind: SyntaxExtensionKind, - /// Span of the macro definition. - pub span: Span, - /// Whitelist of unstable features that are treated as stable inside this macro. - pub allow_internal_unstable: Option<Lrc<[Symbol]>>, - /// Suppresses the `unsafe_code` lint for code produced by this macro. - pub allow_internal_unsafe: bool, - /// Enables the macro helper hack (`ident!(...)` -> `$crate::ident!(...)`) for this macro. - pub local_inner_macros: bool, - /// The macro's stability info. - pub stability: Option<Stability>, - /// The macro's deprecation info. - pub deprecation: Option<Deprecation>, - /// Names of helper attributes registered by this macro. - pub helper_attrs: Vec<Symbol>, - /// Edition of the crate in which this macro is defined. - pub edition: Edition, - /// Built-in macros have a couple of special properties like availability - /// in `#[no_implicit_prelude]` modules, so we have to keep this flag. - pub is_builtin: bool, - /// We have to identify macros providing a `Copy` impl early for compatibility reasons. - pub is_derive_copy: bool, -} - -impl SyntaxExtension { - /// Returns which kind of macro calls this syntax extension. - pub fn macro_kind(&self) -> MacroKind { - match self.kind { - SyntaxExtensionKind::Bang(..) | - SyntaxExtensionKind::LegacyBang(..) => MacroKind::Bang, - SyntaxExtensionKind::Attr(..) | - SyntaxExtensionKind::LegacyAttr(..) | - SyntaxExtensionKind::NonMacroAttr { .. } => MacroKind::Attr, - SyntaxExtensionKind::Derive(..) | - SyntaxExtensionKind::LegacyDerive(..) => MacroKind::Derive, - } - } - - /// Constructs a syntax extension with default properties. - pub fn default(kind: SyntaxExtensionKind, edition: Edition) -> SyntaxExtension { - SyntaxExtension { - span: DUMMY_SP, - allow_internal_unstable: None, - allow_internal_unsafe: false, - local_inner_macros: false, - stability: None, - deprecation: None, - helper_attrs: Vec::new(), - edition, - is_builtin: false, - is_derive_copy: false, - kind, - } - } - - /// Constructs a syntax extension with the given properties - /// and other properties converted from attributes. - pub fn new( - sess: &ParseSess, - kind: SyntaxExtensionKind, - span: Span, - helper_attrs: Vec<Symbol>, - edition: Edition, - name: Name, - attrs: &[ast::Attribute], - ) -> SyntaxExtension { - let allow_internal_unstable = attr::allow_internal_unstable( - &attrs, &sess.span_diagnostic, - ).map(|features| features.collect::<Vec<Symbol>>().into()); - - let mut local_inner_macros = false; - if let Some(macro_export) = attr::find_by_name(attrs, sym::macro_export) { - if let Some(l) = macro_export.meta_item_list() { - local_inner_macros = attr::list_contains_name(&l, sym::local_inner_macros); - } - } - - let is_builtin = attr::contains_name(attrs, sym::rustc_builtin_macro); - - SyntaxExtension { - kind, - span, - allow_internal_unstable, - allow_internal_unsafe: attr::contains_name(attrs, sym::allow_internal_unsafe), - local_inner_macros, - stability: attr::find_stability(&sess, attrs, span), - deprecation: attr::find_deprecation(&sess, attrs, span), - helper_attrs, - edition, - is_builtin, - is_derive_copy: is_builtin && name == sym::Copy, - } - } - - pub fn dummy_bang(edition: Edition) -> SyntaxExtension { - fn expander<'cx>(_: &'cx mut ExtCtxt<'_>, span: Span, _: TokenStream) - -> Box<dyn MacResult + 'cx> { - DummyResult::any(span) - } - SyntaxExtension::default(SyntaxExtensionKind::LegacyBang(Box::new(expander)), edition) - } - - pub fn dummy_derive(edition: Edition) -> SyntaxExtension { - fn expander(_: &mut ExtCtxt<'_>, _: Span, _: &ast::MetaItem, _: Annotatable) - -> Vec<Annotatable> { - Vec::new() - } - SyntaxExtension::default(SyntaxExtensionKind::Derive(Box::new(expander)), edition) - } - - pub fn non_macro_attr(mark_used: bool, edition: Edition) -> SyntaxExtension { - SyntaxExtension::default(SyntaxExtensionKind::NonMacroAttr { mark_used }, edition) - } - - pub fn expn_data(&self, parent: ExpnId, call_site: Span, descr: Symbol) -> ExpnData { - ExpnData { - kind: ExpnKind::Macro(self.macro_kind(), descr), - parent, - call_site, - def_site: self.span, - allow_internal_unstable: self.allow_internal_unstable.clone(), - allow_internal_unsafe: self.allow_internal_unsafe, - local_inner_macros: self.local_inner_macros, - edition: self.edition, - } - } -} - -pub type NamedSyntaxExtension = (Name, SyntaxExtension); - -/// Result of resolving a macro invocation. -pub enum InvocationRes { - Single(Lrc<SyntaxExtension>), - DeriveContainer(Vec<Lrc<SyntaxExtension>>), -} - -/// Error type that denotes indeterminacy. -pub struct Indeterminate; - -bitflags::bitflags! { - /// Built-in derives that need some extra tracking beyond the usual macro functionality. - #[derive(Default)] - pub struct SpecialDerives: u8 { - const PARTIAL_EQ = 1 << 0; - const EQ = 1 << 1; - const COPY = 1 << 2; - } -} - -pub trait Resolver { - fn next_node_id(&mut self) -> NodeId; - - fn resolve_dollar_crates(&mut self); - fn visit_ast_fragment_with_placeholders(&mut self, expn_id: ExpnId, fragment: &AstFragment, - extra_placeholders: &[NodeId]); - fn register_builtin_macro(&mut self, ident: ast::Ident, ext: SyntaxExtension); - - fn expansion_for_ast_pass( - &mut self, - call_site: Span, - pass: AstPass, - features: &[Symbol], - parent_module_id: Option<NodeId>, - ) -> ExpnId; - - fn resolve_imports(&mut self); - - fn resolve_macro_invocation( - &mut self, invoc: &Invocation, eager_expansion_root: ExpnId, force: bool - ) -> Result<InvocationRes, Indeterminate>; - - fn check_unused_macros(&self); - - fn has_derives(&self, expn_id: ExpnId, derives: SpecialDerives) -> bool; - fn add_derives(&mut self, expn_id: ExpnId, derives: SpecialDerives); -} - -#[derive(Clone)] -pub struct ModuleData { - pub mod_path: Vec<ast::Ident>, - pub directory: PathBuf, -} - -#[derive(Clone)] -pub struct ExpansionData { - pub id: ExpnId, - pub depth: usize, - pub module: Rc<ModuleData>, - pub directory_ownership: DirectoryOwnership, - pub prior_type_ascription: Option<(Span, bool)>, -} - -/// One of these is made during expansion and incrementally updated as we go; -/// when a macro expansion occurs, the resulting nodes have the `backtrace() -/// -> expn_data` of their expansion context stored into their span. -pub struct ExtCtxt<'a> { - pub parse_sess: &'a parse::ParseSess, - pub ecfg: expand::ExpansionConfig<'a>, - pub root_path: PathBuf, - pub resolver: &'a mut dyn Resolver, - pub current_expansion: ExpansionData, - pub expansions: FxHashMap<Span, Vec<String>>, -} - -impl<'a> ExtCtxt<'a> { - pub fn new(parse_sess: &'a parse::ParseSess, - ecfg: expand::ExpansionConfig<'a>, - resolver: &'a mut dyn Resolver) - -> ExtCtxt<'a> { - ExtCtxt { - parse_sess, - ecfg, - root_path: PathBuf::new(), - resolver, - current_expansion: ExpansionData { - id: ExpnId::root(), - depth: 0, - module: Rc::new(ModuleData { mod_path: Vec::new(), directory: PathBuf::new() }), - directory_ownership: DirectoryOwnership::Owned { relative: None }, - prior_type_ascription: None, - }, - expansions: FxHashMap::default(), - } - } - - /// Returns a `Folder` for deeply expanding all macros in an AST node. - pub fn expander<'b>(&'b mut self) -> expand::MacroExpander<'b, 'a> { - expand::MacroExpander::new(self, false) - } - - /// Returns a `Folder` that deeply expands all macros and assigns all `NodeId`s in an AST node. - /// Once `NodeId`s are assigned, the node may not be expanded, removed, or otherwise modified. - pub fn monotonic_expander<'b>(&'b mut self) -> expand::MacroExpander<'b, 'a> { - expand::MacroExpander::new(self, true) - } - pub fn new_parser_from_tts(&self, stream: TokenStream) -> parser::Parser<'a> { - parse::stream_to_parser(self.parse_sess, stream, MACRO_ARGUMENTS) - } - pub fn source_map(&self) -> &'a SourceMap { self.parse_sess.source_map() } - pub fn parse_sess(&self) -> &'a parse::ParseSess { self.parse_sess } - pub fn cfg(&self) -> &ast::CrateConfig { &self.parse_sess.config } - pub fn call_site(&self) -> Span { - self.current_expansion.id.expn_data().call_site - } - - /// Equivalent of `Span::def_site` from the proc macro API, - /// except that the location is taken from the span passed as an argument. - pub fn with_def_site_ctxt(&self, span: Span) -> Span { - span.with_def_site_ctxt(self.current_expansion.id) - } - - /// Equivalent of `Span::call_site` from the proc macro API, - /// except that the location is taken from the span passed as an argument. - pub fn with_call_site_ctxt(&self, span: Span) -> Span { - span.with_call_site_ctxt(self.current_expansion.id) - } - - /// Equivalent of `Span::mixed_site` from the proc macro API, - /// except that the location is taken from the span passed as an argument. - pub fn with_mixed_site_ctxt(&self, span: Span) -> Span { - span.with_mixed_site_ctxt(self.current_expansion.id) - } - - /// Returns span for the macro which originally caused the current expansion to happen. - /// - /// Stops backtracing at include! boundary. - pub fn expansion_cause(&self) -> Option<Span> { - let mut expn_id = self.current_expansion.id; - let mut last_macro = None; - loop { - let expn_data = expn_id.expn_data(); - // Stop going up the backtrace once include! is encountered - if expn_data.is_root() || expn_data.kind.descr() == sym::include { - break; - } - expn_id = expn_data.call_site.ctxt().outer_expn(); - last_macro = Some(expn_data.call_site); - } - last_macro - } - - pub fn struct_span_warn<S: Into<MultiSpan>>(&self, - sp: S, - msg: &str) - -> DiagnosticBuilder<'a> { - self.parse_sess.span_diagnostic.struct_span_warn(sp, msg) - } - pub fn struct_span_err<S: Into<MultiSpan>>(&self, - sp: S, - msg: &str) - -> DiagnosticBuilder<'a> { - self.parse_sess.span_diagnostic.struct_span_err(sp, msg) - } - pub fn struct_span_fatal<S: Into<MultiSpan>>(&self, - sp: S, - msg: &str) - -> DiagnosticBuilder<'a> { - self.parse_sess.span_diagnostic.struct_span_fatal(sp, msg) - } - - /// Emit `msg` attached to `sp`, and stop compilation immediately. - /// - /// `span_err` should be strongly preferred where-ever possible: - /// this should *only* be used when: - /// - /// - continuing has a high risk of flow-on errors (e.g., errors in - /// declaring a macro would cause all uses of that macro to - /// complain about "undefined macro"), or - /// - there is literally nothing else that can be done (however, - /// in most cases one can construct a dummy expression/item to - /// substitute; we never hit resolve/type-checking so the dummy - /// value doesn't have to match anything) - pub fn span_fatal<S: Into<MultiSpan>>(&self, sp: S, msg: &str) -> ! { - self.parse_sess.span_diagnostic.span_fatal(sp, msg).raise(); - } - - /// Emit `msg` attached to `sp`, without immediately stopping - /// compilation. - /// - /// Compilation will be stopped in the near future (at the end of - /// the macro expansion phase). - pub fn span_err<S: Into<MultiSpan>>(&self, sp: S, msg: &str) { - self.parse_sess.span_diagnostic.span_err(sp, msg); - } - pub fn span_err_with_code<S: Into<MultiSpan>>(&self, sp: S, msg: &str, code: DiagnosticId) { - self.parse_sess.span_diagnostic.span_err_with_code(sp, msg, code); - } - pub fn span_warn<S: Into<MultiSpan>>(&self, sp: S, msg: &str) { - self.parse_sess.span_diagnostic.span_warn(sp, msg); - } - pub fn span_bug<S: Into<MultiSpan>>(&self, sp: S, msg: &str) -> ! { - self.parse_sess.span_diagnostic.span_bug(sp, msg); - } - pub fn trace_macros_diag(&mut self) { - for (sp, notes) in self.expansions.iter() { - let mut db = self.parse_sess.span_diagnostic.span_note_diag(*sp, "trace_macro"); - for note in notes { - db.note(note); - } - db.emit(); - } - // Fixme: does this result in errors? - self.expansions.clear(); - } - pub fn bug(&self, msg: &str) -> ! { - self.parse_sess.span_diagnostic.bug(msg); - } - pub fn trace_macros(&self) -> bool { - self.ecfg.trace_mac - } - pub fn set_trace_macros(&mut self, x: bool) { - self.ecfg.trace_mac = x - } - pub fn ident_of(&self, st: &str, sp: Span) -> ast::Ident { - ast::Ident::from_str_and_span(st, sp) - } - pub fn std_path(&self, components: &[Symbol]) -> Vec<ast::Ident> { - let def_site = self.with_def_site_ctxt(DUMMY_SP); - iter::once(Ident::new(kw::DollarCrate, def_site)) - .chain(components.iter().map(|&s| Ident::with_dummy_span(s))) - .collect() - } - pub fn name_of(&self, st: &str) -> ast::Name { - Symbol::intern(st) - } - - pub fn check_unused_macros(&self) { - self.resolver.check_unused_macros(); - } - - /// Resolves a path mentioned inside Rust code. - /// - /// This unifies the logic used for resolving `include_X!`, and `#[doc(include)]` file paths. - /// - /// Returns an absolute path to the file that `path` refers to. - pub fn resolve_path(&self, path: impl Into<PathBuf>, span: Span) -> PathBuf { - let path = path.into(); - - // Relative paths are resolved relative to the file in which they are found - // after macro expansion (that is, they are unhygienic). - if !path.is_absolute() { - let callsite = span.source_callsite(); - let mut result = match self.source_map().span_to_unmapped_path(callsite) { - FileName::Real(path) => path, - FileName::DocTest(path, _) => path, - other => panic!("cannot resolve relative path in non-file source `{}`", other), - }; - result.pop(); - result.push(path); - result - } else { - path - } - } -} - -/// Extracts a string literal from the macro expanded version of `expr`, -/// emitting `err_msg` if `expr` is not a string literal. This does not stop -/// compilation on error, merely emits a non-fatal error and returns `None`. -pub fn expr_to_spanned_string<'a>( - cx: &'a mut ExtCtxt<'_>, - expr: P<ast::Expr>, - err_msg: &str, -) -> Result<(Symbol, ast::StrStyle, Span), Option<DiagnosticBuilder<'a>>> { - // Perform eager expansion on the expression. - // We want to be able to handle e.g., `concat!("foo", "bar")`. - let expr = cx.expander().fully_expand_fragment(AstFragment::Expr(expr)).make_expr(); - - Err(match expr.kind { - ast::ExprKind::Lit(ref l) => match l.kind { - ast::LitKind::Str(s, style) => return Ok((s, style, expr.span)), - ast::LitKind::Err(_) => None, - _ => Some(cx.struct_span_err(l.span, err_msg)) - }, - ast::ExprKind::Err => None, - _ => Some(cx.struct_span_err(expr.span, err_msg)) - }) -} - -pub fn expr_to_string(cx: &mut ExtCtxt<'_>, expr: P<ast::Expr>, err_msg: &str) - -> Option<(Symbol, ast::StrStyle)> { - expr_to_spanned_string(cx, expr, err_msg) - .map_err(|err| err.map(|mut err| err.emit())) - .ok() - .map(|(symbol, style, _)| (symbol, style)) -} - -/// Non-fatally assert that `tts` is empty. Note that this function -/// returns even when `tts` is non-empty, macros that *need* to stop -/// compilation should call -/// `cx.parse_sess.span_diagnostic.abort_if_errors()` (this should be -/// done as rarely as possible). -pub fn check_zero_tts(cx: &ExtCtxt<'_>, - sp: Span, - tts: TokenStream, - name: &str) { - if !tts.is_empty() { - cx.span_err(sp, &format!("{} takes no arguments", name)); - } -} - -/// Interpreting `tts` as a comma-separated sequence of expressions, -/// expect exactly one string literal, or emit an error and return `None`. -pub fn get_single_str_from_tts(cx: &mut ExtCtxt<'_>, - sp: Span, - tts: TokenStream, - name: &str) - -> Option<String> { - let mut p = cx.new_parser_from_tts(tts); - if p.token == token::Eof { - cx.span_err(sp, &format!("{} takes 1 argument", name)); - return None - } - let ret = panictry!(p.parse_expr()); - let _ = p.eat(&token::Comma); - - if p.token != token::Eof { - cx.span_err(sp, &format!("{} takes 1 argument", name)); - } - expr_to_string(cx, ret, "argument must be a string literal").map(|(s, _)| { - s.to_string() - }) -} - -/// Extracts comma-separated expressions from `tts`. If there is a -/// parsing error, emit a non-fatal error and return `None`. -pub fn get_exprs_from_tts(cx: &mut ExtCtxt<'_>, - sp: Span, - tts: TokenStream) -> Option<Vec<P<ast::Expr>>> { - let mut p = cx.new_parser_from_tts(tts); - let mut es = Vec::new(); - while p.token != token::Eof { - let expr = panictry!(p.parse_expr()); - - // Perform eager expansion on the expression. - // We want to be able to handle e.g., `concat!("foo", "bar")`. - let expr = cx.expander().fully_expand_fragment(AstFragment::Expr(expr)).make_expr(); - - es.push(expr); - if p.eat(&token::Comma) { - continue; - } - if p.token != token::Eof { - cx.span_err(sp, "expected token: `,`"); - return None; - } - } - Some(es) -} diff --git a/src/libsyntax/ext/build.rs b/src/libsyntax/ext/build.rs deleted file mode 100644 index 8c5289671c9..00000000000 --- a/src/libsyntax/ext/build.rs +++ /dev/null @@ -1,639 +0,0 @@ -use crate::ast::{self, Ident, Expr, BlockCheckMode, UnOp, PatKind}; -use crate::attr; -use crate::source_map::{respan, Spanned}; -use crate::ext::base::ExtCtxt; -use crate::ptr::P; -use crate::symbol::{kw, sym, Symbol}; -use crate::ThinVec; - -use syntax_pos::{Pos, Span}; - -impl<'a> ExtCtxt<'a> { - pub fn path(&self, span: Span, strs: Vec<ast::Ident> ) -> ast::Path { - self.path_all(span, false, strs, vec![]) - } - pub fn path_ident(&self, span: Span, id: ast::Ident) -> ast::Path { - self.path(span, vec![id]) - } - pub fn path_global(&self, span: Span, strs: Vec<ast::Ident> ) -> ast::Path { - self.path_all(span, true, strs, vec![]) - } - pub fn path_all(&self, - span: Span, - global: bool, - mut idents: Vec<ast::Ident> , - args: Vec<ast::GenericArg>) - -> ast::Path { - assert!(!idents.is_empty()); - let add_root = global && !idents[0].is_path_segment_keyword(); - let mut segments = Vec::with_capacity(idents.len() + add_root as usize); - if add_root { - segments.push(ast::PathSegment::path_root(span)); - } - let last_ident = idents.pop().unwrap(); - segments.extend(idents.into_iter().map(|ident| { - ast::PathSegment::from_ident(ident.with_span_pos(span)) - })); - let args = if !args.is_empty() { - ast::AngleBracketedArgs { args, constraints: Vec::new(), span }.into() - } else { - None - }; - segments.push(ast::PathSegment { - ident: last_ident.with_span_pos(span), - id: ast::DUMMY_NODE_ID, - args, - }); - ast::Path { span, segments } - } - - pub fn ty_mt(&self, ty: P<ast::Ty>, mutbl: ast::Mutability) -> ast::MutTy { - ast::MutTy { - ty, - mutbl, - } - } - - pub fn ty(&self, span: Span, kind: ast::TyKind) -> P<ast::Ty> { - P(ast::Ty { - id: ast::DUMMY_NODE_ID, - span, - kind, - }) - } - - pub fn ty_path(&self, path: ast::Path) -> P<ast::Ty> { - self.ty(path.span, ast::TyKind::Path(None, path)) - } - - // Might need to take bounds as an argument in the future, if you ever want - // to generate a bounded existential trait type. - pub fn ty_ident(&self, span: Span, ident: ast::Ident) - -> P<ast::Ty> { - self.ty_path(self.path_ident(span, ident)) - } - - pub fn anon_const(&self, span: Span, kind: ast::ExprKind) -> ast::AnonConst { - ast::AnonConst { - id: ast::DUMMY_NODE_ID, - value: P(ast::Expr { - id: ast::DUMMY_NODE_ID, - kind, - span, - attrs: ThinVec::new(), - }) - } - } - - pub fn const_ident(&self, span: Span, ident: ast::Ident) -> ast::AnonConst { - self.anon_const(span, ast::ExprKind::Path(None, self.path_ident(span, ident))) - } - - pub fn ty_rptr(&self, - span: Span, - ty: P<ast::Ty>, - lifetime: Option<ast::Lifetime>, - mutbl: ast::Mutability) - -> P<ast::Ty> { - self.ty(span, - ast::TyKind::Rptr(lifetime, self.ty_mt(ty, mutbl))) - } - - pub fn ty_ptr(&self, - span: Span, - ty: P<ast::Ty>, - mutbl: ast::Mutability) - -> P<ast::Ty> { - self.ty(span, - ast::TyKind::Ptr(self.ty_mt(ty, mutbl))) - } - - pub fn typaram(&self, - span: Span, - ident: ast::Ident, - attrs: Vec<ast::Attribute>, - bounds: ast::GenericBounds, - default: Option<P<ast::Ty>>) -> ast::GenericParam { - ast::GenericParam { - ident: ident.with_span_pos(span), - id: ast::DUMMY_NODE_ID, - attrs: attrs.into(), - bounds, - kind: ast::GenericParamKind::Type { - default, - }, - is_placeholder: false - } - } - - pub fn trait_ref(&self, path: ast::Path) -> ast::TraitRef { - ast::TraitRef { - path, - ref_id: ast::DUMMY_NODE_ID, - } - } - - pub fn poly_trait_ref(&self, span: Span, path: ast::Path) -> ast::PolyTraitRef { - ast::PolyTraitRef { - bound_generic_params: Vec::new(), - trait_ref: self.trait_ref(path), - span, - } - } - - pub fn trait_bound(&self, path: ast::Path) -> ast::GenericBound { - ast::GenericBound::Trait(self.poly_trait_ref(path.span, path), - ast::TraitBoundModifier::None) - } - - pub fn lifetime(&self, span: Span, ident: ast::Ident) -> ast::Lifetime { - ast::Lifetime { id: ast::DUMMY_NODE_ID, ident: ident.with_span_pos(span) } - } - - pub fn lifetime_def(&self, - span: Span, - ident: ast::Ident, - attrs: Vec<ast::Attribute>, - bounds: ast::GenericBounds) - -> ast::GenericParam { - let lifetime = self.lifetime(span, ident); - ast::GenericParam { - ident: lifetime.ident, - id: lifetime.id, - attrs: attrs.into(), - bounds, - kind: ast::GenericParamKind::Lifetime, - is_placeholder: false - } - } - - pub fn stmt_expr(&self, expr: P<ast::Expr>) -> ast::Stmt { - ast::Stmt { - id: ast::DUMMY_NODE_ID, - span: expr.span, - kind: ast::StmtKind::Expr(expr), - } - } - - pub fn stmt_let(&self, sp: Span, mutbl: bool, ident: ast::Ident, - ex: P<ast::Expr>) -> ast::Stmt { - let pat = if mutbl { - let binding_mode = ast::BindingMode::ByValue(ast::Mutability::Mutable); - self.pat_ident_binding_mode(sp, ident, binding_mode) - } else { - self.pat_ident(sp, ident) - }; - let local = P(ast::Local { - pat, - ty: None, - init: Some(ex), - id: ast::DUMMY_NODE_ID, - span: sp, - attrs: ThinVec::new(), - }); - ast::Stmt { - id: ast::DUMMY_NODE_ID, - kind: ast::StmtKind::Local(local), - span: sp, - } - } - - // Generates `let _: Type;`, which is usually used for type assertions. - pub fn stmt_let_type_only(&self, span: Span, ty: P<ast::Ty>) -> ast::Stmt { - let local = P(ast::Local { - pat: self.pat_wild(span), - ty: Some(ty), - init: None, - id: ast::DUMMY_NODE_ID, - span, - attrs: ThinVec::new(), - }); - ast::Stmt { - id: ast::DUMMY_NODE_ID, - kind: ast::StmtKind::Local(local), - span, - } - } - - pub fn stmt_item(&self, sp: Span, item: P<ast::Item>) -> ast::Stmt { - ast::Stmt { - id: ast::DUMMY_NODE_ID, - kind: ast::StmtKind::Item(item), - span: sp, - } - } - - pub fn block_expr(&self, expr: P<ast::Expr>) -> P<ast::Block> { - self.block(expr.span, vec![ast::Stmt { - id: ast::DUMMY_NODE_ID, - span: expr.span, - kind: ast::StmtKind::Expr(expr), - }]) - } - pub fn block(&self, span: Span, stmts: Vec<ast::Stmt>) -> P<ast::Block> { - P(ast::Block { - stmts, - id: ast::DUMMY_NODE_ID, - rules: BlockCheckMode::Default, - span, - }) - } - - pub fn expr(&self, span: Span, kind: ast::ExprKind) -> P<ast::Expr> { - P(ast::Expr { - id: ast::DUMMY_NODE_ID, - kind, - span, - attrs: ThinVec::new(), - }) - } - - pub fn expr_path(&self, path: ast::Path) -> P<ast::Expr> { - self.expr(path.span, ast::ExprKind::Path(None, path)) - } - - pub fn expr_ident(&self, span: Span, id: ast::Ident) -> P<ast::Expr> { - self.expr_path(self.path_ident(span, id)) - } - pub fn expr_self(&self, span: Span) -> P<ast::Expr> { - self.expr_ident(span, Ident::with_dummy_span(kw::SelfLower)) - } - - pub fn expr_binary(&self, sp: Span, op: ast::BinOpKind, - lhs: P<ast::Expr>, rhs: P<ast::Expr>) -> P<ast::Expr> { - self.expr(sp, ast::ExprKind::Binary(Spanned { node: op, span: sp }, lhs, rhs)) - } - - pub fn expr_deref(&self, sp: Span, e: P<ast::Expr>) -> P<ast::Expr> { - self.expr(sp, ast::ExprKind::Unary(UnOp::Deref, e)) - } - - pub fn expr_addr_of(&self, sp: Span, e: P<ast::Expr>) -> P<ast::Expr> { - self.expr(sp, ast::ExprKind::AddrOf(ast::Mutability::Immutable, e)) - } - - pub fn expr_call( - &self, span: Span, expr: P<ast::Expr>, args: Vec<P<ast::Expr>>, - ) -> P<ast::Expr> { - self.expr(span, ast::ExprKind::Call(expr, args)) - } - pub fn expr_call_ident(&self, span: Span, id: ast::Ident, - args: Vec<P<ast::Expr>>) -> P<ast::Expr> { - self.expr(span, ast::ExprKind::Call(self.expr_ident(span, id), args)) - } - pub fn expr_call_global(&self, sp: Span, fn_path: Vec<ast::Ident> , - args: Vec<P<ast::Expr>> ) -> P<ast::Expr> { - let pathexpr = self.expr_path(self.path_global(sp, fn_path)); - self.expr_call(sp, pathexpr, args) - } - pub fn expr_method_call(&self, span: Span, - expr: P<ast::Expr>, - ident: ast::Ident, - mut args: Vec<P<ast::Expr>> ) -> P<ast::Expr> { - args.insert(0, expr); - let segment = ast::PathSegment::from_ident(ident.with_span_pos(span)); - self.expr(span, ast::ExprKind::MethodCall(segment, args)) - } - pub fn expr_block(&self, b: P<ast::Block>) -> P<ast::Expr> { - self.expr(b.span, ast::ExprKind::Block(b, None)) - } - pub fn field_imm(&self, span: Span, ident: Ident, e: P<ast::Expr>) -> ast::Field { - ast::Field { - ident: ident.with_span_pos(span), - expr: e, - span, - is_shorthand: false, - attrs: ThinVec::new(), - id: ast::DUMMY_NODE_ID, - is_placeholder: false, - } - } - pub fn expr_struct( - &self, span: Span, path: ast::Path, fields: Vec<ast::Field> - ) -> P<ast::Expr> { - self.expr(span, ast::ExprKind::Struct(path, fields, None)) - } - pub fn expr_struct_ident(&self, span: Span, - id: ast::Ident, fields: Vec<ast::Field>) -> P<ast::Expr> { - self.expr_struct(span, self.path_ident(span, id), fields) - } - - pub fn expr_lit(&self, span: Span, lit_kind: ast::LitKind) -> P<ast::Expr> { - let lit = ast::Lit::from_lit_kind(lit_kind, span); - self.expr(span, ast::ExprKind::Lit(lit)) - } - pub fn expr_usize(&self, span: Span, i: usize) -> P<ast::Expr> { - self.expr_lit(span, ast::LitKind::Int(i as u128, - ast::LitIntType::Unsigned(ast::UintTy::Usize))) - } - pub fn expr_u32(&self, sp: Span, u: u32) -> P<ast::Expr> { - self.expr_lit(sp, ast::LitKind::Int(u as u128, - ast::LitIntType::Unsigned(ast::UintTy::U32))) - } - pub fn expr_bool(&self, sp: Span, value: bool) -> P<ast::Expr> { - self.expr_lit(sp, ast::LitKind::Bool(value)) - } - - pub fn expr_vec(&self, sp: Span, exprs: Vec<P<ast::Expr>>) -> P<ast::Expr> { - self.expr(sp, ast::ExprKind::Array(exprs)) - } - pub fn expr_vec_slice(&self, sp: Span, exprs: Vec<P<ast::Expr>>) -> P<ast::Expr> { - self.expr_addr_of(sp, self.expr_vec(sp, exprs)) - } - pub fn expr_str(&self, sp: Span, s: Symbol) -> P<ast::Expr> { - self.expr_lit(sp, ast::LitKind::Str(s, ast::StrStyle::Cooked)) - } - - pub fn expr_cast(&self, sp: Span, expr: P<ast::Expr>, ty: P<ast::Ty>) -> P<ast::Expr> { - self.expr(sp, ast::ExprKind::Cast(expr, ty)) - } - - pub fn expr_some(&self, sp: Span, expr: P<ast::Expr>) -> P<ast::Expr> { - let some = self.std_path(&[sym::option, sym::Option, sym::Some]); - self.expr_call_global(sp, some, vec![expr]) - } - - pub fn expr_tuple(&self, sp: Span, exprs: Vec<P<ast::Expr>>) -> P<ast::Expr> { - self.expr(sp, ast::ExprKind::Tup(exprs)) - } - - pub fn expr_fail(&self, span: Span, msg: Symbol) -> P<ast::Expr> { - let loc = self.source_map().lookup_char_pos(span.lo()); - let expr_file = self.expr_str(span, Symbol::intern(&loc.file.name.to_string())); - let expr_line = self.expr_u32(span, loc.line as u32); - let expr_col = self.expr_u32(span, loc.col.to_usize() as u32 + 1); - let expr_loc_tuple = self.expr_tuple(span, vec![expr_file, expr_line, expr_col]); - let expr_loc_ptr = self.expr_addr_of(span, expr_loc_tuple); - self.expr_call_global( - span, - [sym::std, sym::rt, sym::begin_panic].iter().map(|s| Ident::new(*s, span)).collect(), - vec![ - self.expr_str(span, msg), - expr_loc_ptr]) - } - - pub fn expr_unreachable(&self, span: Span) -> P<ast::Expr> { - self.expr_fail(span, Symbol::intern("internal error: entered unreachable code")) - } - - pub fn expr_ok(&self, sp: Span, expr: P<ast::Expr>) -> P<ast::Expr> { - let ok = self.std_path(&[sym::result, sym::Result, sym::Ok]); - self.expr_call_global(sp, ok, vec![expr]) - } - - pub fn expr_try(&self, sp: Span, head: P<ast::Expr>) -> P<ast::Expr> { - let ok = self.std_path(&[sym::result, sym::Result, sym::Ok]); - let ok_path = self.path_global(sp, ok); - let err = self.std_path(&[sym::result, sym::Result, sym::Err]); - let err_path = self.path_global(sp, err); - - let binding_variable = self.ident_of("__try_var", sp); - let binding_pat = self.pat_ident(sp, binding_variable); - let binding_expr = self.expr_ident(sp, binding_variable); - - // `Ok(__try_var)` pattern - let ok_pat = self.pat_tuple_struct(sp, ok_path, vec![binding_pat.clone()]); - - // `Err(__try_var)` (pattern and expression respectively) - let err_pat = self.pat_tuple_struct(sp, err_path.clone(), vec![binding_pat]); - let err_inner_expr = self.expr_call(sp, self.expr_path(err_path), - vec![binding_expr.clone()]); - // `return Err(__try_var)` - let err_expr = self.expr(sp, ast::ExprKind::Ret(Some(err_inner_expr))); - - // `Ok(__try_var) => __try_var` - let ok_arm = self.arm(sp, ok_pat, binding_expr); - // `Err(__try_var) => return Err(__try_var)` - let err_arm = self.arm(sp, err_pat, err_expr); - - // `match head { Ok() => ..., Err() => ... }` - self.expr_match(sp, head, vec![ok_arm, err_arm]) - } - - - pub fn pat(&self, span: Span, kind: PatKind) -> P<ast::Pat> { - P(ast::Pat { id: ast::DUMMY_NODE_ID, kind, span }) - } - pub fn pat_wild(&self, span: Span) -> P<ast::Pat> { - self.pat(span, PatKind::Wild) - } - pub fn pat_lit(&self, span: Span, expr: P<ast::Expr>) -> P<ast::Pat> { - self.pat(span, PatKind::Lit(expr)) - } - pub fn pat_ident(&self, span: Span, ident: ast::Ident) -> P<ast::Pat> { - let binding_mode = ast::BindingMode::ByValue(ast::Mutability::Immutable); - self.pat_ident_binding_mode(span, ident, binding_mode) - } - - pub fn pat_ident_binding_mode(&self, - span: Span, - ident: ast::Ident, - bm: ast::BindingMode) -> P<ast::Pat> { - let pat = PatKind::Ident(bm, ident.with_span_pos(span), None); - self.pat(span, pat) - } - pub fn pat_path(&self, span: Span, path: ast::Path) -> P<ast::Pat> { - self.pat(span, PatKind::Path(None, path)) - } - pub fn pat_tuple_struct(&self, span: Span, path: ast::Path, - subpats: Vec<P<ast::Pat>>) -> P<ast::Pat> { - self.pat(span, PatKind::TupleStruct(path, subpats)) - } - pub fn pat_struct(&self, span: Span, path: ast::Path, - field_pats: Vec<ast::FieldPat>) -> P<ast::Pat> { - self.pat(span, PatKind::Struct(path, field_pats, false)) - } - pub fn pat_tuple(&self, span: Span, pats: Vec<P<ast::Pat>>) -> P<ast::Pat> { - self.pat(span, PatKind::Tuple(pats)) - } - - pub fn pat_some(&self, span: Span, pat: P<ast::Pat>) -> P<ast::Pat> { - let some = self.std_path(&[sym::option, sym::Option, sym::Some]); - let path = self.path_global(span, some); - self.pat_tuple_struct(span, path, vec![pat]) - } - - pub fn pat_none(&self, span: Span) -> P<ast::Pat> { - let some = self.std_path(&[sym::option, sym::Option, sym::None]); - let path = self.path_global(span, some); - self.pat_path(span, path) - } - - pub fn pat_ok(&self, span: Span, pat: P<ast::Pat>) -> P<ast::Pat> { - let some = self.std_path(&[sym::result, sym::Result, sym::Ok]); - let path = self.path_global(span, some); - self.pat_tuple_struct(span, path, vec![pat]) - } - - pub fn pat_err(&self, span: Span, pat: P<ast::Pat>) -> P<ast::Pat> { - let some = self.std_path(&[sym::result, sym::Result, sym::Err]); - let path = self.path_global(span, some); - self.pat_tuple_struct(span, path, vec![pat]) - } - - pub fn arm(&self, span: Span, pat: P<ast::Pat>, expr: P<ast::Expr>) -> ast::Arm { - ast::Arm { - attrs: vec![], - pat, - guard: None, - body: expr, - span, - id: ast::DUMMY_NODE_ID, - is_placeholder: false, - } - } - - pub fn arm_unreachable(&self, span: Span) -> ast::Arm { - self.arm(span, self.pat_wild(span), self.expr_unreachable(span)) - } - - pub fn expr_match(&self, span: Span, arg: P<ast::Expr>, arms: Vec<ast::Arm>) -> P<Expr> { - self.expr(span, ast::ExprKind::Match(arg, arms)) - } - - pub fn expr_if(&self, span: Span, cond: P<ast::Expr>, - then: P<ast::Expr>, els: Option<P<ast::Expr>>) -> P<ast::Expr> { - let els = els.map(|x| self.expr_block(self.block_expr(x))); - self.expr(span, ast::ExprKind::If(cond, self.block_expr(then), els)) - } - - pub fn lambda_fn_decl(&self, - span: Span, - fn_decl: P<ast::FnDecl>, - body: P<ast::Expr>, - fn_decl_span: Span) // span of the `|...|` part - -> P<ast::Expr> { - self.expr(span, ast::ExprKind::Closure(ast::CaptureBy::Ref, - ast::IsAsync::NotAsync, - ast::Movability::Movable, - fn_decl, - body, - fn_decl_span)) - } - - pub fn lambda(&self, - span: Span, - ids: Vec<ast::Ident>, - body: P<ast::Expr>) - -> P<ast::Expr> { - let fn_decl = self.fn_decl( - ids.iter().map(|id| self.param(span, *id, self.ty(span, ast::TyKind::Infer))).collect(), - ast::FunctionRetTy::Default(span)); - - // FIXME -- We are using `span` as the span of the `|...|` - // part of the lambda, but it probably (maybe?) corresponds to - // the entire lambda body. Probably we should extend the API - // here, but that's not entirely clear. - self.expr(span, ast::ExprKind::Closure(ast::CaptureBy::Ref, - ast::IsAsync::NotAsync, - ast::Movability::Movable, - fn_decl, - body, - span)) - } - - pub fn lambda0(&self, span: Span, body: P<ast::Expr>) -> P<ast::Expr> { - self.lambda(span, Vec::new(), body) - } - - pub fn lambda1(&self, span: Span, body: P<ast::Expr>, ident: ast::Ident) -> P<ast::Expr> { - self.lambda(span, vec![ident], body) - } - - pub fn lambda_stmts_1(&self, span: Span, stmts: Vec<ast::Stmt>, - ident: ast::Ident) -> P<ast::Expr> { - self.lambda1(span, self.expr_block(self.block(span, stmts)), ident) - } - - pub fn param(&self, span: Span, ident: ast::Ident, ty: P<ast::Ty>) -> ast::Param { - let arg_pat = self.pat_ident(span, ident); - ast::Param { - attrs: ThinVec::default(), - id: ast::DUMMY_NODE_ID, - pat: arg_pat, - span, - ty, - is_placeholder: false, - } - } - - // FIXME: unused `self` - pub fn fn_decl(&self, inputs: Vec<ast::Param>, output: ast::FunctionRetTy) -> P<ast::FnDecl> { - P(ast::FnDecl { - inputs, - output, - }) - } - - pub fn item(&self, span: Span, name: Ident, - attrs: Vec<ast::Attribute>, kind: ast::ItemKind) -> P<ast::Item> { - // FIXME: Would be nice if our generated code didn't violate - // Rust coding conventions - P(ast::Item { - ident: name, - attrs, - id: ast::DUMMY_NODE_ID, - kind, - vis: respan(span.shrink_to_lo(), ast::VisibilityKind::Inherited), - span, - tokens: None, - }) - } - - pub fn variant(&self, span: Span, ident: Ident, tys: Vec<P<ast::Ty>> ) -> ast::Variant { - let fields: Vec<_> = tys.into_iter().map(|ty| { - ast::StructField { - span: ty.span, - ty, - ident: None, - vis: respan(span.shrink_to_lo(), ast::VisibilityKind::Inherited), - attrs: Vec::new(), - id: ast::DUMMY_NODE_ID, - is_placeholder: false, - } - }).collect(); - - let vdata = if fields.is_empty() { - ast::VariantData::Unit(ast::DUMMY_NODE_ID) - } else { - ast::VariantData::Tuple(fields, ast::DUMMY_NODE_ID) - }; - - ast::Variant { - attrs: Vec::new(), - data: vdata, - disr_expr: None, - id: ast::DUMMY_NODE_ID, - ident, - span, - is_placeholder: false, - } - } - - pub fn item_static(&self, - span: Span, - name: Ident, - ty: P<ast::Ty>, - mutbl: ast::Mutability, - expr: P<ast::Expr>) - -> P<ast::Item> { - self.item(span, name, Vec::new(), ast::ItemKind::Static(ty, mutbl, expr)) - } - - pub fn item_const(&self, - span: Span, - name: Ident, - ty: P<ast::Ty>, - expr: P<ast::Expr>) - -> P<ast::Item> { - self.item(span, name, Vec::new(), ast::ItemKind::Const(ty, expr)) - } - - pub fn attribute(&self, mi: ast::MetaItem) -> ast::Attribute { - attr::mk_attr_outer(mi) - } - - pub fn meta_word(&self, sp: Span, w: ast::Name) -> ast::MetaItem { - attr::mk_word_item(Ident::new(w, sp)) - } -} diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs deleted file mode 100644 index bbd8da2acef..00000000000 --- a/src/libsyntax/ext/expand.rs +++ /dev/null @@ -1,1579 +0,0 @@ -use crate::ast::{self, AttrItem, Block, Ident, LitKind, NodeId, PatKind, Path}; -use crate::ast::{MacStmtStyle, StmtKind, ItemKind}; -use crate::attr::{self, HasAttrs}; -use crate::source_map::respan; -use crate::config::StripUnconfigured; -use crate::ext::base::*; -use crate::ext::proc_macro::{collect_derives, MarkAttrs}; -use crate::ext::hygiene::{ExpnId, SyntaxContext, ExpnData, ExpnKind}; -use crate::ext::mbe::macro_rules::annotate_err_with_kind; -use crate::ext::placeholders::{placeholder, PlaceholderExpander}; -use crate::feature_gate::{self, Features, GateIssue, is_builtin_attr, emit_feature_err}; -use crate::mut_visit::*; -use crate::parse::{DirectoryOwnership, PResult, ParseSess}; -use crate::parse::token; -use crate::parse::parser::Parser; -use crate::ptr::P; -use crate::symbol::{sym, Symbol}; -use crate::tokenstream::{TokenStream, TokenTree}; -use crate::visit::{self, Visitor}; -use crate::util::map_in_place::MapInPlace; - -use errors::{Applicability, FatalError}; -use smallvec::{smallvec, SmallVec}; -use syntax_pos::{Span, DUMMY_SP, FileName}; - -use rustc_data_structures::fx::FxHashMap; -use rustc_data_structures::sync::Lrc; -use std::io::ErrorKind; -use std::{iter, mem, slice}; -use std::ops::DerefMut; -use std::rc::Rc; -use std::path::PathBuf; - -macro_rules! ast_fragments { - ( - $($Kind:ident($AstTy:ty) { - $kind_name:expr; - $(one fn $mut_visit_ast:ident; fn $visit_ast:ident;)? - $(many fn $flat_map_ast_elt:ident; fn $visit_ast_elt:ident;)? - fn $make_ast:ident; - })* - ) => { - /// A fragment of AST that can be produced by a single macro expansion. - /// Can also serve as an input and intermediate result for macro expansion operations. - pub enum AstFragment { - OptExpr(Option<P<ast::Expr>>), - $($Kind($AstTy),)* - } - - /// "Discriminant" of an AST fragment. - #[derive(Copy, Clone, PartialEq, Eq)] - pub enum AstFragmentKind { - OptExpr, - $($Kind,)* - } - - impl AstFragmentKind { - pub fn name(self) -> &'static str { - match self { - AstFragmentKind::OptExpr => "expression", - $(AstFragmentKind::$Kind => $kind_name,)* - } - } - - fn make_from<'a>(self, result: Box<dyn MacResult + 'a>) -> Option<AstFragment> { - match self { - AstFragmentKind::OptExpr => - result.make_expr().map(Some).map(AstFragment::OptExpr), - $(AstFragmentKind::$Kind => result.$make_ast().map(AstFragment::$Kind),)* - } - } - } - - impl AstFragment { - pub fn make_opt_expr(self) -> Option<P<ast::Expr>> { - match self { - AstFragment::OptExpr(expr) => expr, - _ => panic!("AstFragment::make_* called on the wrong kind of fragment"), - } - } - - $(pub fn $make_ast(self) -> $AstTy { - match self { - AstFragment::$Kind(ast) => ast, - _ => panic!("AstFragment::make_* called on the wrong kind of fragment"), - } - })* - - pub fn mut_visit_with<F: MutVisitor>(&mut self, vis: &mut F) { - match self { - AstFragment::OptExpr(opt_expr) => { - visit_clobber(opt_expr, |opt_expr| { - if let Some(expr) = opt_expr { - vis.filter_map_expr(expr) - } else { - None - } - }); - } - $($(AstFragment::$Kind(ast) => vis.$mut_visit_ast(ast),)?)* - $($(AstFragment::$Kind(ast) => - ast.flat_map_in_place(|ast| vis.$flat_map_ast_elt(ast)),)?)* - } - } - - pub fn visit_with<'a, V: Visitor<'a>>(&'a self, visitor: &mut V) { - match *self { - AstFragment::OptExpr(Some(ref expr)) => visitor.visit_expr(expr), - AstFragment::OptExpr(None) => {} - $($(AstFragment::$Kind(ref ast) => visitor.$visit_ast(ast),)?)* - $($(AstFragment::$Kind(ref ast) => for ast_elt in &ast[..] { - visitor.$visit_ast_elt(ast_elt); - })?)* - } - } - } - - impl<'a> MacResult for crate::ext::mbe::macro_rules::ParserAnyMacro<'a> { - $(fn $make_ast(self: Box<crate::ext::mbe::macro_rules::ParserAnyMacro<'a>>) - -> Option<$AstTy> { - Some(self.make(AstFragmentKind::$Kind).$make_ast()) - })* - } - } -} - -ast_fragments! { - Expr(P<ast::Expr>) { "expression"; one fn visit_expr; fn visit_expr; fn make_expr; } - Pat(P<ast::Pat>) { "pattern"; one fn visit_pat; fn visit_pat; fn make_pat; } - Ty(P<ast::Ty>) { "type"; one fn visit_ty; fn visit_ty; fn make_ty; } - Stmts(SmallVec<[ast::Stmt; 1]>) { - "statement"; many fn flat_map_stmt; fn visit_stmt; fn make_stmts; - } - Items(SmallVec<[P<ast::Item>; 1]>) { - "item"; many fn flat_map_item; fn visit_item; fn make_items; - } - TraitItems(SmallVec<[ast::TraitItem; 1]>) { - "trait item"; many fn flat_map_trait_item; fn visit_trait_item; fn make_trait_items; - } - ImplItems(SmallVec<[ast::ImplItem; 1]>) { - "impl item"; many fn flat_map_impl_item; fn visit_impl_item; fn make_impl_items; - } - ForeignItems(SmallVec<[ast::ForeignItem; 1]>) { - "foreign item"; - many fn flat_map_foreign_item; - fn visit_foreign_item; - fn make_foreign_items; - } - Arms(SmallVec<[ast::Arm; 1]>) { - "match arm"; many fn flat_map_arm; fn visit_arm; fn make_arms; - } - Fields(SmallVec<[ast::Field; 1]>) { - "field expression"; many fn flat_map_field; fn visit_field; fn make_fields; - } - FieldPats(SmallVec<[ast::FieldPat; 1]>) { - "field pattern"; - many fn flat_map_field_pattern; - fn visit_field_pattern; - fn make_field_patterns; - } - GenericParams(SmallVec<[ast::GenericParam; 1]>) { - "generic parameter"; - many fn flat_map_generic_param; - fn visit_generic_param; - fn make_generic_params; - } - Params(SmallVec<[ast::Param; 1]>) { - "function parameter"; many fn flat_map_param; fn visit_param; fn make_params; - } - StructFields(SmallVec<[ast::StructField; 1]>) { - "field"; - many fn flat_map_struct_field; - fn visit_struct_field; - fn make_struct_fields; - } - Variants(SmallVec<[ast::Variant; 1]>) { - "variant"; many fn flat_map_variant; fn visit_variant; fn make_variants; - } -} - -impl AstFragmentKind { - fn dummy(self, span: Span) -> AstFragment { - self.make_from(DummyResult::any(span)).expect("couldn't create a dummy AST fragment") - } - - fn expect_from_annotatables<I: IntoIterator<Item = Annotatable>>(self, items: I) - -> AstFragment { - let mut items = items.into_iter(); - match self { - AstFragmentKind::Arms => - AstFragment::Arms(items.map(Annotatable::expect_arm).collect()), - AstFragmentKind::Fields => - AstFragment::Fields(items.map(Annotatable::expect_field).collect()), - AstFragmentKind::FieldPats => - AstFragment::FieldPats(items.map(Annotatable::expect_field_pattern).collect()), - AstFragmentKind::GenericParams => - AstFragment::GenericParams(items.map(Annotatable::expect_generic_param).collect()), - AstFragmentKind::Params => - AstFragment::Params(items.map(Annotatable::expect_param).collect()), - AstFragmentKind::StructFields => AstFragment::StructFields( - items.map(Annotatable::expect_struct_field).collect() - ), - AstFragmentKind::Variants => - AstFragment::Variants(items.map(Annotatable::expect_variant).collect()), - AstFragmentKind::Items => - AstFragment::Items(items.map(Annotatable::expect_item).collect()), - AstFragmentKind::ImplItems => - AstFragment::ImplItems(items.map(Annotatable::expect_impl_item).collect()), - AstFragmentKind::TraitItems => - AstFragment::TraitItems(items.map(Annotatable::expect_trait_item).collect()), - AstFragmentKind::ForeignItems => - AstFragment::ForeignItems(items.map(Annotatable::expect_foreign_item).collect()), - AstFragmentKind::Stmts => - AstFragment::Stmts(items.map(Annotatable::expect_stmt).collect()), - AstFragmentKind::Expr => AstFragment::Expr( - items.next().expect("expected exactly one expression").expect_expr() - ), - AstFragmentKind::OptExpr => - AstFragment::OptExpr(items.next().map(Annotatable::expect_expr)), - AstFragmentKind::Pat | AstFragmentKind::Ty => - panic!("patterns and types aren't annotatable"), - } - } -} - -pub struct Invocation { - pub kind: InvocationKind, - pub fragment_kind: AstFragmentKind, - pub expansion_data: ExpansionData, -} - -pub enum InvocationKind { - Bang { - mac: ast::Mac, - span: Span, - }, - Attr { - attr: ast::Attribute, - item: Annotatable, - // Required for resolving derive helper attributes. - derives: Vec<Path>, - // We temporarily report errors for attribute macros placed after derives - after_derive: bool, - }, - Derive { - path: Path, - item: Annotatable, - }, - /// "Invocation" that contains all derives from an item, - /// broken into multiple `Derive` invocations when expanded. - /// FIXME: Find a way to remove it. - DeriveContainer { - derives: Vec<Path>, - item: Annotatable, - }, -} - -impl Invocation { - pub fn span(&self) -> Span { - match &self.kind { - InvocationKind::Bang { span, .. } => *span, - InvocationKind::Attr { attr, .. } => attr.span, - InvocationKind::Derive { path, .. } => path.span, - InvocationKind::DeriveContainer { item, .. } => item.span(), - } - } -} - -pub struct MacroExpander<'a, 'b> { - pub cx: &'a mut ExtCtxt<'b>, - monotonic: bool, // cf. `cx.monotonic_expander()` -} - -impl<'a, 'b> MacroExpander<'a, 'b> { - pub fn new(cx: &'a mut ExtCtxt<'b>, monotonic: bool) -> Self { - MacroExpander { cx, monotonic } - } - - pub fn expand_crate(&mut self, mut krate: ast::Crate) -> ast::Crate { - let mut module = ModuleData { - mod_path: vec![Ident::from_str(&self.cx.ecfg.crate_name)], - directory: match self.cx.source_map().span_to_unmapped_path(krate.span) { - FileName::Real(path) => path, - other => PathBuf::from(other.to_string()), - }, - }; - module.directory.pop(); - self.cx.root_path = module.directory.clone(); - self.cx.current_expansion.module = Rc::new(module); - - let orig_mod_span = krate.module.inner; - - let krate_item = AstFragment::Items(smallvec![P(ast::Item { - attrs: krate.attrs, - span: krate.span, - kind: ast::ItemKind::Mod(krate.module), - ident: Ident::invalid(), - id: ast::DUMMY_NODE_ID, - vis: respan(krate.span.shrink_to_lo(), ast::VisibilityKind::Public), - tokens: None, - })]); - - match self.fully_expand_fragment(krate_item).make_items().pop().map(P::into_inner) { - Some(ast::Item { attrs, kind: ast::ItemKind::Mod(module), .. }) => { - krate.attrs = attrs; - krate.module = module; - }, - None => { - // Resolution failed so we return an empty expansion - krate.attrs = vec![]; - krate.module = ast::Mod { - inner: orig_mod_span, - items: vec![], - inline: true, - }; - }, - _ => unreachable!(), - }; - self.cx.trace_macros_diag(); - krate - } - - // Recursively expand all macro invocations in this AST fragment. - pub fn fully_expand_fragment(&mut self, input_fragment: AstFragment) -> AstFragment { - let orig_expansion_data = self.cx.current_expansion.clone(); - self.cx.current_expansion.depth = 0; - - // Collect all macro invocations and replace them with placeholders. - let (mut fragment_with_placeholders, mut invocations) - = self.collect_invocations(input_fragment, &[]); - - // Optimization: if we resolve all imports now, - // we'll be able to immediately resolve most of imported macros. - self.resolve_imports(); - - // Resolve paths in all invocations and produce output expanded fragments for them, but - // do not insert them into our input AST fragment yet, only store in `expanded_fragments`. - // The output fragments also go through expansion recursively until no invocations are left. - // Unresolved macros produce dummy outputs as a recovery measure. - invocations.reverse(); - let mut expanded_fragments = Vec::new(); - let mut all_derive_placeholders: FxHashMap<ExpnId, Vec<_>> = FxHashMap::default(); - let mut undetermined_invocations = Vec::new(); - let (mut progress, mut force) = (false, !self.monotonic); - loop { - let invoc = if let Some(invoc) = invocations.pop() { - invoc - } else { - self.resolve_imports(); - if undetermined_invocations.is_empty() { break } - invocations = mem::take(&mut undetermined_invocations); - force = !mem::replace(&mut progress, false); - continue - }; - - let eager_expansion_root = - if self.monotonic { invoc.expansion_data.id } else { orig_expansion_data.id }; - let res = match self.cx.resolver.resolve_macro_invocation( - &invoc, eager_expansion_root, force - ) { - Ok(res) => res, - Err(Indeterminate) => { - undetermined_invocations.push(invoc); - continue - } - }; - - progress = true; - let ExpansionData { depth, id: expn_id, .. } = invoc.expansion_data; - self.cx.current_expansion = invoc.expansion_data.clone(); - - // FIXME(jseyfried): Refactor out the following logic - let (expanded_fragment, new_invocations) = match res { - InvocationRes::Single(ext) => { - let fragment = self.expand_invoc(invoc, &ext.kind); - self.collect_invocations(fragment, &[]) - } - InvocationRes::DeriveContainer(exts) => { - let (derives, item) = match invoc.kind { - InvocationKind::DeriveContainer { derives, item } => (derives, item), - _ => unreachable!(), - }; - if !item.derive_allowed() { - let attr = attr::find_by_name(item.attrs(), sym::derive) - .expect("`derive` attribute should exist"); - let span = attr.span; - let mut err = self.cx.struct_span_err(span, - "`derive` may only be applied to structs, enums and unions"); - if let ast::AttrStyle::Inner = attr.style { - let trait_list = derives.iter() - .map(|t| t.to_string()).collect::<Vec<_>>(); - let suggestion = format!("#[derive({})]", trait_list.join(", ")); - err.span_suggestion( - span, "try an outer attribute", suggestion, - // We don't 𝑘𝑛𝑜𝑤 that the following item is an ADT - Applicability::MaybeIncorrect - ); - } - err.emit(); - } - - let mut item = self.fully_configure(item); - item.visit_attrs(|attrs| attrs.retain(|a| a.path != sym::derive)); - let mut helper_attrs = Vec::new(); - let mut has_copy = false; - for ext in exts { - helper_attrs.extend(&ext.helper_attrs); - has_copy |= ext.is_derive_copy; - } - // Mark derive helpers inside this item as known and used. - // FIXME: This is a hack, derive helpers should be integrated with regular name - // resolution instead. For example, helpers introduced by a derive container - // can be in scope for all code produced by that container's expansion. - item.visit_with(&mut MarkAttrs(&helper_attrs)); - if has_copy { - self.cx.resolver.add_derives(invoc.expansion_data.id, SpecialDerives::COPY); - } - - let derive_placeholders = - all_derive_placeholders.entry(invoc.expansion_data.id).or_default(); - derive_placeholders.reserve(derives.len()); - invocations.reserve(derives.len()); - for path in derives { - let expn_id = ExpnId::fresh(None); - derive_placeholders.push(NodeId::placeholder_from_expn_id(expn_id)); - invocations.push(Invocation { - kind: InvocationKind::Derive { path, item: item.clone() }, - fragment_kind: invoc.fragment_kind, - expansion_data: ExpansionData { - id: expn_id, - ..invoc.expansion_data.clone() - }, - }); - } - let fragment = invoc.fragment_kind - .expect_from_annotatables(::std::iter::once(item)); - self.collect_invocations(fragment, derive_placeholders) - } - }; - - if expanded_fragments.len() < depth { - expanded_fragments.push(Vec::new()); - } - expanded_fragments[depth - 1].push((expn_id, expanded_fragment)); - if !self.cx.ecfg.single_step { - invocations.extend(new_invocations.into_iter().rev()); - } - } - - self.cx.current_expansion = orig_expansion_data; - - // Finally incorporate all the expanded macros into the input AST fragment. - let mut placeholder_expander = PlaceholderExpander::new(self.cx, self.monotonic); - while let Some(expanded_fragments) = expanded_fragments.pop() { - for (expn_id, expanded_fragment) in expanded_fragments.into_iter().rev() { - let derive_placeholders = - all_derive_placeholders.remove(&expn_id).unwrap_or_else(Vec::new); - placeholder_expander.add(NodeId::placeholder_from_expn_id(expn_id), - expanded_fragment, derive_placeholders); - } - } - fragment_with_placeholders.mut_visit_with(&mut placeholder_expander); - fragment_with_placeholders - } - - fn resolve_imports(&mut self) { - if self.monotonic { - self.cx.resolver.resolve_imports(); - } - } - - /// Collects all macro invocations reachable at this time in this AST fragment, and replace - /// them with "placeholders" - dummy macro invocations with specially crafted `NodeId`s. - /// Then call into resolver that builds a skeleton ("reduced graph") of the fragment and - /// prepares data for resolving paths of macro invocations. - fn collect_invocations(&mut self, mut fragment: AstFragment, extra_placeholders: &[NodeId]) - -> (AstFragment, Vec<Invocation>) { - // Resolve `$crate`s in the fragment for pretty-printing. - self.cx.resolver.resolve_dollar_crates(); - - let invocations = { - let mut collector = InvocationCollector { - cfg: StripUnconfigured { - sess: self.cx.parse_sess, - features: self.cx.ecfg.features, - }, - cx: self.cx, - invocations: Vec::new(), - monotonic: self.monotonic, - }; - fragment.mut_visit_with(&mut collector); - collector.invocations - }; - - // FIXME: Merge `extra_placeholders` into the `fragment` as regular placeholders. - if self.monotonic { - self.cx.resolver.visit_ast_fragment_with_placeholders( - self.cx.current_expansion.id, &fragment, extra_placeholders); - } - - (fragment, invocations) - } - - fn fully_configure(&mut self, item: Annotatable) -> Annotatable { - let mut cfg = StripUnconfigured { - sess: self.cx.parse_sess, - features: self.cx.ecfg.features, - }; - // Since the item itself has already been configured by the InvocationCollector, - // we know that fold result vector will contain exactly one element - match item { - Annotatable::Item(item) => { - Annotatable::Item(cfg.flat_map_item(item).pop().unwrap()) - } - Annotatable::TraitItem(item) => { - Annotatable::TraitItem( - item.map(|item| cfg.flat_map_trait_item(item).pop().unwrap())) - } - Annotatable::ImplItem(item) => { - Annotatable::ImplItem(item.map(|item| cfg.flat_map_impl_item(item).pop().unwrap())) - } - Annotatable::ForeignItem(item) => { - Annotatable::ForeignItem( - item.map(|item| cfg.flat_map_foreign_item(item).pop().unwrap()) - ) - } - Annotatable::Stmt(stmt) => { - Annotatable::Stmt(stmt.map(|stmt| cfg.flat_map_stmt(stmt).pop().unwrap())) - } - Annotatable::Expr(mut expr) => { - Annotatable::Expr({ cfg.visit_expr(&mut expr); expr }) - } - Annotatable::Arm(arm) => { - Annotatable::Arm(cfg.flat_map_arm(arm).pop().unwrap()) - } - Annotatable::Field(field) => { - Annotatable::Field(cfg.flat_map_field(field).pop().unwrap()) - } - Annotatable::FieldPat(fp) => { - Annotatable::FieldPat(cfg.flat_map_field_pattern(fp).pop().unwrap()) - } - Annotatable::GenericParam(param) => { - Annotatable::GenericParam(cfg.flat_map_generic_param(param).pop().unwrap()) - } - Annotatable::Param(param) => { - Annotatable::Param(cfg.flat_map_param(param).pop().unwrap()) - } - Annotatable::StructField(sf) => { - Annotatable::StructField(cfg.flat_map_struct_field(sf).pop().unwrap()) - } - Annotatable::Variant(v) => { - Annotatable::Variant(cfg.flat_map_variant(v).pop().unwrap()) - } - } - } - - fn expand_invoc(&mut self, invoc: Invocation, ext: &SyntaxExtensionKind) -> AstFragment { - if self.cx.current_expansion.depth > self.cx.ecfg.recursion_limit { - let expn_data = self.cx.current_expansion.id.expn_data(); - let suggested_limit = self.cx.ecfg.recursion_limit * 2; - let mut err = self.cx.struct_span_err(expn_data.call_site, - &format!("recursion limit reached while expanding the macro `{}`", - expn_data.kind.descr())); - err.help(&format!( - "consider adding a `#![recursion_limit=\"{}\"]` attribute to your crate", - suggested_limit)); - err.emit(); - self.cx.trace_macros_diag(); - FatalError.raise(); - } - - let (fragment_kind, span) = (invoc.fragment_kind, invoc.span()); - match invoc.kind { - InvocationKind::Bang { mac, .. } => match ext { - SyntaxExtensionKind::Bang(expander) => { - self.gate_proc_macro_expansion_kind(span, fragment_kind); - let tok_result = expander.expand(self.cx, span, mac.stream()); - let result = - self.parse_ast_fragment(tok_result, fragment_kind, &mac.path, span); - self.gate_proc_macro_expansion(span, &result); - result - } - SyntaxExtensionKind::LegacyBang(expander) => { - let prev = self.cx.current_expansion.prior_type_ascription; - self.cx.current_expansion.prior_type_ascription = mac.prior_type_ascription; - let tok_result = expander.expand(self.cx, span, mac.stream()); - let result = if let Some(result) = fragment_kind.make_from(tok_result) { - result - } else { - let msg = format!("non-{kind} macro in {kind} position: {path}", - kind = fragment_kind.name(), path = mac.path); - self.cx.span_err(span, &msg); - self.cx.trace_macros_diag(); - fragment_kind.dummy(span) - }; - self.cx.current_expansion.prior_type_ascription = prev; - result - } - _ => unreachable!() - } - InvocationKind::Attr { attr, mut item, .. } => match ext { - SyntaxExtensionKind::Attr(expander) => { - self.gate_proc_macro_attr_item(span, &item); - let item_tok = TokenTree::token(token::Interpolated(Lrc::new(match item { - Annotatable::Item(item) => token::NtItem(item), - Annotatable::TraitItem(item) => token::NtTraitItem(item.into_inner()), - Annotatable::ImplItem(item) => token::NtImplItem(item.into_inner()), - Annotatable::ForeignItem(item) => token::NtForeignItem(item.into_inner()), - Annotatable::Stmt(stmt) => token::NtStmt(stmt.into_inner()), - Annotatable::Expr(expr) => token::NtExpr(expr), - Annotatable::Arm(..) - | Annotatable::Field(..) - | Annotatable::FieldPat(..) - | Annotatable::GenericParam(..) - | Annotatable::Param(..) - | Annotatable::StructField(..) - | Annotatable::Variant(..) - => panic!("unexpected annotatable"), - })), DUMMY_SP).into(); - let input = self.extract_proc_macro_attr_input(attr.item.tokens, span); - let tok_result = expander.expand(self.cx, span, input, item_tok); - let res = - self.parse_ast_fragment(tok_result, fragment_kind, &attr.item.path, span); - self.gate_proc_macro_expansion(span, &res); - res - } - SyntaxExtensionKind::LegacyAttr(expander) => { - match attr.parse_meta(self.cx.parse_sess) { - Ok(meta) => { - let item = expander.expand(self.cx, span, &meta, item); - fragment_kind.expect_from_annotatables(item) - } - Err(mut err) => { - err.emit(); - fragment_kind.dummy(span) - } - } - } - SyntaxExtensionKind::NonMacroAttr { mark_used } => { - attr::mark_known(&attr); - if *mark_used { - attr::mark_used(&attr); - } - item.visit_attrs(|attrs| attrs.push(attr)); - fragment_kind.expect_from_annotatables(iter::once(item)) - } - _ => unreachable!() - } - InvocationKind::Derive { path, item } => match ext { - SyntaxExtensionKind::Derive(expander) | - SyntaxExtensionKind::LegacyDerive(expander) => { - if !item.derive_allowed() { - return fragment_kind.dummy(span); - } - let meta = ast::MetaItem { kind: ast::MetaItemKind::Word, span, path }; - let items = expander.expand(self.cx, span, &meta, item); - fragment_kind.expect_from_annotatables(items) - } - _ => unreachable!() - } - InvocationKind::DeriveContainer { .. } => unreachable!() - } - } - - fn extract_proc_macro_attr_input(&self, tokens: TokenStream, span: Span) -> TokenStream { - let mut trees = tokens.trees(); - match trees.next() { - Some(TokenTree::Delimited(_, _, tts)) => { - if trees.next().is_none() { - return tts.into() - } - } - Some(TokenTree::Token(..)) => {} - None => return TokenStream::empty(), - } - self.cx.span_err(span, "custom attribute invocations must be \ - of the form `#[foo]` or `#[foo(..)]`, the macro name must only be \ - followed by a delimiter token"); - TokenStream::empty() - } - - fn gate_proc_macro_attr_item(&self, span: Span, item: &Annotatable) { - let (kind, gate) = match *item { - Annotatable::Item(ref item) => { - match item.kind { - ItemKind::Mod(_) if self.cx.ecfg.proc_macro_hygiene() => return, - ItemKind::Mod(_) => ("modules", sym::proc_macro_hygiene), - _ => return, - } - } - Annotatable::TraitItem(_) => return, - Annotatable::ImplItem(_) => return, - Annotatable::ForeignItem(_) => return, - Annotatable::Stmt(_) | - Annotatable::Expr(_) if self.cx.ecfg.proc_macro_hygiene() => return, - Annotatable::Stmt(_) => ("statements", sym::proc_macro_hygiene), - Annotatable::Expr(_) => ("expressions", sym::proc_macro_hygiene), - Annotatable::Arm(..) - | Annotatable::Field(..) - | Annotatable::FieldPat(..) - | Annotatable::GenericParam(..) - | Annotatable::Param(..) - | Annotatable::StructField(..) - | Annotatable::Variant(..) - => panic!("unexpected annotatable"), - }; - emit_feature_err( - self.cx.parse_sess, - gate, - span, - GateIssue::Language, - &format!("custom attributes cannot be applied to {}", kind), - ); - } - - fn gate_proc_macro_expansion(&self, span: Span, fragment: &AstFragment) { - if self.cx.ecfg.proc_macro_hygiene() { - return - } - - fragment.visit_with(&mut DisallowMacros { - span, - parse_sess: self.cx.parse_sess, - }); - - struct DisallowMacros<'a> { - span: Span, - parse_sess: &'a ParseSess, - } - - impl<'ast, 'a> Visitor<'ast> for DisallowMacros<'a> { - fn visit_item(&mut self, i: &'ast ast::Item) { - if let ast::ItemKind::MacroDef(_) = i.kind { - emit_feature_err( - self.parse_sess, - sym::proc_macro_hygiene, - self.span, - GateIssue::Language, - "procedural macros cannot expand to macro definitions", - ); - } - visit::walk_item(self, i); - } - - fn visit_mac(&mut self, _mac: &'ast ast::Mac) { - // ... - } - } - } - - fn gate_proc_macro_expansion_kind(&self, span: Span, kind: AstFragmentKind) { - let kind = match kind { - AstFragmentKind::Expr | - AstFragmentKind::OptExpr => "expressions", - AstFragmentKind::Pat => "patterns", - AstFragmentKind::Stmts => "statements", - AstFragmentKind::Ty | - AstFragmentKind::Items | - AstFragmentKind::TraitItems | - AstFragmentKind::ImplItems | - AstFragmentKind::ForeignItems => return, - AstFragmentKind::Arms - | AstFragmentKind::Fields - | AstFragmentKind::FieldPats - | AstFragmentKind::GenericParams - | AstFragmentKind::Params - | AstFragmentKind::StructFields - | AstFragmentKind::Variants - => panic!("unexpected AST fragment kind"), - }; - if self.cx.ecfg.proc_macro_hygiene() { - return - } - emit_feature_err( - self.cx.parse_sess, - sym::proc_macro_hygiene, - span, - GateIssue::Language, - &format!("procedural macros cannot be expanded to {}", kind), - ); - } - - fn parse_ast_fragment( - &mut self, - toks: TokenStream, - kind: AstFragmentKind, - path: &Path, - span: Span, - ) -> AstFragment { - let mut parser = self.cx.new_parser_from_tts(toks); - match parser.parse_ast_fragment(kind, false) { - Ok(fragment) => { - parser.ensure_complete_parse(path, kind.name(), span); - fragment - } - Err(mut err) => { - err.set_span(span); - annotate_err_with_kind(&mut err, kind, span); - err.emit(); - self.cx.trace_macros_diag(); - kind.dummy(span) - } - } - } -} - -impl<'a> Parser<'a> { - pub fn parse_ast_fragment(&mut self, kind: AstFragmentKind, macro_legacy_warnings: bool) - -> PResult<'a, AstFragment> { - Ok(match kind { - AstFragmentKind::Items => { - let mut items = SmallVec::new(); - while let Some(item) = self.parse_item()? { - items.push(item); - } - AstFragment::Items(items) - } - AstFragmentKind::TraitItems => { - let mut items = SmallVec::new(); - while self.token != token::Eof { - items.push(self.parse_trait_item(&mut false)?); - } - AstFragment::TraitItems(items) - } - AstFragmentKind::ImplItems => { - let mut items = SmallVec::new(); - while self.token != token::Eof { - items.push(self.parse_impl_item(&mut false)?); - } - AstFragment::ImplItems(items) - } - AstFragmentKind::ForeignItems => { - let mut items = SmallVec::new(); - while self.token != token::Eof { - items.push(self.parse_foreign_item(DUMMY_SP)?); - } - AstFragment::ForeignItems(items) - } - AstFragmentKind::Stmts => { - let mut stmts = SmallVec::new(); - while self.token != token::Eof && - // won't make progress on a `}` - self.token != token::CloseDelim(token::Brace) { - if let Some(stmt) = self.parse_full_stmt(macro_legacy_warnings)? { - stmts.push(stmt); - } - } - AstFragment::Stmts(stmts) - } - AstFragmentKind::Expr => AstFragment::Expr(self.parse_expr()?), - AstFragmentKind::OptExpr => { - if self.token != token::Eof { - AstFragment::OptExpr(Some(self.parse_expr()?)) - } else { - AstFragment::OptExpr(None) - } - }, - AstFragmentKind::Ty => AstFragment::Ty(self.parse_ty()?), - AstFragmentKind::Pat => AstFragment::Pat(self.parse_pat(None)?), - AstFragmentKind::Arms - | AstFragmentKind::Fields - | AstFragmentKind::FieldPats - | AstFragmentKind::GenericParams - | AstFragmentKind::Params - | AstFragmentKind::StructFields - | AstFragmentKind::Variants - => panic!("unexpected AST fragment kind"), - }) - } - - pub fn ensure_complete_parse(&mut self, macro_path: &Path, kind_name: &str, span: Span) { - if self.token != token::Eof { - let msg = format!("macro expansion ignores token `{}` and any following", - self.this_token_to_string()); - // Avoid emitting backtrace info twice. - let def_site_span = self.token.span.with_ctxt(SyntaxContext::root()); - let mut err = self.diagnostic().struct_span_err(def_site_span, &msg); - err.span_label(span, "caused by the macro expansion here"); - let msg = format!( - "the usage of `{}!` is likely invalid in {} context", - macro_path, - kind_name, - ); - err.note(&msg); - let semi_span = self.sess.source_map().next_point(span); - - let semi_full_span = semi_span.to(self.sess.source_map().next_point(semi_span)); - match self.sess.source_map().span_to_snippet(semi_full_span) { - Ok(ref snippet) if &snippet[..] != ";" && kind_name == "expression" => { - err.span_suggestion( - semi_span, - "you might be missing a semicolon here", - ";".to_owned(), - Applicability::MaybeIncorrect, - ); - } - _ => {} - } - err.emit(); - } - } -} - -struct InvocationCollector<'a, 'b> { - cx: &'a mut ExtCtxt<'b>, - cfg: StripUnconfigured<'a>, - invocations: Vec<Invocation>, - monotonic: bool, -} - -impl<'a, 'b> InvocationCollector<'a, 'b> { - fn collect(&mut self, fragment_kind: AstFragmentKind, kind: InvocationKind) -> AstFragment { - // Expansion data for all the collected invocations is set upon their resolution, - // with exception of the derive container case which is not resolved and can get - // its expansion data immediately. - let expn_data = match &kind { - InvocationKind::DeriveContainer { item, .. } => Some(ExpnData { - parent: self.cx.current_expansion.id, - ..ExpnData::default( - ExpnKind::Macro(MacroKind::Attr, sym::derive), - item.span(), self.cx.parse_sess.edition, - ) - }), - _ => None, - }; - let expn_id = ExpnId::fresh(expn_data); - self.invocations.push(Invocation { - kind, - fragment_kind, - expansion_data: ExpansionData { - id: expn_id, - depth: self.cx.current_expansion.depth + 1, - ..self.cx.current_expansion.clone() - }, - }); - placeholder(fragment_kind, NodeId::placeholder_from_expn_id(expn_id)) - } - - fn collect_bang(&mut self, mac: ast::Mac, span: Span, kind: AstFragmentKind) -> AstFragment { - self.collect(kind, InvocationKind::Bang { mac, span }) - } - - fn collect_attr(&mut self, - attr: Option<ast::Attribute>, - derives: Vec<Path>, - item: Annotatable, - kind: AstFragmentKind, - after_derive: bool) - -> AstFragment { - self.collect(kind, match attr { - Some(attr) => InvocationKind::Attr { attr, item, derives, after_derive }, - None => InvocationKind::DeriveContainer { derives, item }, - }) - } - - fn find_attr_invoc(&self, attrs: &mut Vec<ast::Attribute>, after_derive: &mut bool) - -> Option<ast::Attribute> { - let attr = attrs.iter() - .position(|a| { - if a.path == sym::derive { - *after_derive = true; - } - !attr::is_known(a) && !is_builtin_attr(a) - }) - .map(|i| attrs.remove(i)); - if let Some(attr) = &attr { - if !self.cx.ecfg.custom_inner_attributes() && - attr.style == ast::AttrStyle::Inner && attr.path != sym::test { - emit_feature_err(&self.cx.parse_sess, sym::custom_inner_attributes, - attr.span, GateIssue::Language, - "non-builtin inner attributes are unstable"); - } - } - attr - } - - /// If `item` is an attr invocation, remove and return the macro attribute and derive traits. - fn classify_item<T>(&mut self, item: &mut T) - -> (Option<ast::Attribute>, Vec<Path>, /* after_derive */ bool) - where T: HasAttrs, - { - let (mut attr, mut traits, mut after_derive) = (None, Vec::new(), false); - - item.visit_attrs(|mut attrs| { - attr = self.find_attr_invoc(&mut attrs, &mut after_derive); - traits = collect_derives(&mut self.cx, &mut attrs); - }); - - (attr, traits, after_derive) - } - - /// Alternative to `classify_item()` that ignores `#[derive]` so invocations fallthrough - /// to the unused-attributes lint (making it an error on statements and expressions - /// is a breaking change) - fn classify_nonitem<T: HasAttrs>(&mut self, nonitem: &mut T) - -> (Option<ast::Attribute>, /* after_derive */ bool) { - let (mut attr, mut after_derive) = (None, false); - - nonitem.visit_attrs(|mut attrs| { - attr = self.find_attr_invoc(&mut attrs, &mut after_derive); - }); - - (attr, after_derive) - } - - fn configure<T: HasAttrs>(&mut self, node: T) -> Option<T> { - self.cfg.configure(node) - } - - // Detect use of feature-gated or invalid attributes on macro invocations - // since they will not be detected after macro expansion. - fn check_attributes(&mut self, attrs: &[ast::Attribute]) { - let features = self.cx.ecfg.features.unwrap(); - for attr in attrs.iter() { - feature_gate::check_attribute(attr, self.cx.parse_sess, features); - - // macros are expanded before any lint passes so this warning has to be hardcoded - if attr.path == sym::derive { - self.cx.struct_span_warn(attr.span, "`#[derive]` does nothing on macro invocations") - .note("this may become a hard error in a future release") - .emit(); - } - } - } -} - -impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> { - fn visit_expr(&mut self, expr: &mut P<ast::Expr>) { - self.cfg.configure_expr(expr); - visit_clobber(expr.deref_mut(), |mut expr| { - self.cfg.configure_expr_kind(&mut expr.kind); - - // ignore derives so they remain unused - let (attr, after_derive) = self.classify_nonitem(&mut expr); - - if attr.is_some() { - // Collect the invoc regardless of whether or not attributes are permitted here - // expansion will eat the attribute so it won't error later. - attr.as_ref().map(|a| self.cfg.maybe_emit_expr_attr_err(a)); - - // AstFragmentKind::Expr requires the macro to emit an expression. - return self.collect_attr(attr, vec![], Annotatable::Expr(P(expr)), - AstFragmentKind::Expr, after_derive) - .make_expr() - .into_inner() - } - - if let ast::ExprKind::Mac(mac) = expr.kind { - self.check_attributes(&expr.attrs); - self.collect_bang(mac, expr.span, AstFragmentKind::Expr) - .make_expr() - .into_inner() - } else { - noop_visit_expr(&mut expr, self); - expr - } - }); - } - - fn flat_map_arm(&mut self, arm: ast::Arm) -> SmallVec<[ast::Arm; 1]> { - let mut arm = configure!(self, arm); - - let (attr, traits, after_derive) = self.classify_item(&mut arm); - if attr.is_some() || !traits.is_empty() { - return self.collect_attr(attr, traits, Annotatable::Arm(arm), - AstFragmentKind::Arms, after_derive) - .make_arms(); - } - - noop_flat_map_arm(arm, self) - } - - fn flat_map_field(&mut self, field: ast::Field) -> SmallVec<[ast::Field; 1]> { - let mut field = configure!(self, field); - - let (attr, traits, after_derive) = self.classify_item(&mut field); - if attr.is_some() || !traits.is_empty() { - return self.collect_attr(attr, traits, Annotatable::Field(field), - AstFragmentKind::Fields, after_derive) - .make_fields(); - } - - noop_flat_map_field(field, self) - } - - fn flat_map_field_pattern(&mut self, fp: ast::FieldPat) -> SmallVec<[ast::FieldPat; 1]> { - let mut fp = configure!(self, fp); - - let (attr, traits, after_derive) = self.classify_item(&mut fp); - if attr.is_some() || !traits.is_empty() { - return self.collect_attr(attr, traits, Annotatable::FieldPat(fp), - AstFragmentKind::FieldPats, after_derive) - .make_field_patterns(); - } - - noop_flat_map_field_pattern(fp, self) - } - - fn flat_map_param(&mut self, p: ast::Param) -> SmallVec<[ast::Param; 1]> { - let mut p = configure!(self, p); - - let (attr, traits, after_derive) = self.classify_item(&mut p); - if attr.is_some() || !traits.is_empty() { - return self.collect_attr(attr, traits, Annotatable::Param(p), - AstFragmentKind::Params, after_derive) - .make_params(); - } - - noop_flat_map_param(p, self) - } - - fn flat_map_struct_field(&mut self, sf: ast::StructField) -> SmallVec<[ast::StructField; 1]> { - let mut sf = configure!(self, sf); - - let (attr, traits, after_derive) = self.classify_item(&mut sf); - if attr.is_some() || !traits.is_empty() { - return self.collect_attr(attr, traits, Annotatable::StructField(sf), - AstFragmentKind::StructFields, after_derive) - .make_struct_fields(); - } - - noop_flat_map_struct_field(sf, self) - } - - fn flat_map_variant(&mut self, variant: ast::Variant) -> SmallVec<[ast::Variant; 1]> { - let mut variant = configure!(self, variant); - - let (attr, traits, after_derive) = self.classify_item(&mut variant); - if attr.is_some() || !traits.is_empty() { - return self.collect_attr(attr, traits, Annotatable::Variant(variant), - AstFragmentKind::Variants, after_derive) - .make_variants(); - } - - noop_flat_map_variant(variant, self) - } - - fn filter_map_expr(&mut self, expr: P<ast::Expr>) -> Option<P<ast::Expr>> { - let expr = configure!(self, expr); - expr.filter_map(|mut expr| { - self.cfg.configure_expr_kind(&mut expr.kind); - - // Ignore derives so they remain unused. - let (attr, after_derive) = self.classify_nonitem(&mut expr); - - if attr.is_some() { - attr.as_ref().map(|a| self.cfg.maybe_emit_expr_attr_err(a)); - - return self.collect_attr(attr, vec![], Annotatable::Expr(P(expr)), - AstFragmentKind::OptExpr, after_derive) - .make_opt_expr() - .map(|expr| expr.into_inner()) - } - - if let ast::ExprKind::Mac(mac) = expr.kind { - self.check_attributes(&expr.attrs); - self.collect_bang(mac, expr.span, AstFragmentKind::OptExpr) - .make_opt_expr() - .map(|expr| expr.into_inner()) - } else { - Some({ noop_visit_expr(&mut expr, self); expr }) - } - }) - } - - fn visit_pat(&mut self, pat: &mut P<ast::Pat>) { - self.cfg.configure_pat(pat); - match pat.kind { - PatKind::Mac(_) => {} - _ => return noop_visit_pat(pat, self), - } - - visit_clobber(pat, |mut pat| { - match mem::replace(&mut pat.kind, PatKind::Wild) { - PatKind::Mac(mac) => - self.collect_bang(mac, pat.span, AstFragmentKind::Pat).make_pat(), - _ => unreachable!(), - } - }); - } - - fn flat_map_stmt(&mut self, stmt: ast::Stmt) -> SmallVec<[ast::Stmt; 1]> { - let mut stmt = configure!(self, stmt); - - // we'll expand attributes on expressions separately - if !stmt.is_expr() { - let (attr, derives, after_derive) = if stmt.is_item() { - self.classify_item(&mut stmt) - } else { - // ignore derives on non-item statements so it falls through - // to the unused-attributes lint - let (attr, after_derive) = self.classify_nonitem(&mut stmt); - (attr, vec![], after_derive) - }; - - if attr.is_some() || !derives.is_empty() { - return self.collect_attr(attr, derives, Annotatable::Stmt(P(stmt)), - AstFragmentKind::Stmts, after_derive).make_stmts(); - } - } - - if let StmtKind::Mac(mac) = stmt.kind { - let (mac, style, attrs) = mac.into_inner(); - self.check_attributes(&attrs); - let mut placeholder = self.collect_bang(mac, stmt.span, AstFragmentKind::Stmts) - .make_stmts(); - - // If this is a macro invocation with a semicolon, then apply that - // semicolon to the final statement produced by expansion. - if style == MacStmtStyle::Semicolon { - if let Some(stmt) = placeholder.pop() { - placeholder.push(stmt.add_trailing_semicolon()); - } - } - - return placeholder; - } - - // The placeholder expander gives ids to statements, so we avoid folding the id here. - let ast::Stmt { id, kind, span } = stmt; - noop_flat_map_stmt_kind(kind, self).into_iter().map(|kind| { - ast::Stmt { id, kind, span } - }).collect() - - } - - fn visit_block(&mut self, block: &mut P<Block>) { - let old_directory_ownership = self.cx.current_expansion.directory_ownership; - self.cx.current_expansion.directory_ownership = DirectoryOwnership::UnownedViaBlock; - noop_visit_block(block, self); - self.cx.current_expansion.directory_ownership = old_directory_ownership; - } - - fn flat_map_item(&mut self, item: P<ast::Item>) -> SmallVec<[P<ast::Item>; 1]> { - let mut item = configure!(self, item); - - let (attr, traits, after_derive) = self.classify_item(&mut item); - if attr.is_some() || !traits.is_empty() { - return self.collect_attr(attr, traits, Annotatable::Item(item), - AstFragmentKind::Items, after_derive).make_items(); - } - - match item.kind { - ast::ItemKind::Mac(..) => { - self.check_attributes(&item.attrs); - item.and_then(|item| match item.kind { - ItemKind::Mac(mac) => self.collect( - AstFragmentKind::Items, InvocationKind::Bang { mac, span: item.span } - ).make_items(), - _ => unreachable!(), - }) - } - ast::ItemKind::Mod(ast::Mod { inner, .. }) => { - if item.ident == Ident::invalid() { - return noop_flat_map_item(item, self); - } - - let orig_directory_ownership = self.cx.current_expansion.directory_ownership; - let mut module = (*self.cx.current_expansion.module).clone(); - module.mod_path.push(item.ident); - - // Detect if this is an inline module (`mod m { ... }` as opposed to `mod m;`). - // In the non-inline case, `inner` is never the dummy span (cf. `parse_item_mod`). - // Thus, if `inner` is the dummy span, we know the module is inline. - let inline_module = item.span.contains(inner) || inner.is_dummy(); - - if inline_module { - if let Some(path) = attr::first_attr_value_str_by_name(&item.attrs, sym::path) { - self.cx.current_expansion.directory_ownership = - DirectoryOwnership::Owned { relative: None }; - module.directory.push(&*path.as_str()); - } else { - module.directory.push(&*item.ident.as_str()); - } - } else { - let path = self.cx.parse_sess.source_map().span_to_unmapped_path(inner); - let mut path = match path { - FileName::Real(path) => path, - other => PathBuf::from(other.to_string()), - }; - let directory_ownership = match path.file_name().unwrap().to_str() { - Some("mod.rs") => DirectoryOwnership::Owned { relative: None }, - Some(_) => DirectoryOwnership::Owned { - relative: Some(item.ident), - }, - None => DirectoryOwnership::UnownedViaMod(false), - }; - path.pop(); - module.directory = path; - self.cx.current_expansion.directory_ownership = directory_ownership; - } - - let orig_module = - mem::replace(&mut self.cx.current_expansion.module, Rc::new(module)); - let result = noop_flat_map_item(item, self); - self.cx.current_expansion.module = orig_module; - self.cx.current_expansion.directory_ownership = orig_directory_ownership; - result - } - - _ => noop_flat_map_item(item, self), - } - } - - fn flat_map_trait_item(&mut self, item: ast::TraitItem) -> SmallVec<[ast::TraitItem; 1]> { - let mut item = configure!(self, item); - - let (attr, traits, after_derive) = self.classify_item(&mut item); - if attr.is_some() || !traits.is_empty() { - return self.collect_attr(attr, traits, Annotatable::TraitItem(P(item)), - AstFragmentKind::TraitItems, after_derive).make_trait_items() - } - - match item.kind { - ast::TraitItemKind::Macro(mac) => { - let ast::TraitItem { attrs, span, .. } = item; - self.check_attributes(&attrs); - self.collect_bang(mac, span, AstFragmentKind::TraitItems).make_trait_items() - } - _ => noop_flat_map_trait_item(item, self), - } - } - - fn flat_map_impl_item(&mut self, item: ast::ImplItem) -> SmallVec<[ast::ImplItem; 1]> { - let mut item = configure!(self, item); - - let (attr, traits, after_derive) = self.classify_item(&mut item); - if attr.is_some() || !traits.is_empty() { - return self.collect_attr(attr, traits, Annotatable::ImplItem(P(item)), - AstFragmentKind::ImplItems, after_derive).make_impl_items(); - } - - match item.kind { - ast::ImplItemKind::Macro(mac) => { - let ast::ImplItem { attrs, span, .. } = item; - self.check_attributes(&attrs); - self.collect_bang(mac, span, AstFragmentKind::ImplItems).make_impl_items() - } - _ => noop_flat_map_impl_item(item, self), - } - } - - fn visit_ty(&mut self, ty: &mut P<ast::Ty>) { - match ty.kind { - ast::TyKind::Mac(_) => {} - _ => return noop_visit_ty(ty, self), - }; - - visit_clobber(ty, |mut ty| { - match mem::replace(&mut ty.kind, ast::TyKind::Err) { - ast::TyKind::Mac(mac) => - self.collect_bang(mac, ty.span, AstFragmentKind::Ty).make_ty(), - _ => unreachable!(), - } - }); - } - - fn visit_foreign_mod(&mut self, foreign_mod: &mut ast::ForeignMod) { - self.cfg.configure_foreign_mod(foreign_mod); - noop_visit_foreign_mod(foreign_mod, self); - } - - fn flat_map_foreign_item(&mut self, mut foreign_item: ast::ForeignItem) - -> SmallVec<[ast::ForeignItem; 1]> - { - let (attr, traits, after_derive) = self.classify_item(&mut foreign_item); - - if attr.is_some() || !traits.is_empty() { - return self.collect_attr(attr, traits, Annotatable::ForeignItem(P(foreign_item)), - AstFragmentKind::ForeignItems, after_derive) - .make_foreign_items(); - } - - if let ast::ForeignItemKind::Macro(mac) = foreign_item.kind { - self.check_attributes(&foreign_item.attrs); - return self.collect_bang(mac, foreign_item.span, AstFragmentKind::ForeignItems) - .make_foreign_items(); - } - - noop_flat_map_foreign_item(foreign_item, self) - } - - fn visit_item_kind(&mut self, item: &mut ast::ItemKind) { - match item { - ast::ItemKind::MacroDef(..) => {} - _ => { - self.cfg.configure_item_kind(item); - noop_visit_item_kind(item, self); - } - } - } - - fn flat_map_generic_param( - &mut self, - param: ast::GenericParam - ) -> SmallVec<[ast::GenericParam; 1]> - { - let mut param = configure!(self, param); - - let (attr, traits, after_derive) = self.classify_item(&mut param); - if attr.is_some() || !traits.is_empty() { - return self.collect_attr(attr, traits, Annotatable::GenericParam(param), - AstFragmentKind::GenericParams, after_derive) - .make_generic_params(); - } - - noop_flat_map_generic_param(param, self) - } - - fn visit_attribute(&mut self, at: &mut ast::Attribute) { - // turn `#[doc(include="filename")]` attributes into `#[doc(include(file="filename", - // contents="file contents")]` attributes - if !at.check_name(sym::doc) { - return noop_visit_attribute(at, self); - } - - if let Some(list) = at.meta_item_list() { - if !list.iter().any(|it| it.check_name(sym::include)) { - return noop_visit_attribute(at, self); - } - - let mut items = vec![]; - - for mut it in list { - if !it.check_name(sym::include) { - items.push({ noop_visit_meta_list_item(&mut it, self); it }); - continue; - } - - if let Some(file) = it.value_str() { - let err_count = self.cx.parse_sess.span_diagnostic.err_count(); - self.check_attributes(slice::from_ref(at)); - if self.cx.parse_sess.span_diagnostic.err_count() > err_count { - // avoid loading the file if they haven't enabled the feature - return noop_visit_attribute(at, self); - } - - let filename = self.cx.resolve_path(&*file.as_str(), it.span()); - match self.cx.source_map().load_file(&filename) { - Ok(source_file) => { - let src = source_file.src.as_ref() - .expect("freshly loaded file should have a source"); - let src_interned = Symbol::intern(src.as_str()); - - let include_info = vec![ - ast::NestedMetaItem::MetaItem( - attr::mk_name_value_item_str( - Ident::with_dummy_span(sym::file), - file, - DUMMY_SP, - ), - ), - ast::NestedMetaItem::MetaItem( - attr::mk_name_value_item_str( - Ident::with_dummy_span(sym::contents), - src_interned, - DUMMY_SP, - ), - ), - ]; - - let include_ident = Ident::with_dummy_span(sym::include); - let item = attr::mk_list_item(include_ident, include_info); - items.push(ast::NestedMetaItem::MetaItem(item)); - } - Err(e) => { - let lit = it - .meta_item() - .and_then(|item| item.name_value_literal()) - .unwrap(); - - if e.kind() == ErrorKind::InvalidData { - self.cx - .struct_span_err( - lit.span, - &format!("{} wasn't a utf-8 file", filename.display()), - ) - .span_label(lit.span, "contains invalid utf-8") - .emit(); - } else { - let mut err = self.cx.struct_span_err( - lit.span, - &format!("couldn't read {}: {}", filename.display(), e), - ); - err.span_label(lit.span, "couldn't read file"); - - err.emit(); - } - } - } - } else { - let mut err = self.cx.struct_span_err( - it.span(), - &format!("expected path to external documentation"), - ); - - // Check if the user erroneously used `doc(include(...))` syntax. - let literal = it.meta_item_list().and_then(|list| { - if list.len() == 1 { - list[0].literal().map(|literal| &literal.kind) - } else { - None - } - }); - - let (path, applicability) = match &literal { - Some(LitKind::Str(path, ..)) => { - (path.to_string(), Applicability::MachineApplicable) - } - _ => (String::from("<path>"), Applicability::HasPlaceholders), - }; - - err.span_suggestion( - it.span(), - "provide a file path with `=`", - format!("include = \"{}\"", path), - applicability, - ); - - err.emit(); - } - } - - let meta = attr::mk_list_item(Ident::with_dummy_span(sym::doc), items); - *at = attr::Attribute { - item: AttrItem { path: meta.path, tokens: meta.kind.tokens(meta.span) }, - span: at.span, - id: at.id, - style: at.style, - is_sugared_doc: false, - }; - } else { - noop_visit_attribute(at, self) - } - } - - fn visit_id(&mut self, id: &mut ast::NodeId) { - if self.monotonic { - debug_assert_eq!(*id, ast::DUMMY_NODE_ID); - *id = self.cx.resolver.next_node_id() - } - } - - fn visit_fn_decl(&mut self, mut fn_decl: &mut P<ast::FnDecl>) { - self.cfg.configure_fn_decl(&mut fn_decl); - noop_visit_fn_decl(fn_decl, self); - } -} - -pub struct ExpansionConfig<'feat> { - pub crate_name: String, - pub features: Option<&'feat Features>, - pub recursion_limit: usize, - pub trace_mac: bool, - pub should_test: bool, // If false, strip `#[test]` nodes - pub single_step: bool, - pub keep_macs: bool, -} - -impl<'feat> ExpansionConfig<'feat> { - pub fn default(crate_name: String) -> ExpansionConfig<'static> { - ExpansionConfig { - crate_name, - features: None, - recursion_limit: 1024, - trace_mac: false, - should_test: false, - single_step: false, - keep_macs: false, - } - } - - fn proc_macro_hygiene(&self) -> bool { - self.features.map_or(false, |features| features.proc_macro_hygiene) - } - fn custom_inner_attributes(&self) -> bool { - self.features.map_or(false, |features| features.custom_inner_attributes) - } -} diff --git a/src/libsyntax/ext/mbe.rs b/src/libsyntax/ext/mbe.rs deleted file mode 100644 index a87da791c9b..00000000000 --- a/src/libsyntax/ext/mbe.rs +++ /dev/null @@ -1,166 +0,0 @@ -//! This module implements declarative macros: old `macro_rules` and the newer -//! `macro`. Declarative macros are also known as "macro by example", and that's -//! why we call this module `mbe`. For external documentation, prefer the -//! official terminology: "declarative macros". - -crate mod transcribe; -crate mod macro_check; -crate mod macro_parser; -crate mod macro_rules; -crate mod quoted; - -use crate::ast; -use crate::parse::token::{self, Token, TokenKind}; -use crate::tokenstream::{DelimSpan}; - -use syntax_pos::{BytePos, Span}; - -use rustc_data_structures::sync::Lrc; - -/// Contains the sub-token-trees of a "delimited" token tree, such as the contents of `(`. Note -/// that the delimiter itself might be `NoDelim`. -#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug)] -struct Delimited { - delim: token::DelimToken, - tts: Vec<TokenTree>, -} - -impl Delimited { - /// Returns a `self::TokenTree` with a `Span` corresponding to the opening delimiter. - fn open_tt(&self, span: Span) -> TokenTree { - let open_span = if span.is_dummy() { - span - } else { - span.with_hi(span.lo() + BytePos(self.delim.len() as u32)) - }; - TokenTree::token(token::OpenDelim(self.delim), open_span) - } - - /// Returns a `self::TokenTree` with a `Span` corresponding to the closing delimiter. - fn close_tt(&self, span: Span) -> TokenTree { - let close_span = if span.is_dummy() { - span - } else { - span.with_lo(span.hi() - BytePos(self.delim.len() as u32)) - }; - TokenTree::token(token::CloseDelim(self.delim), close_span) - } -} - -#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug)] -struct SequenceRepetition { - /// The sequence of token trees - tts: Vec<TokenTree>, - /// The optional separator - separator: Option<Token>, - /// Whether the sequence can be repeated zero (*), or one or more times (+) - kleene: KleeneToken, - /// The number of `Match`s that appear in the sequence (and subsequences) - num_captures: usize, -} - -#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug, Copy)] -struct KleeneToken { - span: Span, - op: KleeneOp, -} - -impl KleeneToken { - fn new(op: KleeneOp, span: Span) -> KleeneToken { - KleeneToken { span, op } - } -} - -/// A Kleene-style [repetition operator](http://en.wikipedia.org/wiki/Kleene_star) -/// for token sequences. -#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] -enum KleeneOp { - /// Kleene star (`*`) for zero or more repetitions - ZeroOrMore, - /// Kleene plus (`+`) for one or more repetitions - OneOrMore, - /// Kleene optional (`?`) for zero or one reptitions - ZeroOrOne, -} - -/// Similar to `tokenstream::TokenTree`, except that `$i`, `$i:ident`, and `$(...)` -/// are "first-class" token trees. Useful for parsing macros. -#[derive(Debug, Clone, PartialEq, RustcEncodable, RustcDecodable)] -enum TokenTree { - Token(Token), - Delimited(DelimSpan, Lrc<Delimited>), - /// A kleene-style repetition sequence - Sequence(DelimSpan, Lrc<SequenceRepetition>), - /// e.g., `$var` - MetaVar(Span, ast::Ident), - /// e.g., `$var:expr`. This is only used in the left hand side of MBE macros. - MetaVarDecl( - Span, - ast::Ident, /* name to bind */ - ast::Ident, /* kind of nonterminal */ - ), -} - -impl TokenTree { - /// Return the number of tokens in the tree. - fn len(&self) -> usize { - match *self { - TokenTree::Delimited(_, ref delimed) => match delimed.delim { - token::NoDelim => delimed.tts.len(), - _ => delimed.tts.len() + 2, - }, - TokenTree::Sequence(_, ref seq) => seq.tts.len(), - _ => 0, - } - } - - /// Returns `true` if the given token tree is delimited. - fn is_delimited(&self) -> bool { - match *self { - TokenTree::Delimited(..) => true, - _ => false, - } - } - - /// Returns `true` if the given token tree is a token of the given kind. - fn is_token(&self, expected_kind: &TokenKind) -> bool { - match self { - TokenTree::Token(Token { kind: actual_kind, .. }) => actual_kind == expected_kind, - _ => false, - } - } - - /// Gets the `index`-th sub-token-tree. This only makes sense for delimited trees and sequences. - fn get_tt(&self, index: usize) -> TokenTree { - match (self, index) { - (&TokenTree::Delimited(_, ref delimed), _) if delimed.delim == token::NoDelim => { - delimed.tts[index].clone() - } - (&TokenTree::Delimited(span, ref delimed), _) => { - if index == 0 { - return delimed.open_tt(span.open); - } - if index == delimed.tts.len() + 1 { - return delimed.close_tt(span.close); - } - delimed.tts[index - 1].clone() - } - (&TokenTree::Sequence(_, ref seq), _) => seq.tts[index].clone(), - _ => panic!("Cannot expand a token tree"), - } - } - - /// Retrieves the `TokenTree`'s span. - fn span(&self) -> Span { - match *self { - TokenTree::Token(Token { span, .. }) - | TokenTree::MetaVar(span, _) - | TokenTree::MetaVarDecl(span, _, _) => span, - TokenTree::Delimited(span, _) | TokenTree::Sequence(span, _) => span.entire(), - } - } - - fn token(kind: TokenKind, span: Span) -> TokenTree { - TokenTree::Token(Token::new(kind, span)) - } -} diff --git a/src/libsyntax/ext/mbe/macro_check.rs b/src/libsyntax/ext/mbe/macro_check.rs deleted file mode 100644 index 97074f5cbe4..00000000000 --- a/src/libsyntax/ext/mbe/macro_check.rs +++ /dev/null @@ -1,626 +0,0 @@ -//! Checks that meta-variables in macro definition are correctly declared and used. -//! -//! # What is checked -//! -//! ## Meta-variables must not be bound twice -//! -//! ``` -//! macro_rules! foo { ($x:tt $x:tt) => { $x }; } -//! ``` -//! -//! This check is sound (no false-negative) and complete (no false-positive). -//! -//! ## Meta-variables must not be free -//! -//! ``` -//! macro_rules! foo { () => { $x }; } -//! ``` -//! -//! This check is also done at macro instantiation but only if the branch is taken. -//! -//! ## Meta-variables must repeat at least as many times as their binder -//! -//! ``` -//! macro_rules! foo { ($($x:tt)*) => { $x }; } -//! ``` -//! -//! This check is also done at macro instantiation but only if the branch is taken. -//! -//! ## Meta-variables must repeat with the same Kleene operators as their binder -//! -//! ``` -//! macro_rules! foo { ($($x:tt)+) => { $($x)* }; } -//! ``` -//! -//! This check is not done at macro instantiation. -//! -//! # Disclaimer -//! -//! In the presence of nested macros (a macro defined in a macro), those checks may have false -//! positives and false negatives. We try to detect those cases by recognizing potential macro -//! definitions in RHSes, but nested macros may be hidden through the use of particular values of -//! meta-variables. -//! -//! ## Examples of false positive -//! -//! False positives can come from cases where we don't recognize a nested macro, because it depends -//! on particular values of meta-variables. In the following example, we think both instances of -//! `$x` are free, which is a correct statement if `$name` is anything but `macro_rules`. But when -//! `$name` is `macro_rules`, like in the instantiation below, then `$x:tt` is actually a binder of -//! the nested macro and `$x` is bound to it. -//! -//! ``` -//! macro_rules! foo { ($name:ident) => { $name! bar { ($x:tt) => { $x }; } }; } -//! foo!(macro_rules); -//! ``` -//! -//! False positives can also come from cases where we think there is a nested macro while there -//! isn't. In the following example, we think `$x` is free, which is incorrect because `bar` is not -//! a nested macro since it is not evaluated as code by `stringify!`. -//! -//! ``` -//! macro_rules! foo { () => { stringify!(macro_rules! bar { () => { $x }; }) }; } -//! ``` -//! -//! ## Examples of false negative -//! -//! False negatives can come from cases where we don't recognize a meta-variable, because it depends -//! on particular values of meta-variables. In the following examples, we don't see that if `$d` is -//! instantiated with `$` then `$d z` becomes `$z` in the nested macro definition and is thus a free -//! meta-variable. Note however, that if `foo` is instantiated, then we would check the definition -//! of `bar` and would see the issue. -//! -//! ``` -//! macro_rules! foo { ($d:tt) => { macro_rules! bar { ($y:tt) => { $d z }; } }; } -//! ``` -//! -//! # How it is checked -//! -//! There are 3 main functions: `check_binders`, `check_occurrences`, and `check_nested_macro`. They -//! all need some kind of environment. -//! -//! ## Environments -//! -//! Environments are used to pass information. -//! -//! ### From LHS to RHS -//! -//! When checking a LHS with `check_binders`, we produce (and use) an environment for binders, -//! namely `Binders`. This is a mapping from binder name to information about that binder: the span -//! of the binder for error messages and the stack of Kleene operators under which it was bound in -//! the LHS. -//! -//! This environment is used by both the LHS and RHS. The LHS uses it to detect duplicate binders. -//! The RHS uses it to detect the other errors. -//! -//! ### From outer macro to inner macro -//! -//! When checking the RHS of an outer macro and we detect a nested macro definition, we push the -//! current state, namely `MacroState`, to an environment of nested macro definitions. Each state -//! stores the LHS binders when entering the macro definition as well as the stack of Kleene -//! operators under which the inner macro is defined in the RHS. -//! -//! This environment is a stack representing the nesting of macro definitions. As such, the stack of -//! Kleene operators under which a meta-variable is repeating is the concatenation of the stacks -//! stored when entering a macro definition starting from the state in which the meta-variable is -//! bound. -use crate::ast::NodeId; -use crate::early_buffered_lints::BufferedEarlyLintId; -use crate::ext::mbe::{KleeneToken, TokenTree}; -use crate::parse::token::TokenKind; -use crate::parse::token::{DelimToken, Token}; -use crate::parse::ParseSess; -use crate::symbol::{kw, sym}; - -use rustc_data_structures::fx::FxHashMap; -use smallvec::SmallVec; -use syntax_pos::{symbol::Ident, MultiSpan, Span}; - -/// Stack represented as linked list. -/// -/// Those are used for environments because they grow incrementally and are not mutable. -enum Stack<'a, T> { - /// Empty stack. - Empty, - /// A non-empty stack. - Push { - /// The top element. - top: T, - /// The previous elements. - prev: &'a Stack<'a, T>, - }, -} - -impl<'a, T> Stack<'a, T> { - /// Returns whether a stack is empty. - fn is_empty(&self) -> bool { - match *self { - Stack::Empty => true, - _ => false, - } - } - - /// Returns a new stack with an element of top. - fn push(&'a self, top: T) -> Stack<'a, T> { - Stack::Push { top, prev: self } - } -} - -impl<'a, T> Iterator for &'a Stack<'a, T> { - type Item = &'a T; - - // Iterates from top to bottom of the stack. - fn next(&mut self) -> Option<&'a T> { - match *self { - Stack::Empty => None, - Stack::Push { ref top, ref prev } => { - *self = prev; - Some(top) - } - } - } -} - -impl From<&Stack<'_, KleeneToken>> for SmallVec<[KleeneToken; 1]> { - fn from(ops: &Stack<'_, KleeneToken>) -> SmallVec<[KleeneToken; 1]> { - let mut ops: SmallVec<[KleeneToken; 1]> = ops.cloned().collect(); - // The stack is innermost on top. We want outermost first. - ops.reverse(); - ops - } -} - -/// Information attached to a meta-variable binder in LHS. -struct BinderInfo { - /// The span of the meta-variable in LHS. - span: Span, - /// The stack of Kleene operators (outermost first). - ops: SmallVec<[KleeneToken; 1]>, -} - -/// An environment of meta-variables to their binder information. -type Binders = FxHashMap<Ident, BinderInfo>; - -/// The state at which we entered a macro definition in the RHS of another macro definition. -struct MacroState<'a> { - /// The binders of the branch where we entered the macro definition. - binders: &'a Binders, - /// The stack of Kleene operators (outermost first) where we entered the macro definition. - ops: SmallVec<[KleeneToken; 1]>, -} - -/// Checks that meta-variables are used correctly in a macro definition. -/// -/// Arguments: -/// - `sess` is used to emit diagnostics and lints -/// - `node_id` is used to emit lints -/// - `span` is used when no spans are available -/// - `lhses` and `rhses` should have the same length and represent the macro definition -pub(super) fn check_meta_variables( - sess: &ParseSess, - node_id: NodeId, - span: Span, - lhses: &[TokenTree], - rhses: &[TokenTree], -) -> bool { - if lhses.len() != rhses.len() { - sess.span_diagnostic.span_bug(span, "length mismatch between LHSes and RHSes") - } - let mut valid = true; - for (lhs, rhs) in lhses.iter().zip(rhses.iter()) { - let mut binders = Binders::default(); - check_binders(sess, node_id, lhs, &Stack::Empty, &mut binders, &Stack::Empty, &mut valid); - check_occurrences(sess, node_id, rhs, &Stack::Empty, &binders, &Stack::Empty, &mut valid); - } - valid -} - -/// Checks `lhs` as part of the LHS of a macro definition, extends `binders` with new binders, and -/// sets `valid` to false in case of errors. -/// -/// Arguments: -/// - `sess` is used to emit diagnostics and lints -/// - `node_id` is used to emit lints -/// - `lhs` is checked as part of a LHS -/// - `macros` is the stack of possible outer macros -/// - `binders` contains the binders of the LHS -/// - `ops` is the stack of Kleene operators from the LHS -/// - `valid` is set in case of errors -fn check_binders( - sess: &ParseSess, - node_id: NodeId, - lhs: &TokenTree, - macros: &Stack<'_, MacroState<'_>>, - binders: &mut Binders, - ops: &Stack<'_, KleeneToken>, - valid: &mut bool, -) { - match *lhs { - TokenTree::Token(..) => {} - // This can only happen when checking a nested macro because this LHS is then in the RHS of - // the outer macro. See ui/macros/macro-of-higher-order.rs where $y:$fragment in the - // LHS of the nested macro (and RHS of the outer macro) is parsed as MetaVar(y) Colon - // MetaVar(fragment) and not as MetaVarDecl(y, fragment). - TokenTree::MetaVar(span, name) => { - if macros.is_empty() { - sess.span_diagnostic.span_bug(span, "unexpected MetaVar in lhs"); - } - // There are 3 possibilities: - if let Some(prev_info) = binders.get(&name) { - // 1. The meta-variable is already bound in the current LHS: This is an error. - let mut span = MultiSpan::from_span(span); - span.push_span_label(prev_info.span, "previous declaration".into()); - buffer_lint(sess, span, node_id, "duplicate matcher binding"); - } else if get_binder_info(macros, binders, name).is_none() { - // 2. The meta-variable is free: This is a binder. - binders.insert(name, BinderInfo { span, ops: ops.into() }); - } else { - // 3. The meta-variable is bound: This is an occurrence. - check_occurrences(sess, node_id, lhs, macros, binders, ops, valid); - } - } - // Similarly, this can only happen when checking a toplevel macro. - TokenTree::MetaVarDecl(span, name, _kind) => { - if !macros.is_empty() { - sess.span_diagnostic.span_bug(span, "unexpected MetaVarDecl in nested lhs"); - } - if let Some(prev_info) = get_binder_info(macros, binders, name) { - // Duplicate binders at the top-level macro definition are errors. The lint is only - // for nested macro definitions. - sess.span_diagnostic - .struct_span_err(span, "duplicate matcher binding") - .span_note(prev_info.span, "previous declaration was here") - .emit(); - *valid = false; - } else { - binders.insert(name, BinderInfo { span, ops: ops.into() }); - } - } - TokenTree::Delimited(_, ref del) => { - for tt in &del.tts { - check_binders(sess, node_id, tt, macros, binders, ops, valid); - } - } - TokenTree::Sequence(_, ref seq) => { - let ops = ops.push(seq.kleene); - for tt in &seq.tts { - check_binders(sess, node_id, tt, macros, binders, &ops, valid); - } - } - } -} - -/// Returns the binder information of a meta-variable. -/// -/// Arguments: -/// - `macros` is the stack of possible outer macros -/// - `binders` contains the current binders -/// - `name` is the name of the meta-variable we are looking for -fn get_binder_info<'a>( - mut macros: &'a Stack<'a, MacroState<'a>>, - binders: &'a Binders, - name: Ident, -) -> Option<&'a BinderInfo> { - binders.get(&name).or_else(|| macros.find_map(|state| state.binders.get(&name))) -} - -/// Checks `rhs` as part of the RHS of a macro definition and sets `valid` to false in case of -/// errors. -/// -/// Arguments: -/// - `sess` is used to emit diagnostics and lints -/// - `node_id` is used to emit lints -/// - `rhs` is checked as part of a RHS -/// - `macros` is the stack of possible outer macros -/// - `binders` contains the binders of the associated LHS -/// - `ops` is the stack of Kleene operators from the RHS -/// - `valid` is set in case of errors -fn check_occurrences( - sess: &ParseSess, - node_id: NodeId, - rhs: &TokenTree, - macros: &Stack<'_, MacroState<'_>>, - binders: &Binders, - ops: &Stack<'_, KleeneToken>, - valid: &mut bool, -) { - match *rhs { - TokenTree::Token(..) => {} - TokenTree::MetaVarDecl(span, _name, _kind) => { - sess.span_diagnostic.span_bug(span, "unexpected MetaVarDecl in rhs") - } - TokenTree::MetaVar(span, name) => { - check_ops_is_prefix(sess, node_id, macros, binders, ops, span, name); - } - TokenTree::Delimited(_, ref del) => { - check_nested_occurrences(sess, node_id, &del.tts, macros, binders, ops, valid); - } - TokenTree::Sequence(_, ref seq) => { - let ops = ops.push(seq.kleene); - check_nested_occurrences(sess, node_id, &seq.tts, macros, binders, &ops, valid); - } - } -} - -/// Represents the processed prefix of a nested macro. -#[derive(Clone, Copy, PartialEq, Eq)] -enum NestedMacroState { - /// Nothing that matches a nested macro definition was processed yet. - Empty, - /// The token `macro_rules` was processed. - MacroRules, - /// The tokens `macro_rules!` were processed. - MacroRulesNot, - /// The tokens `macro_rules!` followed by a name were processed. The name may be either directly - /// an identifier or a meta-variable (that hopefully would be instantiated by an identifier). - MacroRulesNotName, - /// The keyword `macro` was processed. - Macro, - /// The keyword `macro` followed by a name was processed. - MacroName, - /// The keyword `macro` followed by a name and a token delimited by parentheses was processed. - MacroNameParen, -} - -/// Checks `tts` as part of the RHS of a macro definition, tries to recognize nested macro -/// definitions, and sets `valid` to false in case of errors. -/// -/// Arguments: -/// - `sess` is used to emit diagnostics and lints -/// - `node_id` is used to emit lints -/// - `tts` is checked as part of a RHS and may contain macro definitions -/// - `macros` is the stack of possible outer macros -/// - `binders` contains the binders of the associated LHS -/// - `ops` is the stack of Kleene operators from the RHS -/// - `valid` is set in case of errors -fn check_nested_occurrences( - sess: &ParseSess, - node_id: NodeId, - tts: &[TokenTree], - macros: &Stack<'_, MacroState<'_>>, - binders: &Binders, - ops: &Stack<'_, KleeneToken>, - valid: &mut bool, -) { - let mut state = NestedMacroState::Empty; - let nested_macros = macros.push(MacroState { binders, ops: ops.into() }); - let mut nested_binders = Binders::default(); - for tt in tts { - match (state, tt) { - ( - NestedMacroState::Empty, - &TokenTree::Token(Token { kind: TokenKind::Ident(name, false), .. }), - ) => { - if name == sym::macro_rules { - state = NestedMacroState::MacroRules; - } else if name == kw::Macro { - state = NestedMacroState::Macro; - } - } - ( - NestedMacroState::MacroRules, - &TokenTree::Token(Token { kind: TokenKind::Not, .. }), - ) => { - state = NestedMacroState::MacroRulesNot; - } - ( - NestedMacroState::MacroRulesNot, - &TokenTree::Token(Token { kind: TokenKind::Ident(..), .. }), - ) => { - state = NestedMacroState::MacroRulesNotName; - } - (NestedMacroState::MacroRulesNot, &TokenTree::MetaVar(..)) => { - state = NestedMacroState::MacroRulesNotName; - // We check that the meta-variable is correctly used. - check_occurrences(sess, node_id, tt, macros, binders, ops, valid); - } - (NestedMacroState::MacroRulesNotName, &TokenTree::Delimited(_, ref del)) - | (NestedMacroState::MacroName, &TokenTree::Delimited(_, ref del)) - if del.delim == DelimToken::Brace => - { - let legacy = state == NestedMacroState::MacroRulesNotName; - state = NestedMacroState::Empty; - let rest = - check_nested_macro(sess, node_id, legacy, &del.tts, &nested_macros, valid); - // If we did not check the whole macro definition, then check the rest as if outside - // the macro definition. - check_nested_occurrences( - sess, - node_id, - &del.tts[rest..], - macros, - binders, - ops, - valid, - ); - } - ( - NestedMacroState::Macro, - &TokenTree::Token(Token { kind: TokenKind::Ident(..), .. }), - ) => { - state = NestedMacroState::MacroName; - } - (NestedMacroState::Macro, &TokenTree::MetaVar(..)) => { - state = NestedMacroState::MacroName; - // We check that the meta-variable is correctly used. - check_occurrences(sess, node_id, tt, macros, binders, ops, valid); - } - (NestedMacroState::MacroName, &TokenTree::Delimited(_, ref del)) - if del.delim == DelimToken::Paren => - { - state = NestedMacroState::MacroNameParen; - nested_binders = Binders::default(); - check_binders( - sess, - node_id, - tt, - &nested_macros, - &mut nested_binders, - &Stack::Empty, - valid, - ); - } - (NestedMacroState::MacroNameParen, &TokenTree::Delimited(_, ref del)) - if del.delim == DelimToken::Brace => - { - state = NestedMacroState::Empty; - check_occurrences( - sess, - node_id, - tt, - &nested_macros, - &nested_binders, - &Stack::Empty, - valid, - ); - } - (_, ref tt) => { - state = NestedMacroState::Empty; - check_occurrences(sess, node_id, tt, macros, binders, ops, valid); - } - } - } -} - -/// Checks the body of nested macro, returns where the check stopped, and sets `valid` to false in -/// case of errors. -/// -/// The token trees are checked as long as they look like a list of (LHS) => {RHS} token trees. This -/// check is a best-effort to detect a macro definition. It returns the position in `tts` where we -/// stopped checking because we detected we were not in a macro definition anymore. -/// -/// Arguments: -/// - `sess` is used to emit diagnostics and lints -/// - `node_id` is used to emit lints -/// - `legacy` specifies whether the macro is legacy -/// - `tts` is checked as a list of (LHS) => {RHS} -/// - `macros` is the stack of outer macros -/// - `valid` is set in case of errors -fn check_nested_macro( - sess: &ParseSess, - node_id: NodeId, - legacy: bool, - tts: &[TokenTree], - macros: &Stack<'_, MacroState<'_>>, - valid: &mut bool, -) -> usize { - let n = tts.len(); - let mut i = 0; - let separator = if legacy { TokenKind::Semi } else { TokenKind::Comma }; - loop { - // We expect 3 token trees: `(LHS) => {RHS}`. The separator is checked after. - if i + 2 >= n - || !tts[i].is_delimited() - || !tts[i + 1].is_token(&TokenKind::FatArrow) - || !tts[i + 2].is_delimited() - { - break; - } - let lhs = &tts[i]; - let rhs = &tts[i + 2]; - let mut binders = Binders::default(); - check_binders(sess, node_id, lhs, macros, &mut binders, &Stack::Empty, valid); - check_occurrences(sess, node_id, rhs, macros, &binders, &Stack::Empty, valid); - // Since the last semicolon is optional for legacy macros and decl_macro are not terminated, - // we increment our checked position by how many token trees we already checked (the 3 - // above) before checking for the separator. - i += 3; - if i == n || !tts[i].is_token(&separator) { - break; - } - // We increment our checked position for the semicolon. - i += 1; - } - i -} - -/// Checks that a meta-variable occurrence is valid. -/// -/// Arguments: -/// - `sess` is used to emit diagnostics and lints -/// - `node_id` is used to emit lints -/// - `macros` is the stack of possible outer macros -/// - `binders` contains the binders of the associated LHS -/// - `ops` is the stack of Kleene operators from the RHS -/// - `span` is the span of the meta-variable to check -/// - `name` is the name of the meta-variable to check -fn check_ops_is_prefix( - sess: &ParseSess, - node_id: NodeId, - macros: &Stack<'_, MacroState<'_>>, - binders: &Binders, - ops: &Stack<'_, KleeneToken>, - span: Span, - name: Ident, -) { - let macros = macros.push(MacroState { binders, ops: ops.into() }); - // Accumulates the stacks the operators of each state until (and including when) the - // meta-variable is found. The innermost stack is first. - let mut acc: SmallVec<[&SmallVec<[KleeneToken; 1]>; 1]> = SmallVec::new(); - for state in ¯os { - acc.push(&state.ops); - if let Some(binder) = state.binders.get(&name) { - // This variable concatenates the stack of operators from the RHS of the LHS where the - // meta-variable was defined to where it is used (in possibly nested macros). The - // outermost operator is first. - let mut occurrence_ops: SmallVec<[KleeneToken; 2]> = SmallVec::new(); - // We need to iterate from the end to start with outermost stack. - for ops in acc.iter().rev() { - occurrence_ops.extend_from_slice(ops); - } - ops_is_prefix(sess, node_id, span, name, &binder.ops, &occurrence_ops); - return; - } - } - buffer_lint(sess, span.into(), node_id, &format!("unknown macro variable `{}`", name)); -} - -/// Returns whether `binder_ops` is a prefix of `occurrence_ops`. -/// -/// The stack of Kleene operators of a meta-variable occurrence just needs to have the stack of -/// Kleene operators of its binder as a prefix. -/// -/// Consider $i in the following example: -/// -/// ( $( $i:ident = $($j:ident),+ );* ) => { $($( $i += $j; )+)* } -/// -/// It occurs under the Kleene stack ["*", "+"] and is bound under ["*"] only. -/// -/// Arguments: -/// - `sess` is used to emit diagnostics and lints -/// - `node_id` is used to emit lints -/// - `span` is the span of the meta-variable being check -/// - `name` is the name of the meta-variable being check -/// - `binder_ops` is the stack of Kleene operators for the binder -/// - `occurrence_ops` is the stack of Kleene operators for the occurrence -fn ops_is_prefix( - sess: &ParseSess, - node_id: NodeId, - span: Span, - name: Ident, - binder_ops: &[KleeneToken], - occurrence_ops: &[KleeneToken], -) { - for (i, binder) in binder_ops.iter().enumerate() { - if i >= occurrence_ops.len() { - let mut span = MultiSpan::from_span(span); - span.push_span_label(binder.span, "expected repetition".into()); - let message = &format!("variable '{}' is still repeating at this depth", name); - buffer_lint(sess, span, node_id, message); - return; - } - let occurrence = &occurrence_ops[i]; - if occurrence.op != binder.op { - let mut span = MultiSpan::from_span(span); - span.push_span_label(binder.span, "expected repetition".into()); - span.push_span_label(occurrence.span, "conflicting repetition".into()); - let message = "meta-variable repeats with different Kleene operator"; - buffer_lint(sess, span, node_id, message); - return; - } - } -} - -fn buffer_lint(sess: &ParseSess, span: MultiSpan, node_id: NodeId, message: &str) { - sess.buffer_lint(BufferedEarlyLintId::MetaVariableMisuse, span, node_id, message); -} diff --git a/src/libsyntax/ext/mbe/macro_parser.rs b/src/libsyntax/ext/mbe/macro_parser.rs deleted file mode 100644 index 0cb5eff1ef2..00000000000 --- a/src/libsyntax/ext/mbe/macro_parser.rs +++ /dev/null @@ -1,942 +0,0 @@ -//! This is an NFA-based parser, which calls out to the main rust parser for named non-terminals -//! (which it commits to fully when it hits one in a grammar). There's a set of current NFA threads -//! and a set of next ones. Instead of NTs, we have a special case for Kleene star. The big-O, in -//! pathological cases, is worse than traditional use of NFA or Earley parsing, but it's an easier -//! fit for Macro-by-Example-style rules. -//! -//! (In order to prevent the pathological case, we'd need to lazily construct the resulting -//! `NamedMatch`es at the very end. It'd be a pain, and require more memory to keep around old -//! items, but it would also save overhead) -//! -//! We don't say this parser uses the Earley algorithm, because it's unnecessarily inaccurate. -//! The macro parser restricts itself to the features of finite state automata. Earley parsers -//! can be described as an extension of NFAs with completion rules, prediction rules, and recursion. -//! -//! Quick intro to how the parser works: -//! -//! A 'position' is a dot in the middle of a matcher, usually represented as a -//! dot. For example `· a $( a )* a b` is a position, as is `a $( · a )* a b`. -//! -//! The parser walks through the input a character at a time, maintaining a list -//! of threads consistent with the current position in the input string: `cur_items`. -//! -//! As it processes them, it fills up `eof_items` with threads that would be valid if -//! the macro invocation is now over, `bb_items` with threads that are waiting on -//! a Rust non-terminal like `$e:expr`, and `next_items` with threads that are waiting -//! on a particular token. Most of the logic concerns moving the · through the -//! repetitions indicated by Kleene stars. The rules for moving the · without -//! consuming any input are called epsilon transitions. It only advances or calls -//! out to the real Rust parser when no `cur_items` threads remain. -//! -//! Example: -//! -//! ```text, ignore -//! Start parsing a a a a b against [· a $( a )* a b]. -//! -//! Remaining input: a a a a b -//! next: [· a $( a )* a b] -//! -//! - - - Advance over an a. - - - -//! -//! Remaining input: a a a b -//! cur: [a · $( a )* a b] -//! Descend/Skip (first item). -//! next: [a $( · a )* a b] [a $( a )* · a b]. -//! -//! - - - Advance over an a. - - - -//! -//! Remaining input: a a b -//! cur: [a $( a · )* a b] [a $( a )* a · b] -//! Follow epsilon transition: Finish/Repeat (first item) -//! next: [a $( a )* · a b] [a $( · a )* a b] [a $( a )* a · b] -//! -//! - - - Advance over an a. - - - (this looks exactly like the last step) -//! -//! Remaining input: a b -//! cur: [a $( a · )* a b] [a $( a )* a · b] -//! Follow epsilon transition: Finish/Repeat (first item) -//! next: [a $( a )* · a b] [a $( · a )* a b] [a $( a )* a · b] -//! -//! - - - Advance over an a. - - - (this looks exactly like the last step) -//! -//! Remaining input: b -//! cur: [a $( a · )* a b] [a $( a )* a · b] -//! Follow epsilon transition: Finish/Repeat (first item) -//! next: [a $( a )* · a b] [a $( · a )* a b] [a $( a )* a · b] -//! -//! - - - Advance over a b. - - - -//! -//! Remaining input: '' -//! eof: [a $( a )* a b ·] -//! ``` - -crate use NamedMatch::*; -crate use ParseResult::*; -use TokenTreeOrTokenTreeSlice::*; - -use crate::ast::{Ident, Name}; -use crate::ext::mbe::{self, TokenTree}; -use crate::parse::{Directory, ParseSess, PResult}; -use crate::parse::parser::{Parser, PathStyle}; -use crate::parse::token::{self, DocComment, Nonterminal, Token}; -use crate::print::pprust; -use crate::symbol::{kw, sym, Symbol}; -use crate::tokenstream::{DelimSpan, TokenStream}; - -use errors::FatalError; -use smallvec::{smallvec, SmallVec}; -use syntax_pos::Span; - -use rustc_data_structures::fx::FxHashMap; -use rustc_data_structures::sync::Lrc; -use std::collections::hash_map::Entry::{Occupied, Vacant}; -use std::mem; -use std::ops::{Deref, DerefMut}; - -// To avoid costly uniqueness checks, we require that `MatchSeq` always has a nonempty body. - -/// Either a sequence of token trees or a single one. This is used as the representation of the -/// sequence of tokens that make up a matcher. -#[derive(Clone)] -enum TokenTreeOrTokenTreeSlice<'tt> { - Tt(TokenTree), - TtSeq(&'tt [TokenTree]), -} - -impl<'tt> TokenTreeOrTokenTreeSlice<'tt> { - /// Returns the number of constituent top-level token trees of `self` (top-level in that it - /// will not recursively descend into subtrees). - fn len(&self) -> usize { - match *self { - TtSeq(ref v) => v.len(), - Tt(ref tt) => tt.len(), - } - } - - /// The `index`-th token tree of `self`. - fn get_tt(&self, index: usize) -> TokenTree { - match *self { - TtSeq(ref v) => v[index].clone(), - Tt(ref tt) => tt.get_tt(index), - } - } -} - -/// An unzipping of `TokenTree`s... see the `stack` field of `MatcherPos`. -/// -/// This is used by `inner_parse_loop` to keep track of delimited submatchers that we have -/// descended into. -#[derive(Clone)] -struct MatcherTtFrame<'tt> { - /// The "parent" matcher that we are descending into. - elts: TokenTreeOrTokenTreeSlice<'tt>, - /// The position of the "dot" in `elts` at the time we descended. - idx: usize, -} - -type NamedMatchVec = SmallVec<[NamedMatch; 4]>; - -/// Represents a single "position" (aka "matcher position", aka "item"), as -/// described in the module documentation. -/// -/// Here: -/// -/// - `'root` represents the lifetime of the stack slot that holds the root -/// `MatcherPos`. As described in `MatcherPosHandle`, the root `MatcherPos` -/// structure is stored on the stack, but subsequent instances are put into -/// the heap. -/// - `'tt` represents the lifetime of the token trees that this matcher -/// position refers to. -/// -/// It is important to distinguish these two lifetimes because we have a -/// `SmallVec<TokenTreeOrTokenTreeSlice<'tt>>` below, and the destructor of -/// that is considered to possibly access the data from its elements (it lacks -/// a `#[may_dangle]` attribute). As a result, the compiler needs to know that -/// all the elements in that `SmallVec` strictly outlive the root stack slot -/// lifetime. By separating `'tt` from `'root`, we can show that. -#[derive(Clone)] -struct MatcherPos<'root, 'tt> { - /// The token or sequence of tokens that make up the matcher - top_elts: TokenTreeOrTokenTreeSlice<'tt>, - - /// The position of the "dot" in this matcher - idx: usize, - - /// The first span of source that the beginning of this matcher corresponds to. In other - /// words, the token in the source whose span is `sp_open` is matched against the first token of - /// the matcher. - sp_open: Span, - - /// For each named metavar in the matcher, we keep track of token trees matched against the - /// metavar by the black box parser. In particular, there may be more than one match per - /// metavar if we are in a repetition (each repetition matches each of the variables). - /// Moreover, matchers and repetitions can be nested; the `matches` field is shared (hence the - /// `Rc`) among all "nested" matchers. `match_lo`, `match_cur`, and `match_hi` keep track of - /// the current position of the `self` matcher position in the shared `matches` list. - /// - /// Also, note that while we are descending into a sequence, matchers are given their own - /// `matches` vector. Only once we reach the end of a full repetition of the sequence do we add - /// all bound matches from the submatcher into the shared top-level `matches` vector. If `sep` - /// and `up` are `Some`, then `matches` is _not_ the shared top-level list. Instead, if one - /// wants the shared `matches`, one should use `up.matches`. - matches: Box<[Lrc<NamedMatchVec>]>, - /// The position in `matches` corresponding to the first metavar in this matcher's sequence of - /// token trees. In other words, the first metavar in the first token of `top_elts` corresponds - /// to `matches[match_lo]`. - match_lo: usize, - /// The position in `matches` corresponding to the metavar we are currently trying to match - /// against the source token stream. `match_lo <= match_cur <= match_hi`. - match_cur: usize, - /// Similar to `match_lo` except `match_hi` is the position in `matches` of the _last_ metavar - /// in this matcher. - match_hi: usize, - - // The following fields are used if we are matching a repetition. If we aren't, they should be - // `None`. - - /// The KleeneOp of this sequence if we are in a repetition. - seq_op: Option<mbe::KleeneOp>, - - /// The separator if we are in a repetition. - sep: Option<Token>, - - /// The "parent" matcher position if we are in a repetition. That is, the matcher position just - /// before we enter the sequence. - up: Option<MatcherPosHandle<'root, 'tt>>, - - /// Specifically used to "unzip" token trees. By "unzip", we mean to unwrap the delimiters from - /// a delimited token tree (e.g., something wrapped in `(` `)`) or to get the contents of a doc - /// comment... - /// - /// When matching against matchers with nested delimited submatchers (e.g., `pat ( pat ( .. ) - /// pat ) pat`), we need to keep track of the matchers we are descending into. This stack does - /// that where the bottom of the stack is the outermost matcher. - /// Also, throughout the comments, this "descent" is often referred to as "unzipping"... - stack: SmallVec<[MatcherTtFrame<'tt>; 1]>, -} - -impl<'root, 'tt> MatcherPos<'root, 'tt> { - /// Adds `m` as a named match for the `idx`-th metavar. - fn push_match(&mut self, idx: usize, m: NamedMatch) { - let matches = Lrc::make_mut(&mut self.matches[idx]); - matches.push(m); - } -} - -// Lots of MatcherPos instances are created at runtime. Allocating them on the -// heap is slow. Furthermore, using SmallVec<MatcherPos> to allocate them all -// on the stack is also slow, because MatcherPos is quite a large type and -// instances get moved around a lot between vectors, which requires lots of -// slow memcpy calls. -// -// Therefore, the initial MatcherPos is always allocated on the stack, -// subsequent ones (of which there aren't that many) are allocated on the heap, -// and this type is used to encapsulate both cases. -enum MatcherPosHandle<'root, 'tt> { - Ref(&'root mut MatcherPos<'root, 'tt>), - Box(Box<MatcherPos<'root, 'tt>>), -} - -impl<'root, 'tt> Clone for MatcherPosHandle<'root, 'tt> { - // This always produces a new Box. - fn clone(&self) -> Self { - MatcherPosHandle::Box(match *self { - MatcherPosHandle::Ref(ref r) => Box::new((**r).clone()), - MatcherPosHandle::Box(ref b) => b.clone(), - }) - } -} - -impl<'root, 'tt> Deref for MatcherPosHandle<'root, 'tt> { - type Target = MatcherPos<'root, 'tt>; - fn deref(&self) -> &Self::Target { - match *self { - MatcherPosHandle::Ref(ref r) => r, - MatcherPosHandle::Box(ref b) => b, - } - } -} - -impl<'root, 'tt> DerefMut for MatcherPosHandle<'root, 'tt> { - fn deref_mut(&mut self) -> &mut MatcherPos<'root, 'tt> { - match *self { - MatcherPosHandle::Ref(ref mut r) => r, - MatcherPosHandle::Box(ref mut b) => b, - } - } -} - -/// Represents the possible results of an attempted parse. -crate enum ParseResult<T> { - /// Parsed successfully. - Success(T), - /// Arm failed to match. If the second parameter is `token::Eof`, it indicates an unexpected - /// end of macro invocation. Otherwise, it indicates that no rules expected the given token. - Failure(Token, &'static str), - /// Fatal error (malformed macro?). Abort compilation. - Error(syntax_pos::Span, String), -} - -/// A `ParseResult` where the `Success` variant contains a mapping of `Ident`s to `NamedMatch`es. -/// This represents the mapping of metavars to the token trees they bind to. -crate type NamedParseResult = ParseResult<FxHashMap<Ident, NamedMatch>>; - -/// Count how many metavars are named in the given matcher `ms`. -pub(super) fn count_names(ms: &[TokenTree]) -> usize { - ms.iter().fold(0, |count, elt| { - count + match *elt { - TokenTree::Sequence(_, ref seq) => seq.num_captures, - TokenTree::Delimited(_, ref delim) => count_names(&delim.tts), - TokenTree::MetaVar(..) => 0, - TokenTree::MetaVarDecl(..) => 1, - TokenTree::Token(..) => 0, - } - }) -} - -/// `len` `Vec`s (initially shared and empty) that will store matches of metavars. -fn create_matches(len: usize) -> Box<[Lrc<NamedMatchVec>]> { - if len == 0 { - vec![] - } else { - let empty_matches = Lrc::new(SmallVec::new()); - vec![empty_matches; len] - }.into_boxed_slice() -} - -/// Generates the top-level matcher position in which the "dot" is before the first token of the -/// matcher `ms` and we are going to start matching at the span `open` in the source. -fn initial_matcher_pos<'root, 'tt>(ms: &'tt [TokenTree], open: Span) -> MatcherPos<'root, 'tt> { - let match_idx_hi = count_names(ms); - let matches = create_matches(match_idx_hi); - MatcherPos { - // Start with the top level matcher given to us - top_elts: TtSeq(ms), // "elts" is an abbr. for "elements" - // The "dot" is before the first token of the matcher - idx: 0, - // We start matching at the span `open` in the source code - sp_open: open, - - // Initialize `matches` to a bunch of empty `Vec`s -- one for each metavar in `top_elts`. - // `match_lo` for `top_elts` is 0 and `match_hi` is `matches.len()`. `match_cur` is 0 since - // we haven't actually matched anything yet. - matches, - match_lo: 0, - match_cur: 0, - match_hi: match_idx_hi, - - // Haven't descended into any delimiters, so empty stack - stack: smallvec![], - - // Haven't descended into any sequences, so both of these are `None`. - seq_op: None, - sep: None, - up: None, - } -} - -/// `NamedMatch` is a pattern-match result for a single `token::MATCH_NONTERMINAL`: -/// so it is associated with a single ident in a parse, and all -/// `MatchedNonterminal`s in the `NamedMatch` have the same non-terminal type -/// (expr, item, etc). Each leaf in a single `NamedMatch` corresponds to a -/// single `token::MATCH_NONTERMINAL` in the `TokenTree` that produced it. -/// -/// The in-memory structure of a particular `NamedMatch` represents the match -/// that occurred when a particular subset of a matcher was applied to a -/// particular token tree. -/// -/// The width of each `MatchedSeq` in the `NamedMatch`, and the identity of -/// the `MatchedNonterminal`s, will depend on the token tree it was applied -/// to: each `MatchedSeq` corresponds to a single `TTSeq` in the originating -/// token tree. The depth of the `NamedMatch` structure will therefore depend -/// only on the nesting depth of `ast::TTSeq`s in the originating -/// token tree it was derived from. -#[derive(Debug, Clone)] -crate enum NamedMatch { - MatchedSeq(Lrc<NamedMatchVec>, DelimSpan), - MatchedNonterminal(Lrc<Nonterminal>), -} - -/// Takes a sequence of token trees `ms` representing a matcher which successfully matched input -/// and an iterator of items that matched input and produces a `NamedParseResult`. -fn nameize<I: Iterator<Item = NamedMatch>>( - sess: &ParseSess, - ms: &[TokenTree], - mut res: I, -) -> NamedParseResult { - // Recursively descend into each type of matcher (e.g., sequences, delimited, metavars) and make - // sure that each metavar has _exactly one_ binding. If a metavar does not have exactly one - // binding, then there is an error. If it does, then we insert the binding into the - // `NamedParseResult`. - fn n_rec<I: Iterator<Item = NamedMatch>>( - sess: &ParseSess, - m: &TokenTree, - res: &mut I, - ret_val: &mut FxHashMap<Ident, NamedMatch>, - ) -> Result<(), (syntax_pos::Span, String)> { - match *m { - TokenTree::Sequence(_, ref seq) => for next_m in &seq.tts { - n_rec(sess, next_m, res.by_ref(), ret_val)? - }, - TokenTree::Delimited(_, ref delim) => for next_m in &delim.tts { - n_rec(sess, next_m, res.by_ref(), ret_val)?; - }, - TokenTree::MetaVarDecl(span, _, id) if id.name == kw::Invalid => { - if sess.missing_fragment_specifiers.borrow_mut().remove(&span) { - return Err((span, "missing fragment specifier".to_string())); - } - } - TokenTree::MetaVarDecl(sp, bind_name, _) => { - match ret_val.entry(bind_name) { - Vacant(spot) => { - spot.insert(res.next().unwrap()); - } - Occupied(..) => { - return Err((sp, format!("duplicated bind name: {}", bind_name))) - } - } - } - TokenTree::MetaVar(..) | TokenTree::Token(..) => (), - } - - Ok(()) - } - - let mut ret_val = FxHashMap::default(); - for m in ms { - match n_rec(sess, m, res.by_ref(), &mut ret_val) { - Ok(_) => {} - Err((sp, msg)) => return Error(sp, msg), - } - } - - Success(ret_val) -} - -/// Performs a token equality check, ignoring syntax context (that is, an unhygienic comparison) -fn token_name_eq(t1: &Token, t2: &Token) -> bool { - if let (Some((ident1, is_raw1)), Some((ident2, is_raw2))) = (t1.ident(), t2.ident()) { - ident1.name == ident2.name && is_raw1 == is_raw2 - } else if let (Some(ident1), Some(ident2)) = (t1.lifetime(), t2.lifetime()) { - ident1.name == ident2.name - } else { - t1.kind == t2.kind - } -} - -/// Process the matcher positions of `cur_items` until it is empty. In the process, this will -/// produce more items in `next_items`, `eof_items`, and `bb_items`. -/// -/// For more info about the how this happens, see the module-level doc comments and the inline -/// comments of this function. -/// -/// # Parameters -/// -/// - `sess`: the parsing session into which errors are emitted. -/// - `cur_items`: the set of current items to be processed. This should be empty by the end of a -/// successful execution of this function. -/// - `next_items`: the set of newly generated items. These are used to replenish `cur_items` in -/// the function `parse`. -/// - `eof_items`: the set of items that would be valid if this was the EOF. -/// - `bb_items`: the set of items that are waiting for the black-box parser. -/// - `token`: the current token of the parser. -/// - `span`: the `Span` in the source code corresponding to the token trees we are trying to match -/// against the matcher positions in `cur_items`. -/// -/// # Returns -/// -/// A `ParseResult`. Note that matches are kept track of through the items generated. -fn inner_parse_loop<'root, 'tt>( - sess: &ParseSess, - cur_items: &mut SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>, - next_items: &mut Vec<MatcherPosHandle<'root, 'tt>>, - eof_items: &mut SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>, - bb_items: &mut SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>, - token: &Token, -) -> ParseResult<()> { - // Pop items from `cur_items` until it is empty. - while let Some(mut item) = cur_items.pop() { - // When unzipped trees end, remove them. This corresponds to backtracking out of a - // delimited submatcher into which we already descended. In backtracking out again, we need - // to advance the "dot" past the delimiters in the outer matcher. - while item.idx >= item.top_elts.len() { - match item.stack.pop() { - Some(MatcherTtFrame { elts, idx }) => { - item.top_elts = elts; - item.idx = idx + 1; - } - None => break, - } - } - - // Get the current position of the "dot" (`idx`) in `item` and the number of token trees in - // the matcher (`len`). - let idx = item.idx; - let len = item.top_elts.len(); - - // If `idx >= len`, then we are at or past the end of the matcher of `item`. - if idx >= len { - // We are repeating iff there is a parent. If the matcher is inside of a repetition, - // then we could be at the end of a sequence or at the beginning of the next - // repetition. - if item.up.is_some() { - // At this point, regardless of whether there is a separator, we should add all - // matches from the complete repetition of the sequence to the shared, top-level - // `matches` list (actually, `up.matches`, which could itself not be the top-level, - // but anyway...). Moreover, we add another item to `cur_items` in which the "dot" - // is at the end of the `up` matcher. This ensures that the "dot" in the `up` - // matcher is also advanced sufficiently. - // - // NOTE: removing the condition `idx == len` allows trailing separators. - if idx == len { - // Get the `up` matcher - let mut new_pos = item.up.clone().unwrap(); - - // Add matches from this repetition to the `matches` of `up` - for idx in item.match_lo..item.match_hi { - let sub = item.matches[idx].clone(); - let span = DelimSpan::from_pair(item.sp_open, token.span); - new_pos.push_match(idx, MatchedSeq(sub, span)); - } - - // Move the "dot" past the repetition in `up` - new_pos.match_cur = item.match_hi; - new_pos.idx += 1; - cur_items.push(new_pos); - } - - // Check if we need a separator. - if idx == len && item.sep.is_some() { - // We have a separator, and it is the current token. We can advance past the - // separator token. - if item.sep - .as_ref() - .map(|sep| token_name_eq(token, sep)) - .unwrap_or(false) - { - item.idx += 1; - next_items.push(item); - } - } - // We don't need a separator. Move the "dot" back to the beginning of the matcher - // and try to match again UNLESS we are only allowed to have _one_ repetition. - else if item.seq_op != Some(mbe::KleeneOp::ZeroOrOne) { - item.match_cur = item.match_lo; - item.idx = 0; - cur_items.push(item); - } - } - // If we are not in a repetition, then being at the end of a matcher means that we have - // reached the potential end of the input. - else { - eof_items.push(item); - } - } - // We are in the middle of a matcher. - else { - // Look at what token in the matcher we are trying to match the current token (`token`) - // against. Depending on that, we may generate new items. - match item.top_elts.get_tt(idx) { - // Need to descend into a sequence - TokenTree::Sequence(sp, seq) => { - // Examine the case where there are 0 matches of this sequence. We are - // implicitly disallowing OneOrMore from having 0 matches here. Thus, that will - // result in a "no rules expected token" error by virtue of this matcher not - // working. - if seq.kleene.op == mbe::KleeneOp::ZeroOrMore - || seq.kleene.op == mbe::KleeneOp::ZeroOrOne - { - let mut new_item = item.clone(); - new_item.match_cur += seq.num_captures; - new_item.idx += 1; - for idx in item.match_cur..item.match_cur + seq.num_captures { - new_item.push_match(idx, MatchedSeq(Lrc::new(smallvec![]), sp)); - } - cur_items.push(new_item); - } - - let matches = create_matches(item.matches.len()); - cur_items.push(MatcherPosHandle::Box(Box::new(MatcherPos { - stack: smallvec![], - sep: seq.separator.clone(), - seq_op: Some(seq.kleene.op), - idx: 0, - matches, - match_lo: item.match_cur, - match_cur: item.match_cur, - match_hi: item.match_cur + seq.num_captures, - up: Some(item), - sp_open: sp.open, - top_elts: Tt(TokenTree::Sequence(sp, seq)), - }))); - } - - // We need to match a metavar (but the identifier is invalid)... this is an error - TokenTree::MetaVarDecl(span, _, id) if id.name == kw::Invalid => { - if sess.missing_fragment_specifiers.borrow_mut().remove(&span) { - return Error(span, "missing fragment specifier".to_string()); - } - } - - // We need to match a metavar with a valid ident... call out to the black-box - // parser by adding an item to `bb_items`. - TokenTree::MetaVarDecl(_, _, id) => { - // Built-in nonterminals never start with these tokens, - // so we can eliminate them from consideration. - if may_begin_with(token, id.name) { - bb_items.push(item); - } - } - - // We need to descend into a delimited submatcher or a doc comment. To do this, we - // push the current matcher onto a stack and push a new item containing the - // submatcher onto `cur_items`. - // - // At the beginning of the loop, if we reach the end of the delimited submatcher, - // we pop the stack to backtrack out of the descent. - seq @ TokenTree::Delimited(..) | - seq @ TokenTree::Token(Token { kind: DocComment(..), .. }) => { - let lower_elts = mem::replace(&mut item.top_elts, Tt(seq)); - let idx = item.idx; - item.stack.push(MatcherTtFrame { - elts: lower_elts, - idx, - }); - item.idx = 0; - cur_items.push(item); - } - - // We just matched a normal token. We can just advance the parser. - TokenTree::Token(t) if token_name_eq(&t, token) => { - item.idx += 1; - next_items.push(item); - } - - // There was another token that was not `token`... This means we can't add any - // rules. NOTE that this is not necessarily an error unless _all_ items in - // `cur_items` end up doing this. There may still be some other matchers that do - // end up working out. - TokenTree::Token(..) | TokenTree::MetaVar(..) => {} - } - } - } - - // Yay a successful parse (so far)! - Success(()) -} - -/// Use the given sequence of token trees (`ms`) as a matcher. Match the given token stream `tts` -/// against it and return the match. -/// -/// # Parameters -/// -/// - `sess`: The session into which errors are emitted -/// - `tts`: The tokenstream we are matching against the pattern `ms` -/// - `ms`: A sequence of token trees representing a pattern against which we are matching -/// - `directory`: Information about the file locations (needed for the black-box parser) -/// - `recurse_into_modules`: Whether or not to recurse into modules (needed for the black-box -/// parser) -pub(super) fn parse( - sess: &ParseSess, - tts: TokenStream, - ms: &[TokenTree], - directory: Option<Directory<'_>>, - recurse_into_modules: bool, -) -> NamedParseResult { - // Create a parser that can be used for the "black box" parts. - let mut parser = Parser::new( - sess, - tts, - directory, - recurse_into_modules, - true, - crate::MACRO_ARGUMENTS, - ); - - // A queue of possible matcher positions. We initialize it with the matcher position in which - // the "dot" is before the first token of the first token tree in `ms`. `inner_parse_loop` then - // processes all of these possible matcher positions and produces possible next positions into - // `next_items`. After some post-processing, the contents of `next_items` replenish `cur_items` - // and we start over again. - // - // This MatcherPos instance is allocated on the stack. All others -- and - // there are frequently *no* others! -- are allocated on the heap. - let mut initial = initial_matcher_pos(ms, parser.token.span); - let mut cur_items = smallvec![MatcherPosHandle::Ref(&mut initial)]; - let mut next_items = Vec::new(); - - loop { - // Matcher positions black-box parsed by parser.rs (`parser`) - let mut bb_items = SmallVec::new(); - - // Matcher positions that would be valid if the macro invocation was over now - let mut eof_items = SmallVec::new(); - assert!(next_items.is_empty()); - - // Process `cur_items` until either we have finished the input or we need to get some - // parsing from the black-box parser done. The result is that `next_items` will contain a - // bunch of possible next matcher positions in `next_items`. - match inner_parse_loop( - sess, - &mut cur_items, - &mut next_items, - &mut eof_items, - &mut bb_items, - &parser.token, - ) { - Success(_) => {} - Failure(token, msg) => return Failure(token, msg), - Error(sp, msg) => return Error(sp, msg), - } - - // inner parse loop handled all cur_items, so it's empty - assert!(cur_items.is_empty()); - - // We need to do some post processing after the `inner_parser_loop`. - // - // Error messages here could be improved with links to original rules. - - // If we reached the EOF, check that there is EXACTLY ONE possible matcher. Otherwise, - // either the parse is ambiguous (which should never happen) or there is a syntax error. - if parser.token == token::Eof { - if eof_items.len() == 1 { - let matches = eof_items[0] - .matches - .iter_mut() - .map(|dv| Lrc::make_mut(dv).pop().unwrap()); - return nameize(sess, ms, matches); - } else if eof_items.len() > 1 { - return Error( - parser.token.span, - "ambiguity: multiple successful parses".to_string(), - ); - } else { - return Failure( - Token::new(token::Eof, if parser.token.span.is_dummy() { - parser.token.span - } else { - sess.source_map().next_point(parser.token.span) - }), - "missing tokens in macro arguments", - ); - } - } - // Performance hack: eof_items may share matchers via Rc with other things that we want - // to modify. Dropping eof_items now may drop these refcounts to 1, preventing an - // unnecessary implicit clone later in Rc::make_mut. - drop(eof_items); - - // Another possibility is that we need to call out to parse some rust nonterminal - // (black-box) parser. However, if there is not EXACTLY ONE of these, something is wrong. - if (!bb_items.is_empty() && !next_items.is_empty()) || bb_items.len() > 1 { - let nts = bb_items - .iter() - .map(|item| match item.top_elts.get_tt(item.idx) { - TokenTree::MetaVarDecl(_, bind, name) => format!("{} ('{}')", name, bind), - _ => panic!(), - }) - .collect::<Vec<String>>() - .join(" or "); - - return Error( - parser.token.span, - format!( - "local ambiguity: multiple parsing options: {}", - match next_items.len() { - 0 => format!("built-in NTs {}.", nts), - 1 => format!("built-in NTs {} or 1 other option.", nts), - n => format!("built-in NTs {} or {} other options.", nts, n), - } - ), - ); - } - // If there are no possible next positions AND we aren't waiting for the black-box parser, - // then there is a syntax error. - else if bb_items.is_empty() && next_items.is_empty() { - return Failure( - parser.token.take(), - "no rules expected this token in macro call", - ); - } - // Dump all possible `next_items` into `cur_items` for the next iteration. - else if !next_items.is_empty() { - // Now process the next token - cur_items.extend(next_items.drain(..)); - parser.bump(); - } - // Finally, we have the case where we need to call the black-box parser to get some - // nonterminal. - else { - assert_eq!(bb_items.len(), 1); - - let mut item = bb_items.pop().unwrap(); - if let TokenTree::MetaVarDecl(span, _, ident) = item.top_elts.get_tt(item.idx) { - let match_cur = item.match_cur; - item.push_match( - match_cur, - MatchedNonterminal(Lrc::new(parse_nt(&mut parser, span, ident.name))), - ); - item.idx += 1; - item.match_cur += 1; - } else { - unreachable!() - } - cur_items.push(item); - } - - assert!(!cur_items.is_empty()); - } -} - -/// The token is an identifier, but not `_`. -/// We prohibit passing `_` to macros expecting `ident` for now. -fn get_macro_name(token: &Token) -> Option<(Name, bool)> { - match token.kind { - token::Ident(name, is_raw) if name != kw::Underscore => Some((name, is_raw)), - _ => None, - } -} - -/// Checks whether a non-terminal may begin with a particular token. -/// -/// Returning `false` is a *stability guarantee* that such a matcher will *never* begin with that -/// token. Be conservative (return true) if not sure. -fn may_begin_with(token: &Token, name: Name) -> bool { - /// Checks whether the non-terminal may contain a single (non-keyword) identifier. - fn may_be_ident(nt: &token::Nonterminal) -> bool { - match *nt { - token::NtItem(_) | token::NtBlock(_) | token::NtVis(_) => false, - _ => true, - } - } - - match name { - sym::expr => token.can_begin_expr() - // This exception is here for backwards compatibility. - && !token.is_keyword(kw::Let), - sym::ty => token.can_begin_type(), - sym::ident => get_macro_name(token).is_some(), - sym::literal => token.can_begin_literal_or_bool(), - sym::vis => match token.kind { - // The follow-set of :vis + "priv" keyword + interpolated - token::Comma | token::Ident(..) | token::Interpolated(_) => true, - _ => token.can_begin_type(), - }, - sym::block => match token.kind { - token::OpenDelim(token::Brace) => true, - token::Interpolated(ref nt) => match **nt { - token::NtItem(_) - | token::NtPat(_) - | token::NtTy(_) - | token::NtIdent(..) - | token::NtMeta(_) - | token::NtPath(_) - | token::NtVis(_) => false, // none of these may start with '{'. - _ => true, - }, - _ => false, - }, - sym::path | sym::meta => match token.kind { - token::ModSep | token::Ident(..) => true, - token::Interpolated(ref nt) => match **nt { - token::NtPath(_) | token::NtMeta(_) => true, - _ => may_be_ident(&nt), - }, - _ => false, - }, - sym::pat => match token.kind { - token::Ident(..) | // box, ref, mut, and other identifiers (can stricten) - token::OpenDelim(token::Paren) | // tuple pattern - token::OpenDelim(token::Bracket) | // slice pattern - token::BinOp(token::And) | // reference - token::BinOp(token::Minus) | // negative literal - token::AndAnd | // double reference - token::Literal(..) | // literal - token::DotDot | // range pattern (future compat) - token::DotDotDot | // range pattern (future compat) - token::ModSep | // path - token::Lt | // path (UFCS constant) - token::BinOp(token::Shl) => true, // path (double UFCS) - token::Interpolated(ref nt) => may_be_ident(nt), - _ => false, - }, - sym::lifetime => match token.kind { - token::Lifetime(_) => true, - token::Interpolated(ref nt) => match **nt { - token::NtLifetime(_) | token::NtTT(_) => true, - _ => false, - }, - _ => false, - }, - _ => match token.kind { - token::CloseDelim(_) => false, - _ => true, - }, - } -} - -/// A call to the "black-box" parser to parse some Rust non-terminal. -/// -/// # Parameters -/// -/// - `p`: the "black-box" parser to use -/// - `sp`: the `Span` we want to parse -/// - `name`: the name of the metavar _matcher_ we want to match (e.g., `tt`, `ident`, `block`, -/// etc...) -/// -/// # Returns -/// -/// The parsed non-terminal. -fn parse_nt(p: &mut Parser<'_>, sp: Span, name: Symbol) -> Nonterminal { - if name == sym::tt { - return token::NtTT(p.parse_token_tree()); - } - // check at the beginning and the parser checks after each bump - p.process_potential_macro_variable(); - match parse_nt_inner(p, sp, name) { - Ok(nt) => nt, - Err(mut err) => { - err.emit(); - FatalError.raise(); - } - } -} - -fn parse_nt_inner<'a>(p: &mut Parser<'a>, sp: Span, name: Symbol) -> PResult<'a, Nonterminal> { - Ok(match name { - sym::item => match p.parse_item()? { - Some(i) => token::NtItem(i), - None => return Err(p.fatal("expected an item keyword")), - }, - sym::block => token::NtBlock(p.parse_block()?), - sym::stmt => match p.parse_stmt()? { - Some(s) => token::NtStmt(s), - None => return Err(p.fatal("expected a statement")), - }, - sym::pat => token::NtPat(p.parse_pat(None)?), - sym::expr => token::NtExpr(p.parse_expr()?), - sym::literal => token::NtLiteral(p.parse_literal_maybe_minus()?), - sym::ty => token::NtTy(p.parse_ty()?), - // this could be handled like a token, since it is one - sym::ident => if let Some((name, is_raw)) = get_macro_name(&p.token) { - let span = p.token.span; - p.bump(); - token::NtIdent(Ident::new(name, span), is_raw) - } else { - let token_str = pprust::token_to_string(&p.token); - return Err(p.fatal(&format!("expected ident, found {}", &token_str))); - } - sym::path => token::NtPath(p.parse_path(PathStyle::Type)?), - sym::meta => token::NtMeta(p.parse_attr_item()?), - sym::vis => token::NtVis(p.parse_visibility(true)?), - sym::lifetime => if p.check_lifetime() { - token::NtLifetime(p.expect_lifetime().ident) - } else { - let token_str = pprust::token_to_string(&p.token); - return Err(p.fatal(&format!("expected a lifetime, found `{}`", &token_str))); - } - // this is not supposed to happen, since it has been checked - // when compiling the macro. - _ => p.span_bug(sp, "invalid fragment specifier"), - }) -} diff --git a/src/libsyntax/ext/mbe/macro_rules.rs b/src/libsyntax/ext/mbe/macro_rules.rs deleted file mode 100644 index aec4a683141..00000000000 --- a/src/libsyntax/ext/mbe/macro_rules.rs +++ /dev/null @@ -1,1191 +0,0 @@ -use crate::ast; -use crate::attr::{self, TransparencyError}; -use crate::edition::Edition; -use crate::ext::base::{DummyResult, ExtCtxt, MacResult, TTMacroExpander}; -use crate::ext::base::{SyntaxExtension, SyntaxExtensionKind}; -use crate::ext::expand::{AstFragment, AstFragmentKind}; -use crate::ext::mbe; -use crate::ext::mbe::macro_check; -use crate::ext::mbe::macro_parser::parse; -use crate::ext::mbe::macro_parser::{Error, Failure, Success}; -use crate::ext::mbe::macro_parser::{MatchedNonterminal, MatchedSeq, NamedParseResult}; -use crate::ext::mbe::transcribe::transcribe; -use crate::feature_gate::Features; -use crate::parse::parser::Parser; -use crate::parse::token::TokenKind::*; -use crate::parse::token::{self, NtTT, Token}; -use crate::parse::{Directory, ParseSess}; -use crate::print::pprust; -use crate::symbol::{kw, sym, Symbol}; -use crate::tokenstream::{DelimSpan, TokenStream, TokenTree}; - -use errors::{DiagnosticBuilder, FatalError}; -use log::debug; -use syntax_pos::hygiene::Transparency; -use syntax_pos::Span; - -use rustc_data_structures::fx::FxHashMap; -use std::borrow::Cow; -use std::collections::hash_map::Entry; -use std::slice; - -use errors::Applicability; -use rustc_data_structures::sync::Lrc; - -const VALID_FRAGMENT_NAMES_MSG: &str = "valid fragment specifiers are \ - `ident`, `block`, `stmt`, `expr`, `pat`, `ty`, `lifetime`, \ - `literal`, `path`, `meta`, `tt`, `item` and `vis`"; - -crate struct ParserAnyMacro<'a> { - parser: Parser<'a>, - - /// Span of the expansion site of the macro this parser is for - site_span: Span, - /// The ident of the macro we're parsing - macro_ident: ast::Ident, - arm_span: Span, -} - -crate fn annotate_err_with_kind( - err: &mut DiagnosticBuilder<'_>, - kind: AstFragmentKind, - span: Span, -) { - match kind { - AstFragmentKind::Ty => { - err.span_label(span, "this macro call doesn't expand to a type"); - } - AstFragmentKind::Pat => { - err.span_label(span, "this macro call doesn't expand to a pattern"); - } - _ => {} - }; -} - -impl<'a> ParserAnyMacro<'a> { - crate fn make(mut self: Box<ParserAnyMacro<'a>>, kind: AstFragmentKind) -> AstFragment { - let ParserAnyMacro { site_span, macro_ident, ref mut parser, arm_span } = *self; - let fragment = panictry!(parser.parse_ast_fragment(kind, true).map_err(|mut e| { - if parser.token == token::Eof && e.message().ends_with(", found `<eof>`") { - if !e.span.is_dummy() { - // early end of macro arm (#52866) - e.replace_span_with(parser.sess.source_map().next_point(parser.token.span)); - } - let msg = &e.message[0]; - e.message[0] = ( - format!( - "macro expansion ends with an incomplete expression: {}", - msg.0.replace(", found `<eof>`", ""), - ), - msg.1, - ); - } - if e.span.is_dummy() { - // Get around lack of span in error (#30128) - e.replace_span_with(site_span); - if parser.sess.source_map().span_to_filename(arm_span).is_real() { - e.span_label(arm_span, "in this macro arm"); - } - } else if !parser.sess.source_map().span_to_filename(parser.token.span).is_real() { - e.span_label(site_span, "in this macro invocation"); - } - match kind { - AstFragmentKind::Pat if macro_ident.name == sym::vec => { - let mut suggestion = None; - if let Ok(code) = parser.sess.source_map().span_to_snippet(site_span) { - if let Some(bang) = code.find('!') { - suggestion = Some(code[bang + 1..].to_string()); - } - } - if let Some(suggestion) = suggestion { - e.span_suggestion( - site_span, - "use a slice pattern here instead", - suggestion, - Applicability::MachineApplicable, - ); - } else { - e.span_label( - site_span, - "use a slice pattern here instead", - ); - } - e.help("for more information, see https://doc.rust-lang.org/edition-guide/\ - rust-2018/slice-patterns.html"); - } - _ => annotate_err_with_kind(&mut e, kind, site_span), - }; - e - })); - - // We allow semicolons at the end of expressions -- e.g., the semicolon in - // `macro_rules! m { () => { panic!(); } }` isn't parsed by `.parse_expr()`, - // but `m!()` is allowed in expression positions (cf. issue #34706). - if kind == AstFragmentKind::Expr && parser.token == token::Semi { - parser.bump(); - } - - // Make sure we don't have any tokens left to parse so we don't silently drop anything. - let path = ast::Path::from_ident(macro_ident.with_span_pos(site_span)); - parser.ensure_complete_parse(&path, kind.name(), site_span); - fragment - } -} - -struct MacroRulesMacroExpander { - name: ast::Ident, - span: Span, - transparency: Transparency, - lhses: Vec<mbe::TokenTree>, - rhses: Vec<mbe::TokenTree>, - valid: bool, -} - -impl TTMacroExpander for MacroRulesMacroExpander { - fn expand<'cx>( - &self, - cx: &'cx mut ExtCtxt<'_>, - sp: Span, - input: TokenStream, - ) -> Box<dyn MacResult + 'cx> { - if !self.valid { - return DummyResult::any(sp); - } - generic_extension( - cx, sp, self.span, self.name, self.transparency, input, &self.lhses, &self.rhses - ) - } -} - -fn trace_macros_note(cx: &mut ExtCtxt<'_>, sp: Span, message: String) { - let sp = sp.macro_backtrace().last().map(|trace| trace.call_site).unwrap_or(sp); - cx.expansions.entry(sp).or_default().push(message); -} - -/// Given `lhses` and `rhses`, this is the new macro we create -fn generic_extension<'cx>( - cx: &'cx mut ExtCtxt<'_>, - sp: Span, - def_span: Span, - name: ast::Ident, - transparency: Transparency, - arg: TokenStream, - lhses: &[mbe::TokenTree], - rhses: &[mbe::TokenTree], -) -> Box<dyn MacResult + 'cx> { - if cx.trace_macros() { - trace_macros_note(cx, sp, format!("expanding `{}! {{ {} }}`", name, arg)); - } - - // Which arm's failure should we report? (the one furthest along) - let mut best_failure: Option<(Token, &str)> = None; - - for (i, lhs) in lhses.iter().enumerate() { - // try each arm's matchers - let lhs_tt = match *lhs { - mbe::TokenTree::Delimited(_, ref delim) => &delim.tts[..], - _ => cx.span_bug(sp, "malformed macro lhs"), - }; - - match TokenTree::parse(cx, lhs_tt, arg.clone()) { - Success(named_matches) => { - let rhs = match rhses[i] { - // ignore delimiters - mbe::TokenTree::Delimited(_, ref delimed) => delimed.tts.clone(), - _ => cx.span_bug(sp, "malformed macro rhs"), - }; - let arm_span = rhses[i].span(); - - let rhs_spans = rhs.iter().map(|t| t.span()).collect::<Vec<_>>(); - // rhs has holes ( `$id` and `$(...)` that need filled) - let mut tts = transcribe(cx, &named_matches, rhs, transparency); - - // Replace all the tokens for the corresponding positions in the macro, to maintain - // proper positions in error reporting, while maintaining the macro_backtrace. - if rhs_spans.len() == tts.len() { - tts = tts.map_enumerated(|i, mut tt| { - let mut sp = rhs_spans[i]; - sp = sp.with_ctxt(tt.span().ctxt()); - tt.set_span(sp); - tt - }); - } - - if cx.trace_macros() { - trace_macros_note(cx, sp, format!("to `{}`", tts)); - } - - let directory = Directory { - path: Cow::from(cx.current_expansion.module.directory.as_path()), - ownership: cx.current_expansion.directory_ownership, - }; - let mut p = Parser::new(cx.parse_sess(), tts, Some(directory), true, false, None); - p.root_module_name = - cx.current_expansion.module.mod_path.last().map(|id| id.as_str().to_string()); - p.last_type_ascription = cx.current_expansion.prior_type_ascription; - - p.process_potential_macro_variable(); - // Let the context choose how to interpret the result. - // Weird, but useful for X-macros. - return Box::new(ParserAnyMacro { - parser: p, - - // Pass along the original expansion site and the name of the macro - // so we can print a useful error message if the parse of the expanded - // macro leaves unparsed tokens. - site_span: sp, - macro_ident: name, - arm_span, - }); - } - Failure(token, msg) => match best_failure { - Some((ref best_token, _)) if best_token.span.lo() >= token.span.lo() => {} - _ => best_failure = Some((token, msg)), - }, - Error(err_sp, ref msg) => cx.span_fatal(err_sp.substitute_dummy(sp), &msg[..]), - } - } - - let (token, label) = best_failure.expect("ran no matchers"); - let span = token.span.substitute_dummy(sp); - let mut err = cx.struct_span_err(span, &parse_failure_msg(&token)); - err.span_label(span, label); - if !def_span.is_dummy() && cx.source_map().span_to_filename(def_span).is_real() { - err.span_label(cx.source_map().def_span(def_span), "when calling this macro"); - } - - // Check whether there's a missing comma in this macro call, like `println!("{}" a);` - if let Some((arg, comma_span)) = arg.add_comma() { - for lhs in lhses { - // try each arm's matchers - let lhs_tt = match *lhs { - mbe::TokenTree::Delimited(_, ref delim) => &delim.tts[..], - _ => continue, - }; - match TokenTree::parse(cx, lhs_tt, arg.clone()) { - Success(_) => { - if comma_span.is_dummy() { - err.note("you might be missing a comma"); - } else { - err.span_suggestion_short( - comma_span, - "missing comma here", - ", ".to_string(), - Applicability::MachineApplicable, - ); - } - } - _ => {} - } - } - } - err.emit(); - cx.trace_macros_diag(); - DummyResult::any(sp) -} - -// Note that macro-by-example's input is also matched against a token tree: -// $( $lhs:tt => $rhs:tt );+ -// -// Holy self-referential! - -/// Converts a macro item into a syntax extension. -pub fn compile_declarative_macro( - sess: &ParseSess, - features: &Features, - def: &ast::Item, - edition: Edition, -) -> SyntaxExtension { - let diag = &sess.span_diagnostic; - let lhs_nm = ast::Ident::new(sym::lhs, def.span); - let rhs_nm = ast::Ident::new(sym::rhs, def.span); - let tt_spec = ast::Ident::new(sym::tt, def.span); - - // Parse the macro_rules! invocation - let body = match def.kind { - ast::ItemKind::MacroDef(ref body) => body, - _ => unreachable!(), - }; - - // The pattern that macro_rules matches. - // The grammar for macro_rules! is: - // $( $lhs:tt => $rhs:tt );+ - // ...quasiquoting this would be nice. - // These spans won't matter, anyways - let argument_gram = vec![ - mbe::TokenTree::Sequence( - DelimSpan::dummy(), - Lrc::new(mbe::SequenceRepetition { - tts: vec![ - mbe::TokenTree::MetaVarDecl(def.span, lhs_nm, tt_spec), - mbe::TokenTree::token(token::FatArrow, def.span), - mbe::TokenTree::MetaVarDecl(def.span, rhs_nm, tt_spec), - ], - separator: Some(Token::new( - if body.legacy { token::Semi } else { token::Comma }, - def.span, - )), - kleene: mbe::KleeneToken::new(mbe::KleeneOp::OneOrMore, def.span), - num_captures: 2, - }), - ), - // to phase into semicolon-termination instead of semicolon-separation - mbe::TokenTree::Sequence( - DelimSpan::dummy(), - Lrc::new(mbe::SequenceRepetition { - tts: vec![mbe::TokenTree::token( - if body.legacy { token::Semi } else { token::Comma }, - def.span, - )], - separator: None, - kleene: mbe::KleeneToken::new(mbe::KleeneOp::ZeroOrMore, def.span), - num_captures: 0, - }), - ), - ]; - - let argument_map = match parse(sess, body.stream(), &argument_gram, None, true) { - Success(m) => m, - Failure(token, msg) => { - let s = parse_failure_msg(&token); - let sp = token.span.substitute_dummy(def.span); - let mut err = sess.span_diagnostic.struct_span_fatal(sp, &s); - err.span_label(sp, msg); - err.emit(); - FatalError.raise(); - } - Error(sp, s) => { - sess.span_diagnostic.span_fatal(sp.substitute_dummy(def.span), &s).raise(); - } - }; - - let mut valid = true; - - // Extract the arguments: - let lhses = match argument_map[&lhs_nm] { - MatchedSeq(ref s, _) => s - .iter() - .map(|m| { - if let MatchedNonterminal(ref nt) = *m { - if let NtTT(ref tt) = **nt { - let tt = mbe::quoted::parse( - tt.clone().into(), - true, - sess, - ) - .pop() - .unwrap(); - valid &= check_lhs_nt_follows(sess, features, &def.attrs, &tt); - return tt; - } - } - sess.span_diagnostic.span_bug(def.span, "wrong-structured lhs") - }) - .collect::<Vec<mbe::TokenTree>>(), - _ => sess.span_diagnostic.span_bug(def.span, "wrong-structured lhs"), - }; - - let rhses = match argument_map[&rhs_nm] { - MatchedSeq(ref s, _) => s - .iter() - .map(|m| { - if let MatchedNonterminal(ref nt) = *m { - if let NtTT(ref tt) = **nt { - return mbe::quoted::parse( - tt.clone().into(), - false, - sess, - ) - .pop() - .unwrap(); - } - } - sess.span_diagnostic.span_bug(def.span, "wrong-structured lhs") - }) - .collect::<Vec<mbe::TokenTree>>(), - _ => sess.span_diagnostic.span_bug(def.span, "wrong-structured rhs"), - }; - - for rhs in &rhses { - valid &= check_rhs(sess, rhs); - } - - // don't abort iteration early, so that errors for multiple lhses can be reported - for lhs in &lhses { - valid &= check_lhs_no_empty_seq(sess, slice::from_ref(lhs)); - } - - // We use CRATE_NODE_ID instead of `def.id` otherwise we may emit buffered lints for a node id - // that is not lint-checked and trigger the "failed to process buffered lint here" bug. - valid &= macro_check::check_meta_variables(sess, ast::CRATE_NODE_ID, def.span, &lhses, &rhses); - - let (transparency, transparency_error) = attr::find_transparency(&def.attrs, body.legacy); - match transparency_error { - Some(TransparencyError::UnknownTransparency(value, span)) => - diag.span_err(span, &format!("unknown macro transparency: `{}`", value)), - Some(TransparencyError::MultipleTransparencyAttrs(old_span, new_span)) => - diag.span_err(vec![old_span, new_span], "multiple macro transparency attributes"), - None => {} - } - - let expander: Box<_> = Box::new(MacroRulesMacroExpander { - name: def.ident, span: def.span, transparency, lhses, rhses, valid - }); - - SyntaxExtension::new( - sess, - SyntaxExtensionKind::LegacyBang(expander), - def.span, - Vec::new(), - edition, - def.ident.name, - &def.attrs, - ) -} - -fn check_lhs_nt_follows( - sess: &ParseSess, - features: &Features, - attrs: &[ast::Attribute], - lhs: &mbe::TokenTree, -) -> bool { - // lhs is going to be like TokenTree::Delimited(...), where the - // entire lhs is those tts. Or, it can be a "bare sequence", not wrapped in parens. - if let mbe::TokenTree::Delimited(_, ref tts) = *lhs { - check_matcher(sess, features, attrs, &tts.tts) - } else { - let msg = "invalid macro matcher; matchers must be contained in balanced delimiters"; - sess.span_diagnostic.span_err(lhs.span(), msg); - false - } - // we don't abort on errors on rejection, the driver will do that for us - // after parsing/expansion. we can report every error in every macro this way. -} - -/// Checks that the lhs contains no repetition which could match an empty token -/// tree, because then the matcher would hang indefinitely. -fn check_lhs_no_empty_seq(sess: &ParseSess, tts: &[mbe::TokenTree]) -> bool { - use mbe::TokenTree; - for tt in tts { - match *tt { - TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => (), - TokenTree::Delimited(_, ref del) => { - if !check_lhs_no_empty_seq(sess, &del.tts) { - return false; - } - } - TokenTree::Sequence(span, ref seq) => { - if seq.separator.is_none() - && seq.tts.iter().all(|seq_tt| match *seq_tt { - TokenTree::MetaVarDecl(_, _, id) => id.name == sym::vis, - TokenTree::Sequence(_, ref sub_seq) => { - sub_seq.kleene.op == mbe::KleeneOp::ZeroOrMore - || sub_seq.kleene.op == mbe::KleeneOp::ZeroOrOne - } - _ => false, - }) - { - let sp = span.entire(); - sess.span_diagnostic.span_err(sp, "repetition matches empty token tree"); - return false; - } - if !check_lhs_no_empty_seq(sess, &seq.tts) { - return false; - } - } - } - } - - true -} - -fn check_rhs(sess: &ParseSess, rhs: &mbe::TokenTree) -> bool { - match *rhs { - mbe::TokenTree::Delimited(..) => return true, - _ => sess.span_diagnostic.span_err(rhs.span(), "macro rhs must be delimited"), - } - false -} - -fn check_matcher( - sess: &ParseSess, - features: &Features, - attrs: &[ast::Attribute], - matcher: &[mbe::TokenTree], -) -> bool { - let first_sets = FirstSets::new(matcher); - let empty_suffix = TokenSet::empty(); - let err = sess.span_diagnostic.err_count(); - check_matcher_core(sess, features, attrs, &first_sets, matcher, &empty_suffix); - err == sess.span_diagnostic.err_count() -} - -// `The FirstSets` for a matcher is a mapping from subsequences in the -// matcher to the FIRST set for that subsequence. -// -// This mapping is partially precomputed via a backwards scan over the -// token trees of the matcher, which provides a mapping from each -// repetition sequence to its *first* set. -// -// (Hypothetically, sequences should be uniquely identifiable via their -// spans, though perhaps that is false, e.g., for macro-generated macros -// that do not try to inject artificial span information. My plan is -// to try to catch such cases ahead of time and not include them in -// the precomputed mapping.) -struct FirstSets { - // this maps each TokenTree::Sequence `$(tt ...) SEP OP` that is uniquely identified by its - // span in the original matcher to the First set for the inner sequence `tt ...`. - // - // If two sequences have the same span in a matcher, then map that - // span to None (invalidating the mapping here and forcing the code to - // use a slow path). - first: FxHashMap<Span, Option<TokenSet>>, -} - -impl FirstSets { - fn new(tts: &[mbe::TokenTree]) -> FirstSets { - use mbe::TokenTree; - - let mut sets = FirstSets { first: FxHashMap::default() }; - build_recur(&mut sets, tts); - return sets; - - // walks backward over `tts`, returning the FIRST for `tts` - // and updating `sets` at the same time for all sequence - // substructure we find within `tts`. - fn build_recur(sets: &mut FirstSets, tts: &[TokenTree]) -> TokenSet { - let mut first = TokenSet::empty(); - for tt in tts.iter().rev() { - match *tt { - TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => { - first.replace_with(tt.clone()); - } - TokenTree::Delimited(span, ref delimited) => { - build_recur(sets, &delimited.tts[..]); - first.replace_with(delimited.open_tt(span.open)); - } - TokenTree::Sequence(sp, ref seq_rep) => { - let subfirst = build_recur(sets, &seq_rep.tts[..]); - - match sets.first.entry(sp.entire()) { - Entry::Vacant(vac) => { - vac.insert(Some(subfirst.clone())); - } - Entry::Occupied(mut occ) => { - // if there is already an entry, then a span must have collided. - // This should not happen with typical macro_rules macros, - // but syntax extensions need not maintain distinct spans, - // so distinct syntax trees can be assigned the same span. - // In such a case, the map cannot be trusted; so mark this - // entry as unusable. - occ.insert(None); - } - } - - // If the sequence contents can be empty, then the first - // token could be the separator token itself. - - if let (Some(sep), true) = (&seq_rep.separator, subfirst.maybe_empty) { - first.add_one_maybe(TokenTree::Token(sep.clone())); - } - - // Reverse scan: Sequence comes before `first`. - if subfirst.maybe_empty - || seq_rep.kleene.op == mbe::KleeneOp::ZeroOrMore - || seq_rep.kleene.op == mbe::KleeneOp::ZeroOrOne - { - // If sequence is potentially empty, then - // union them (preserving first emptiness). - first.add_all(&TokenSet { maybe_empty: true, ..subfirst }); - } else { - // Otherwise, sequence guaranteed - // non-empty; replace first. - first = subfirst; - } - } - } - } - - first - } - } - - // walks forward over `tts` until all potential FIRST tokens are - // identified. - fn first(&self, tts: &[mbe::TokenTree]) -> TokenSet { - use mbe::TokenTree; - - let mut first = TokenSet::empty(); - for tt in tts.iter() { - assert!(first.maybe_empty); - match *tt { - TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => { - first.add_one(tt.clone()); - return first; - } - TokenTree::Delimited(span, ref delimited) => { - first.add_one(delimited.open_tt(span.open)); - return first; - } - TokenTree::Sequence(sp, ref seq_rep) => { - let subfirst_owned; - let subfirst = match self.first.get(&sp.entire()) { - Some(&Some(ref subfirst)) => subfirst, - Some(&None) => { - subfirst_owned = self.first(&seq_rep.tts[..]); - &subfirst_owned - } - None => { - panic!("We missed a sequence during FirstSets construction"); - } - }; - - // If the sequence contents can be empty, then the first - // token could be the separator token itself. - if let (Some(sep), true) = (&seq_rep.separator, subfirst.maybe_empty) { - first.add_one_maybe(TokenTree::Token(sep.clone())); - } - - assert!(first.maybe_empty); - first.add_all(subfirst); - if subfirst.maybe_empty - || seq_rep.kleene.op == mbe::KleeneOp::ZeroOrMore - || seq_rep.kleene.op == mbe::KleeneOp::ZeroOrOne - { - // Continue scanning for more first - // tokens, but also make sure we - // restore empty-tracking state. - first.maybe_empty = true; - continue; - } else { - return first; - } - } - } - } - - // we only exit the loop if `tts` was empty or if every - // element of `tts` matches the empty sequence. - assert!(first.maybe_empty); - first - } -} - -// A set of `mbe::TokenTree`s, which may include `TokenTree::Match`s -// (for macro-by-example syntactic variables). It also carries the -// `maybe_empty` flag; that is true if and only if the matcher can -// match an empty token sequence. -// -// The First set is computed on submatchers like `$($a:expr b),* $(c)* d`, -// which has corresponding FIRST = {$a:expr, c, d}. -// Likewise, `$($a:expr b),* $(c)+ d` has FIRST = {$a:expr, c}. -// -// (Notably, we must allow for *-op to occur zero times.) -#[derive(Clone, Debug)] -struct TokenSet { - tokens: Vec<mbe::TokenTree>, - maybe_empty: bool, -} - -impl TokenSet { - // Returns a set for the empty sequence. - fn empty() -> Self { - TokenSet { tokens: Vec::new(), maybe_empty: true } - } - - // Returns the set `{ tok }` for the single-token (and thus - // non-empty) sequence [tok]. - fn singleton(tok: mbe::TokenTree) -> Self { - TokenSet { tokens: vec![tok], maybe_empty: false } - } - - // Changes self to be the set `{ tok }`. - // Since `tok` is always present, marks self as non-empty. - fn replace_with(&mut self, tok: mbe::TokenTree) { - self.tokens.clear(); - self.tokens.push(tok); - self.maybe_empty = false; - } - - // Changes self to be the empty set `{}`; meant for use when - // the particular token does not matter, but we want to - // record that it occurs. - fn replace_with_irrelevant(&mut self) { - self.tokens.clear(); - self.maybe_empty = false; - } - - // Adds `tok` to the set for `self`, marking sequence as non-empy. - fn add_one(&mut self, tok: mbe::TokenTree) { - if !self.tokens.contains(&tok) { - self.tokens.push(tok); - } - self.maybe_empty = false; - } - - // Adds `tok` to the set for `self`. (Leaves `maybe_empty` flag alone.) - fn add_one_maybe(&mut self, tok: mbe::TokenTree) { - if !self.tokens.contains(&tok) { - self.tokens.push(tok); - } - } - - // Adds all elements of `other` to this. - // - // (Since this is a set, we filter out duplicates.) - // - // If `other` is potentially empty, then preserves the previous - // setting of the empty flag of `self`. If `other` is guaranteed - // non-empty, then `self` is marked non-empty. - fn add_all(&mut self, other: &Self) { - for tok in &other.tokens { - if !self.tokens.contains(tok) { - self.tokens.push(tok.clone()); - } - } - if !other.maybe_empty { - self.maybe_empty = false; - } - } -} - -// Checks that `matcher` is internally consistent and that it -// can legally be followed by a token `N`, for all `N` in `follow`. -// (If `follow` is empty, then it imposes no constraint on -// the `matcher`.) -// -// Returns the set of NT tokens that could possibly come last in -// `matcher`. (If `matcher` matches the empty sequence, then -// `maybe_empty` will be set to true.) -// -// Requires that `first_sets` is pre-computed for `matcher`; -// see `FirstSets::new`. -fn check_matcher_core( - sess: &ParseSess, - features: &Features, - attrs: &[ast::Attribute], - first_sets: &FirstSets, - matcher: &[mbe::TokenTree], - follow: &TokenSet, -) -> TokenSet { - use mbe::TokenTree; - - let mut last = TokenSet::empty(); - - // 2. For each token and suffix [T, SUFFIX] in M: - // ensure that T can be followed by SUFFIX, and if SUFFIX may be empty, - // then ensure T can also be followed by any element of FOLLOW. - 'each_token: for i in 0..matcher.len() { - let token = &matcher[i]; - let suffix = &matcher[i + 1..]; - - let build_suffix_first = || { - let mut s = first_sets.first(suffix); - if s.maybe_empty { - s.add_all(follow); - } - s - }; - - // (we build `suffix_first` on demand below; you can tell - // which cases are supposed to fall through by looking for the - // initialization of this variable.) - let suffix_first; - - // First, update `last` so that it corresponds to the set - // of NT tokens that might end the sequence `... token`. - match *token { - TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => { - let can_be_followed_by_any; - if let Err(bad_frag) = has_legal_fragment_specifier(sess, features, attrs, token) { - let msg = format!("invalid fragment specifier `{}`", bad_frag); - sess.span_diagnostic - .struct_span_err(token.span(), &msg) - .help(VALID_FRAGMENT_NAMES_MSG) - .emit(); - // (This eliminates false positives and duplicates - // from error messages.) - can_be_followed_by_any = true; - } else { - can_be_followed_by_any = token_can_be_followed_by_any(token); - } - - if can_be_followed_by_any { - // don't need to track tokens that work with any, - last.replace_with_irrelevant(); - // ... and don't need to check tokens that can be - // followed by anything against SUFFIX. - continue 'each_token; - } else { - last.replace_with(token.clone()); - suffix_first = build_suffix_first(); - } - } - TokenTree::Delimited(span, ref d) => { - let my_suffix = TokenSet::singleton(d.close_tt(span.close)); - check_matcher_core(sess, features, attrs, first_sets, &d.tts, &my_suffix); - // don't track non NT tokens - last.replace_with_irrelevant(); - - // also, we don't need to check delimited sequences - // against SUFFIX - continue 'each_token; - } - TokenTree::Sequence(_, ref seq_rep) => { - suffix_first = build_suffix_first(); - // The trick here: when we check the interior, we want - // to include the separator (if any) as a potential - // (but not guaranteed) element of FOLLOW. So in that - // case, we make a temp copy of suffix and stuff - // delimiter in there. - // - // FIXME: Should I first scan suffix_first to see if - // delimiter is already in it before I go through the - // work of cloning it? But then again, this way I may - // get a "tighter" span? - let mut new; - let my_suffix = if let Some(sep) = &seq_rep.separator { - new = suffix_first.clone(); - new.add_one_maybe(TokenTree::Token(sep.clone())); - &new - } else { - &suffix_first - }; - - // At this point, `suffix_first` is built, and - // `my_suffix` is some TokenSet that we can use - // for checking the interior of `seq_rep`. - let next = - check_matcher_core(sess, features, attrs, first_sets, &seq_rep.tts, my_suffix); - if next.maybe_empty { - last.add_all(&next); - } else { - last = next; - } - - // the recursive call to check_matcher_core already ran the 'each_last - // check below, so we can just keep going forward here. - continue 'each_token; - } - } - - // (`suffix_first` guaranteed initialized once reaching here.) - - // Now `last` holds the complete set of NT tokens that could - // end the sequence before SUFFIX. Check that every one works with `suffix`. - 'each_last: for token in &last.tokens { - if let TokenTree::MetaVarDecl(_, name, frag_spec) = *token { - for next_token in &suffix_first.tokens { - match is_in_follow(next_token, frag_spec.name) { - IsInFollow::Invalid(msg, help) => { - sess.span_diagnostic - .struct_span_err(next_token.span(), &msg) - .help(help) - .emit(); - // don't bother reporting every source of - // conflict for a particular element of `last`. - continue 'each_last; - } - IsInFollow::Yes => {} - IsInFollow::No(possible) => { - let may_be = if last.tokens.len() == 1 && suffix_first.tokens.len() == 1 - { - "is" - } else { - "may be" - }; - - let sp = next_token.span(); - let mut err = sess.span_diagnostic.struct_span_err( - sp, - &format!( - "`${name}:{frag}` {may_be} followed by `{next}`, which \ - is not allowed for `{frag}` fragments", - name = name, - frag = frag_spec, - next = quoted_tt_to_string(next_token), - may_be = may_be - ), - ); - err.span_label( - sp, - format!("not allowed after `{}` fragments", frag_spec), - ); - let msg = "allowed there are: "; - match possible { - &[] => {} - &[t] => { - err.note(&format!( - "only {} is allowed after `{}` fragments", - t, frag_spec, - )); - } - ts => { - err.note(&format!( - "{}{} or {}", - msg, - ts[..ts.len() - 1] - .iter() - .map(|s| *s) - .collect::<Vec<_>>() - .join(", "), - ts[ts.len() - 1], - )); - } - } - err.emit(); - } - } - } - } - } - } - last -} - -fn token_can_be_followed_by_any(tok: &mbe::TokenTree) -> bool { - if let mbe::TokenTree::MetaVarDecl(_, _, frag_spec) = *tok { - frag_can_be_followed_by_any(frag_spec.name) - } else { - // (Non NT's can always be followed by anthing in matchers.) - true - } -} - -/// Returns `true` if a fragment of type `frag` can be followed by any sort of -/// token. We use this (among other things) as a useful approximation -/// for when `frag` can be followed by a repetition like `$(...)*` or -/// `$(...)+`. In general, these can be a bit tricky to reason about, -/// so we adopt a conservative position that says that any fragment -/// specifier which consumes at most one token tree can be followed by -/// a fragment specifier (indeed, these fragments can be followed by -/// ANYTHING without fear of future compatibility hazards). -fn frag_can_be_followed_by_any(frag: Symbol) -> bool { - match frag { - sym::item | // always terminated by `}` or `;` - sym::block | // exactly one token tree - sym::ident | // exactly one token tree - sym::literal | // exactly one token tree - sym::meta | // exactly one token tree - sym::lifetime | // exactly one token tree - sym::tt => // exactly one token tree - true, - - _ => - false, - } -} - -enum IsInFollow { - Yes, - No(&'static [&'static str]), - Invalid(String, &'static str), -} - -/// Returns `true` if `frag` can legally be followed by the token `tok`. For -/// fragments that can consume an unbounded number of tokens, `tok` -/// must be within a well-defined follow set. This is intended to -/// guarantee future compatibility: for example, without this rule, if -/// we expanded `expr` to include a new binary operator, we might -/// break macros that were relying on that binary operator as a -/// separator. -// when changing this do not forget to update doc/book/macros.md! -fn is_in_follow(tok: &mbe::TokenTree, frag: Symbol) -> IsInFollow { - use mbe::TokenTree; - - if let TokenTree::Token(Token { kind: token::CloseDelim(_), .. }) = *tok { - // closing a token tree can never be matched by any fragment; - // iow, we always require that `(` and `)` match, etc. - IsInFollow::Yes - } else { - match frag { - sym::item => { - // since items *must* be followed by either a `;` or a `}`, we can - // accept anything after them - IsInFollow::Yes - } - sym::block => { - // anything can follow block, the braces provide an easy boundary to - // maintain - IsInFollow::Yes - } - sym::stmt | sym::expr => { - const TOKENS: &[&str] = &["`=>`", "`,`", "`;`"]; - match tok { - TokenTree::Token(token) => match token.kind { - FatArrow | Comma | Semi => IsInFollow::Yes, - _ => IsInFollow::No(TOKENS), - }, - _ => IsInFollow::No(TOKENS), - } - } - sym::pat => { - const TOKENS: &[&str] = &["`=>`", "`,`", "`=`", "`|`", "`if`", "`in`"]; - match tok { - TokenTree::Token(token) => match token.kind { - FatArrow | Comma | Eq | BinOp(token::Or) => IsInFollow::Yes, - Ident(name, false) if name == kw::If || name == kw::In => IsInFollow::Yes, - _ => IsInFollow::No(TOKENS), - }, - _ => IsInFollow::No(TOKENS), - } - } - sym::path | sym::ty => { - const TOKENS: &[&str] = &[ - "`{`", "`[`", "`=>`", "`,`", "`>`", "`=`", "`:`", "`;`", "`|`", "`as`", - "`where`", - ]; - match tok { - TokenTree::Token(token) => match token.kind { - OpenDelim(token::DelimToken::Brace) - | OpenDelim(token::DelimToken::Bracket) - | Comma - | FatArrow - | Colon - | Eq - | Gt - | BinOp(token::Shr) - | Semi - | BinOp(token::Or) => IsInFollow::Yes, - Ident(name, false) if name == kw::As || name == kw::Where => { - IsInFollow::Yes - } - _ => IsInFollow::No(TOKENS), - }, - TokenTree::MetaVarDecl(_, _, frag) if frag.name == sym::block => { - IsInFollow::Yes - } - _ => IsInFollow::No(TOKENS), - } - } - sym::ident | sym::lifetime => { - // being a single token, idents and lifetimes are harmless - IsInFollow::Yes - } - sym::literal => { - // literals may be of a single token, or two tokens (negative numbers) - IsInFollow::Yes - } - sym::meta | sym::tt => { - // being either a single token or a delimited sequence, tt is - // harmless - IsInFollow::Yes - } - sym::vis => { - // Explicitly disallow `priv`, on the off chance it comes back. - const TOKENS: &[&str] = &["`,`", "an ident", "a type"]; - match tok { - TokenTree::Token(token) => match token.kind { - Comma => IsInFollow::Yes, - Ident(name, is_raw) if is_raw || name != kw::Priv => IsInFollow::Yes, - _ => { - if token.can_begin_type() { - IsInFollow::Yes - } else { - IsInFollow::No(TOKENS) - } - } - }, - TokenTree::MetaVarDecl(_, _, frag) - if frag.name == sym::ident - || frag.name == sym::ty - || frag.name == sym::path => - { - IsInFollow::Yes - } - _ => IsInFollow::No(TOKENS), - } - } - kw::Invalid => IsInFollow::Yes, - _ => IsInFollow::Invalid( - format!("invalid fragment specifier `{}`", frag), - VALID_FRAGMENT_NAMES_MSG, - ), - } - } -} - -fn has_legal_fragment_specifier( - sess: &ParseSess, - features: &Features, - attrs: &[ast::Attribute], - tok: &mbe::TokenTree, -) -> Result<(), String> { - debug!("has_legal_fragment_specifier({:?})", tok); - if let mbe::TokenTree::MetaVarDecl(_, _, ref frag_spec) = *tok { - let frag_span = tok.span(); - if !is_legal_fragment_specifier(sess, features, attrs, frag_spec.name, frag_span) { - return Err(frag_spec.to_string()); - } - } - Ok(()) -} - -fn is_legal_fragment_specifier( - _sess: &ParseSess, - _features: &Features, - _attrs: &[ast::Attribute], - frag_name: Symbol, - _frag_span: Span, -) -> bool { - /* - * If new fragment specifiers are invented in nightly, `_sess`, - * `_features`, `_attrs`, and `_frag_span` will be useful here - * for checking against feature gates. See past versions of - * this function. - */ - match frag_name { - sym::item - | sym::block - | sym::stmt - | sym::expr - | sym::pat - | sym::lifetime - | sym::path - | sym::ty - | sym::ident - | sym::meta - | sym::tt - | sym::vis - | sym::literal - | kw::Invalid => true, - _ => false, - } -} - -fn quoted_tt_to_string(tt: &mbe::TokenTree) -> String { - match *tt { - mbe::TokenTree::Token(ref token) => crate::print::pprust::token_to_string(&token), - mbe::TokenTree::MetaVar(_, name) => format!("${}", name), - mbe::TokenTree::MetaVarDecl(_, name, kind) => format!("${}:{}", name, kind), - _ => panic!( - "unexpected mbe::TokenTree::{{Sequence or Delimited}} \ - in follow set checker" - ), - } -} - -impl TokenTree { - /// Use this token tree as a matcher to parse given tts. - fn parse(cx: &ExtCtxt<'_>, mtch: &[mbe::TokenTree], tts: TokenStream) - -> NamedParseResult { - // `None` is because we're not interpolating - let directory = Directory { - path: Cow::from(cx.current_expansion.module.directory.as_path()), - ownership: cx.current_expansion.directory_ownership, - }; - parse(cx.parse_sess(), tts, mtch, Some(directory), true) - } -} - -/// Generates an appropriate parsing failure message. For EOF, this is "unexpected end...". For -/// other tokens, this is "unexpected token...". -fn parse_failure_msg(tok: &Token) -> String { - match tok.kind { - token::Eof => "unexpected end of macro invocation".to_string(), - _ => format!( - "no rules expected the token `{}`", - pprust::token_to_string(tok), - ), - } -} diff --git a/src/libsyntax/ext/mbe/quoted.rs b/src/libsyntax/ext/mbe/quoted.rs deleted file mode 100644 index 8cb85bdef76..00000000000 --- a/src/libsyntax/ext/mbe/quoted.rs +++ /dev/null @@ -1,263 +0,0 @@ -use crate::ast; -use crate::ext::mbe::macro_parser; -use crate::ext::mbe::{TokenTree, KleeneOp, KleeneToken, SequenceRepetition, Delimited}; -use crate::parse::token::{self, Token}; -use crate::parse::ParseSess; -use crate::print::pprust; -use crate::symbol::kw; -use crate::tokenstream; - -use syntax_pos::Span; - -use rustc_data_structures::sync::Lrc; - -/// Takes a `tokenstream::TokenStream` and returns a `Vec<self::TokenTree>`. Specifically, this -/// takes a generic `TokenStream`, such as is used in the rest of the compiler, and returns a -/// collection of `TokenTree` for use in parsing a macro. -/// -/// # Parameters -/// -/// - `input`: a token stream to read from, the contents of which we are parsing. -/// - `expect_matchers`: `parse` can be used to parse either the "patterns" or the "body" of a -/// macro. Both take roughly the same form _except_ that in a pattern, metavars are declared with -/// their "matcher" type. For example `$var:expr` or `$id:ident`. In this example, `expr` and -/// `ident` are "matchers". They are not present in the body of a macro rule -- just in the -/// pattern, so we pass a parameter to indicate whether to expect them or not. -/// - `sess`: the parsing session. Any errors will be emitted to this session. -/// - `features`, `attrs`: language feature flags and attributes so that we know whether to use -/// unstable features or not. -/// - `edition`: which edition are we in. -/// - `macro_node_id`: the NodeId of the macro we are parsing. -/// -/// # Returns -/// -/// A collection of `self::TokenTree`. There may also be some errors emitted to `sess`. -pub(super) fn parse( - input: tokenstream::TokenStream, - expect_matchers: bool, - sess: &ParseSess, -) -> Vec<TokenTree> { - // Will contain the final collection of `self::TokenTree` - let mut result = Vec::new(); - - // For each token tree in `input`, parse the token into a `self::TokenTree`, consuming - // additional trees if need be. - let mut trees = input.trees(); - while let Some(tree) = trees.next() { - // Given the parsed tree, if there is a metavar and we are expecting matchers, actually - // parse out the matcher (i.e., in `$id:ident` this would parse the `:` and `ident`). - let tree = parse_tree( - tree, - &mut trees, - expect_matchers, - sess, - ); - match tree { - TokenTree::MetaVar(start_sp, ident) if expect_matchers => { - let span = match trees.next() { - Some(tokenstream::TokenTree::Token(Token { kind: token::Colon, span })) => { - match trees.next() { - Some(tokenstream::TokenTree::Token(token)) => match token.ident() { - Some((kind, _)) => { - let span = token.span.with_lo(start_sp.lo()); - result.push(TokenTree::MetaVarDecl(span, ident, kind)); - continue; - } - _ => token.span, - }, - tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span), - } - } - tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(start_sp), - }; - sess.missing_fragment_specifiers.borrow_mut().insert(span); - result.push(TokenTree::MetaVarDecl(span, ident, ast::Ident::invalid())); - } - - // Not a metavar or no matchers allowed, so just return the tree - _ => result.push(tree), - } - } - result -} - -/// Takes a `tokenstream::TokenTree` and returns a `self::TokenTree`. Specifically, this takes a -/// generic `TokenTree`, such as is used in the rest of the compiler, and returns a `TokenTree` -/// for use in parsing a macro. -/// -/// Converting the given tree may involve reading more tokens. -/// -/// # Parameters -/// -/// - `tree`: the tree we wish to convert. -/// - `trees`: an iterator over trees. We may need to read more tokens from it in order to finish -/// converting `tree` -/// - `expect_matchers`: same as for `parse` (see above). -/// - `sess`: the parsing session. Any errors will be emitted to this session. -/// - `features`, `attrs`: language feature flags and attributes so that we know whether to use -/// unstable features or not. -fn parse_tree( - tree: tokenstream::TokenTree, - trees: &mut impl Iterator<Item = tokenstream::TokenTree>, - expect_matchers: bool, - sess: &ParseSess, -) -> TokenTree { - // Depending on what `tree` is, we could be parsing different parts of a macro - match tree { - // `tree` is a `$` token. Look at the next token in `trees` - tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }) => match trees.next() { - // `tree` is followed by a delimited set of token trees. This indicates the beginning - // of a repetition sequence in the macro (e.g. `$(pat)*`). - Some(tokenstream::TokenTree::Delimited(span, delim, tts)) => { - // Must have `(` not `{` or `[` - if delim != token::Paren { - let tok = pprust::token_kind_to_string(&token::OpenDelim(delim)); - let msg = format!("expected `(`, found `{}`", tok); - sess.span_diagnostic.span_err(span.entire(), &msg); - } - // Parse the contents of the sequence itself - let sequence = parse( - tts.into(), - expect_matchers, - sess, - ); - // Get the Kleene operator and optional separator - let (separator, kleene) = parse_sep_and_kleene_op(trees, span.entire(), sess); - // Count the number of captured "names" (i.e., named metavars) - let name_captures = macro_parser::count_names(&sequence); - TokenTree::Sequence( - span, - Lrc::new(SequenceRepetition { - tts: sequence, - separator, - kleene, - num_captures: name_captures, - }), - ) - } - - // `tree` is followed by an `ident`. This could be `$meta_var` or the `$crate` special - // metavariable that names the crate of the invocation. - Some(tokenstream::TokenTree::Token(token)) if token.is_ident() => { - let (ident, is_raw) = token.ident().unwrap(); - let span = ident.span.with_lo(span.lo()); - if ident.name == kw::Crate && !is_raw { - TokenTree::token(token::Ident(kw::DollarCrate, is_raw), span) - } else { - TokenTree::MetaVar(span, ident) - } - } - - // `tree` is followed by a random token. This is an error. - Some(tokenstream::TokenTree::Token(token)) => { - let msg = - format!("expected identifier, found `{}`", pprust::token_to_string(&token),); - sess.span_diagnostic.span_err(token.span, &msg); - TokenTree::MetaVar(token.span, ast::Ident::invalid()) - } - - // There are no more tokens. Just return the `$` we already have. - None => TokenTree::token(token::Dollar, span), - }, - - // `tree` is an arbitrary token. Keep it. - tokenstream::TokenTree::Token(token) => TokenTree::Token(token), - - // `tree` is the beginning of a delimited set of tokens (e.g., `(` or `{`). We need to - // descend into the delimited set and further parse it. - tokenstream::TokenTree::Delimited(span, delim, tts) => TokenTree::Delimited( - span, - Lrc::new(Delimited { - delim, - tts: parse( - tts.into(), - expect_matchers, - sess, - ), - }), - ), - } -} - -/// Takes a token and returns `Some(KleeneOp)` if the token is `+` `*` or `?`. Otherwise, return -/// `None`. -fn kleene_op(token: &Token) -> Option<KleeneOp> { - match token.kind { - token::BinOp(token::Star) => Some(KleeneOp::ZeroOrMore), - token::BinOp(token::Plus) => Some(KleeneOp::OneOrMore), - token::Question => Some(KleeneOp::ZeroOrOne), - _ => None, - } -} - -/// Parse the next token tree of the input looking for a KleeneOp. Returns -/// -/// - Ok(Ok((op, span))) if the next token tree is a KleeneOp -/// - Ok(Err(tok, span)) if the next token tree is a token but not a KleeneOp -/// - Err(span) if the next token tree is not a token -fn parse_kleene_op( - input: &mut impl Iterator<Item = tokenstream::TokenTree>, - span: Span, -) -> Result<Result<(KleeneOp, Span), Token>, Span> { - match input.next() { - Some(tokenstream::TokenTree::Token(token)) => match kleene_op(&token) { - Some(op) => Ok(Ok((op, token.span))), - None => Ok(Err(token)), - }, - tree => Err(tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span)), - } -} - -/// Attempt to parse a single Kleene star, possibly with a separator. -/// -/// For example, in a pattern such as `$(a),*`, `a` is the pattern to be repeated, `,` is the -/// separator, and `*` is the Kleene operator. This function is specifically concerned with parsing -/// the last two tokens of such a pattern: namely, the optional separator and the Kleene operator -/// itself. Note that here we are parsing the _macro_ itself, rather than trying to match some -/// stream of tokens in an invocation of a macro. -/// -/// This function will take some input iterator `input` corresponding to `span` and a parsing -/// session `sess`. If the next one (or possibly two) tokens in `input` correspond to a Kleene -/// operator and separator, then a tuple with `(separator, KleeneOp)` is returned. Otherwise, an -/// error with the appropriate span is emitted to `sess` and a dummy value is returned. -fn parse_sep_and_kleene_op( - input: &mut impl Iterator<Item = tokenstream::TokenTree>, - span: Span, - sess: &ParseSess, -) -> (Option<Token>, KleeneToken) { - // We basically look at two token trees here, denoted as #1 and #2 below - let span = match parse_kleene_op(input, span) { - // #1 is a `?`, `+`, or `*` KleeneOp - Ok(Ok((op, span))) => return (None, KleeneToken::new(op, span)), - - // #1 is a separator followed by #2, a KleeneOp - Ok(Err(token)) => match parse_kleene_op(input, token.span) { - // #2 is the `?` Kleene op, which does not take a separator (error) - Ok(Ok((KleeneOp::ZeroOrOne, span))) => { - // Error! - sess.span_diagnostic.span_err( - token.span, - "the `?` macro repetition operator does not take a separator", - ); - - // Return a dummy - return (None, KleeneToken::new(KleeneOp::ZeroOrMore, span)); - } - - // #2 is a KleeneOp :D - Ok(Ok((op, span))) => return (Some(token), KleeneToken::new(op, span)), - - // #2 is a random token or not a token at all :( - Ok(Err(Token { span, .. })) | Err(span) => span, - }, - - // #1 is not a token - Err(span) => span, - }; - - // If we ever get to this point, we have experienced an "unexpected token" error - sess.span_diagnostic.span_err(span, "expected one of: `*`, `+`, or `?`"); - - // Return a dummy - (None, KleeneToken::new(KleeneOp::ZeroOrMore, span)) -} diff --git a/src/libsyntax/ext/mbe/transcribe.rs b/src/libsyntax/ext/mbe/transcribe.rs deleted file mode 100644 index ba818ebd35c..00000000000 --- a/src/libsyntax/ext/mbe/transcribe.rs +++ /dev/null @@ -1,398 +0,0 @@ -use crate::ast::{Ident, Mac}; -use crate::ext::base::ExtCtxt; -use crate::ext::mbe; -use crate::ext::mbe::macro_parser::{MatchedNonterminal, MatchedSeq, NamedMatch}; -use crate::mut_visit::{self, MutVisitor}; -use crate::parse::token::{self, NtTT, Token}; -use crate::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndJoint}; - -use smallvec::{smallvec, SmallVec}; - -use errors::pluralise; -use rustc_data_structures::fx::FxHashMap; -use rustc_data_structures::sync::Lrc; -use syntax_pos::hygiene::{ExpnId, Transparency}; -use syntax_pos::Span; - -use std::mem; - -// A Marker adds the given mark to the syntax context. -struct Marker(ExpnId, Transparency); - -impl MutVisitor for Marker { - fn visit_span(&mut self, span: &mut Span) { - *span = span.apply_mark(self.0, self.1) - } - - fn visit_mac(&mut self, mac: &mut Mac) { - mut_visit::noop_visit_mac(mac, self) - } -} - -impl Marker { - fn visit_delim_span(&mut self, dspan: &mut DelimSpan) { - self.visit_span(&mut dspan.open); - self.visit_span(&mut dspan.close); - } -} - -/// An iterator over the token trees in a delimited token tree (`{ ... }`) or a sequence (`$(...)`). -enum Frame { - Delimited { forest: Lrc<mbe::Delimited>, idx: usize, span: DelimSpan }, - Sequence { forest: Lrc<mbe::SequenceRepetition>, idx: usize, sep: Option<Token> }, -} - -impl Frame { - /// Construct a new frame around the delimited set of tokens. - fn new(tts: Vec<mbe::TokenTree>) -> Frame { - let forest = Lrc::new(mbe::Delimited { delim: token::NoDelim, tts }); - Frame::Delimited { forest, idx: 0, span: DelimSpan::dummy() } - } -} - -impl Iterator for Frame { - type Item = mbe::TokenTree; - - fn next(&mut self) -> Option<mbe::TokenTree> { - match *self { - Frame::Delimited { ref forest, ref mut idx, .. } => { - *idx += 1; - forest.tts.get(*idx - 1).cloned() - } - Frame::Sequence { ref forest, ref mut idx, .. } => { - *idx += 1; - forest.tts.get(*idx - 1).cloned() - } - } - } -} - -/// This can do Macro-By-Example transcription. -/// - `interp` is a map of meta-variables to the tokens (non-terminals) they matched in the -/// invocation. We are assuming we already know there is a match. -/// - `src` is the RHS of the MBE, that is, the "example" we are filling in. -/// -/// For example, -/// -/// ```rust -/// macro_rules! foo { -/// ($id:ident) => { println!("{}", stringify!($id)); } -/// } -/// -/// foo!(bar); -/// ``` -/// -/// `interp` would contain `$id => bar` and `src` would contain `println!("{}", stringify!($id));`. -/// -/// `transcribe` would return a `TokenStream` containing `println!("{}", stringify!(bar));`. -/// -/// Along the way, we do some additional error checking. -pub(super) fn transcribe( - cx: &ExtCtxt<'_>, - interp: &FxHashMap<Ident, NamedMatch>, - src: Vec<mbe::TokenTree>, - transparency: Transparency, -) -> TokenStream { - // Nothing for us to transcribe... - if src.is_empty() { - return TokenStream::empty(); - } - - // We descend into the RHS (`src`), expanding things as we go. This stack contains the things - // we have yet to expand/are still expanding. We start the stack off with the whole RHS. - let mut stack: SmallVec<[Frame; 1]> = smallvec![Frame::new(src)]; - - // As we descend in the RHS, we will need to be able to match nested sequences of matchers. - // `repeats` keeps track of where we are in matching at each level, with the last element being - // the most deeply nested sequence. This is used as a stack. - let mut repeats = Vec::new(); - - // `result` contains resulting token stream from the TokenTree we just finished processing. At - // the end, this will contain the full result of transcription, but at arbitrary points during - // `transcribe`, `result` will contain subsets of the final result. - // - // Specifically, as we descend into each TokenTree, we will push the existing results onto the - // `result_stack` and clear `results`. We will then produce the results of transcribing the - // TokenTree into `results`. Then, as we unwind back out of the `TokenTree`, we will pop the - // `result_stack` and append `results` too it to produce the new `results` up to that point. - // - // Thus, if we try to pop the `result_stack` and it is empty, we have reached the top-level - // again, and we are done transcribing. - let mut result: Vec<TreeAndJoint> = Vec::new(); - let mut result_stack = Vec::new(); - let mut marker = Marker(cx.current_expansion.id, transparency); - - loop { - // Look at the last frame on the stack. - let tree = if let Some(tree) = stack.last_mut().unwrap().next() { - // If it still has a TokenTree we have not looked at yet, use that tree. - tree - } - // The else-case never produces a value for `tree` (it `continue`s or `return`s). - else { - // Otherwise, if we have just reached the end of a sequence and we can keep repeating, - // go back to the beginning of the sequence. - if let Frame::Sequence { idx, sep, .. } = stack.last_mut().unwrap() { - let (repeat_idx, repeat_len) = repeats.last_mut().unwrap(); - *repeat_idx += 1; - if repeat_idx < repeat_len { - *idx = 0; - if let Some(sep) = sep { - result.push(TokenTree::Token(sep.clone()).into()); - } - continue; - } - } - - // We are done with the top of the stack. Pop it. Depending on what it was, we do - // different things. Note that the outermost item must be the delimited, wrapped RHS - // that was passed in originally to `transcribe`. - match stack.pop().unwrap() { - // Done with a sequence. Pop from repeats. - Frame::Sequence { .. } => { - repeats.pop(); - } - - // We are done processing a Delimited. If this is the top-level delimited, we are - // done. Otherwise, we unwind the result_stack to append what we have produced to - // any previous results. - Frame::Delimited { forest, span, .. } => { - if result_stack.is_empty() { - // No results left to compute! We are back at the top-level. - return TokenStream::new(result); - } - - // Step back into the parent Delimited. - let tree = - TokenTree::Delimited(span, forest.delim, TokenStream::new(result).into()); - result = result_stack.pop().unwrap(); - result.push(tree.into()); - } - } - continue; - }; - - // At this point, we know we are in the middle of a TokenTree (the last one on `stack`). - // `tree` contains the next `TokenTree` to be processed. - match tree { - // We are descending into a sequence. We first make sure that the matchers in the RHS - // and the matches in `interp` have the same shape. Otherwise, either the caller or the - // macro writer has made a mistake. - seq @ mbe::TokenTree::Sequence(..) => { - match lockstep_iter_size(&seq, interp, &repeats) { - LockstepIterSize::Unconstrained => { - cx.span_fatal( - seq.span(), /* blame macro writer */ - "attempted to repeat an expression containing no syntax variables \ - matched as repeating at this depth", - ); - } - - LockstepIterSize::Contradiction(ref msg) => { - // FIXME: this really ought to be caught at macro definition time... It - // happens when two meta-variables are used in the same repetition in a - // sequence, but they come from different sequence matchers and repeat - // different amounts. - cx.span_fatal(seq.span(), &msg[..]); - } - - LockstepIterSize::Constraint(len, _) => { - // We do this to avoid an extra clone above. We know that this is a - // sequence already. - let (sp, seq) = if let mbe::TokenTree::Sequence(sp, seq) = seq { - (sp, seq) - } else { - unreachable!() - }; - - // Is the repetition empty? - if len == 0 { - if seq.kleene.op == mbe::KleeneOp::OneOrMore { - // FIXME: this really ought to be caught at macro definition - // time... It happens when the Kleene operator in the matcher and - // the body for the same meta-variable do not match. - cx.span_fatal(sp.entire(), "this must repeat at least once"); - } - } else { - // 0 is the initial counter (we have done 0 repretitions so far). `len` - // is the total number of reptitions we should generate. - repeats.push((0, len)); - - // The first time we encounter the sequence we push it to the stack. It - // then gets reused (see the beginning of the loop) until we are done - // repeating. - stack.push(Frame::Sequence { - idx: 0, - sep: seq.separator.clone(), - forest: seq, - }); - } - } - } - } - - // Replace the meta-var with the matched token tree from the invocation. - mbe::TokenTree::MetaVar(mut sp, mut ident) => { - // Find the matched nonterminal from the macro invocation, and use it to replace - // the meta-var. - if let Some(cur_matched) = lookup_cur_matched(ident, interp, &repeats) { - if let MatchedNonterminal(ref nt) = cur_matched { - // FIXME #2887: why do we apply a mark when matching a token tree meta-var - // (e.g. `$x:tt`), but not when we are matching any other type of token - // tree? - if let NtTT(ref tt) = **nt { - result.push(tt.clone().into()); - } else { - marker.visit_span(&mut sp); - let token = TokenTree::token(token::Interpolated(nt.clone()), sp); - result.push(token.into()); - } - } else { - // We were unable to descend far enough. This is an error. - cx.span_fatal( - sp, /* blame the macro writer */ - &format!("variable '{}' is still repeating at this depth", ident), - ); - } - } else { - // If we aren't able to match the meta-var, we push it back into the result but - // with modified syntax context. (I believe this supports nested macros). - marker.visit_span(&mut sp); - marker.visit_ident(&mut ident); - result.push(TokenTree::token(token::Dollar, sp).into()); - result.push(TokenTree::Token(Token::from_ast_ident(ident)).into()); - } - } - - // If we are entering a new delimiter, we push its contents to the `stack` to be - // processed, and we push all of the currently produced results to the `result_stack`. - // We will produce all of the results of the inside of the `Delimited` and then we will - // jump back out of the Delimited, pop the result_stack and add the new results back to - // the previous results (from outside the Delimited). - mbe::TokenTree::Delimited(mut span, delimited) => { - marker.visit_delim_span(&mut span); - stack.push(Frame::Delimited { forest: delimited, idx: 0, span }); - result_stack.push(mem::take(&mut result)); - } - - // Nothing much to do here. Just push the token to the result, being careful to - // preserve syntax context. - mbe::TokenTree::Token(token) => { - let mut tt = TokenTree::Token(token); - marker.visit_tt(&mut tt); - result.push(tt.into()); - } - - // There should be no meta-var declarations in the invocation of a macro. - mbe::TokenTree::MetaVarDecl(..) => panic!("unexpected `TokenTree::MetaVarDecl"), - } - } -} - -/// Lookup the meta-var named `ident` and return the matched token tree from the invocation using -/// the set of matches `interpolations`. -/// -/// See the definition of `repeats` in the `transcribe` function. `repeats` is used to descend -/// into the right place in nested matchers. If we attempt to descend too far, the macro writer has -/// made a mistake, and we return `None`. -fn lookup_cur_matched<'a>( - ident: Ident, - interpolations: &'a FxHashMap<Ident, NamedMatch>, - repeats: &[(usize, usize)], -) -> Option<&'a NamedMatch> { - interpolations.get(&ident).map(|matched| { - let mut matched = matched; - for &(idx, _) in repeats { - match matched { - MatchedNonterminal(_) => break, - MatchedSeq(ref ads, _) => matched = ads.get(idx).unwrap(), - } - } - - matched - }) -} - -/// An accumulator over a TokenTree to be used with `fold`. During transcription, we need to make -/// sure that the size of each sequence and all of its nested sequences are the same as the sizes -/// of all the matched (nested) sequences in the macro invocation. If they don't match, somebody -/// has made a mistake (either the macro writer or caller). -#[derive(Clone)] -enum LockstepIterSize { - /// No constraints on length of matcher. This is true for any TokenTree variants except a - /// `MetaVar` with an actual `MatchedSeq` (as opposed to a `MatchedNonterminal`). - Unconstrained, - - /// A `MetaVar` with an actual `MatchedSeq`. The length of the match and the name of the - /// meta-var are returned. - Constraint(usize, Ident), - - /// Two `Constraint`s on the same sequence had different lengths. This is an error. - Contradiction(String), -} - -impl LockstepIterSize { - /// Find incompatibilities in matcher/invocation sizes. - /// - `Unconstrained` is compatible with everything. - /// - `Contradiction` is incompatible with everything. - /// - `Constraint(len)` is only compatible with other constraints of the same length. - fn with(self, other: LockstepIterSize) -> LockstepIterSize { - match self { - LockstepIterSize::Unconstrained => other, - LockstepIterSize::Contradiction(_) => self, - LockstepIterSize::Constraint(l_len, ref l_id) => match other { - LockstepIterSize::Unconstrained => self, - LockstepIterSize::Contradiction(_) => other, - LockstepIterSize::Constraint(r_len, _) if l_len == r_len => self, - LockstepIterSize::Constraint(r_len, r_id) => { - let msg = format!( - "meta-variable `{}` repeats {} time{}, but `{}` repeats {} time{}", - l_id, - l_len, - pluralise!(l_len), - r_id, - r_len, - pluralise!(r_len), - ); - LockstepIterSize::Contradiction(msg) - } - }, - } - } -} - -/// Given a `tree`, make sure that all sequences have the same length as the matches for the -/// appropriate meta-vars in `interpolations`. -/// -/// Note that if `repeats` does not match the exact correct depth of a meta-var, -/// `lookup_cur_matched` will return `None`, which is why this still works even in the presnece of -/// multiple nested matcher sequences. -fn lockstep_iter_size( - tree: &mbe::TokenTree, - interpolations: &FxHashMap<Ident, NamedMatch>, - repeats: &[(usize, usize)], -) -> LockstepIterSize { - use mbe::TokenTree; - match *tree { - TokenTree::Delimited(_, ref delimed) => { - delimed.tts.iter().fold(LockstepIterSize::Unconstrained, |size, tt| { - size.with(lockstep_iter_size(tt, interpolations, repeats)) - }) - } - TokenTree::Sequence(_, ref seq) => { - seq.tts.iter().fold(LockstepIterSize::Unconstrained, |size, tt| { - size.with(lockstep_iter_size(tt, interpolations, repeats)) - }) - } - TokenTree::MetaVar(_, name) | TokenTree::MetaVarDecl(_, name, _) => { - match lookup_cur_matched(name, interpolations, repeats) { - Some(matched) => match matched { - MatchedNonterminal(_) => LockstepIterSize::Unconstrained, - MatchedSeq(ref ads, _) => LockstepIterSize::Constraint(ads.len(), name), - }, - _ => LockstepIterSize::Unconstrained, - } - } - TokenTree::Token(..) => LockstepIterSize::Unconstrained, - } -} diff --git a/src/libsyntax/ext/placeholders.rs b/src/libsyntax/ext/placeholders.rs deleted file mode 100644 index 8eecef1020d..00000000000 --- a/src/libsyntax/ext/placeholders.rs +++ /dev/null @@ -1,349 +0,0 @@ -use crate::ast::{self, NodeId}; -use crate::source_map::{DUMMY_SP, dummy_spanned}; -use crate::ext::base::ExtCtxt; -use crate::ext::expand::{AstFragment, AstFragmentKind}; -use crate::tokenstream::TokenStream; -use crate::mut_visit::*; -use crate::ptr::P; -use crate::ThinVec; - -use smallvec::{smallvec, SmallVec}; - -use rustc_data_structures::fx::FxHashMap; - -pub fn placeholder(kind: AstFragmentKind, id: ast::NodeId) -> AstFragment { - fn mac_placeholder() -> ast::Mac { - ast::Mac { - path: ast::Path { span: DUMMY_SP, segments: Vec::new() }, - tts: TokenStream::empty().into(), - delim: ast::MacDelimiter::Brace, - span: DUMMY_SP, - prior_type_ascription: None, - } - } - - let ident = ast::Ident::invalid(); - let attrs = Vec::new(); - let generics = ast::Generics::default(); - let vis = dummy_spanned(ast::VisibilityKind::Inherited); - let span = DUMMY_SP; - let expr_placeholder = || P(ast::Expr { - id, span, - attrs: ThinVec::new(), - kind: ast::ExprKind::Mac(mac_placeholder()), - }); - let ty = P(ast::Ty { - id, - kind: ast::TyKind::Mac(mac_placeholder()), - span, - }); - let pat = P(ast::Pat { - id, - kind: ast::PatKind::Mac(mac_placeholder()), - span, - }); - - match kind { - AstFragmentKind::Expr => AstFragment::Expr(expr_placeholder()), - AstFragmentKind::OptExpr => AstFragment::OptExpr(Some(expr_placeholder())), - AstFragmentKind::Items => AstFragment::Items(smallvec![P(ast::Item { - id, span, ident, vis, attrs, - kind: ast::ItemKind::Mac(mac_placeholder()), - tokens: None, - })]), - AstFragmentKind::TraitItems => AstFragment::TraitItems(smallvec![ast::TraitItem { - id, span, ident, attrs, generics, - kind: ast::TraitItemKind::Macro(mac_placeholder()), - tokens: None, - }]), - AstFragmentKind::ImplItems => AstFragment::ImplItems(smallvec![ast::ImplItem { - id, span, ident, vis, attrs, generics, - kind: ast::ImplItemKind::Macro(mac_placeholder()), - defaultness: ast::Defaultness::Final, - tokens: None, - }]), - AstFragmentKind::ForeignItems => - AstFragment::ForeignItems(smallvec![ast::ForeignItem { - id, span, ident, vis, attrs, - kind: ast::ForeignItemKind::Macro(mac_placeholder()), - }]), - AstFragmentKind::Pat => AstFragment::Pat(P(ast::Pat { - id, span, kind: ast::PatKind::Mac(mac_placeholder()), - })), - AstFragmentKind::Ty => AstFragment::Ty(P(ast::Ty { - id, span, kind: ast::TyKind::Mac(mac_placeholder()), - })), - AstFragmentKind::Stmts => AstFragment::Stmts(smallvec![{ - let mac = P((mac_placeholder(), ast::MacStmtStyle::Braces, ThinVec::new())); - ast::Stmt { id, span, kind: ast::StmtKind::Mac(mac) } - }]), - AstFragmentKind::Arms => AstFragment::Arms(smallvec![ - ast::Arm { - attrs: Default::default(), - body: expr_placeholder(), - guard: None, - id, - pat, - span, - is_placeholder: true, - } - ]), - AstFragmentKind::Fields => AstFragment::Fields(smallvec![ - ast::Field { - attrs: Default::default(), - expr: expr_placeholder(), - id, - ident, - is_shorthand: false, - span, - is_placeholder: true, - } - ]), - AstFragmentKind::FieldPats => AstFragment::FieldPats(smallvec![ - ast::FieldPat { - attrs: Default::default(), - id, - ident, - is_shorthand: false, - pat, - span, - is_placeholder: true, - } - ]), - AstFragmentKind::GenericParams => AstFragment::GenericParams(smallvec![{ - ast::GenericParam { - attrs: Default::default(), - bounds: Default::default(), - id, - ident, - is_placeholder: true, - kind: ast::GenericParamKind::Lifetime, - } - }]), - AstFragmentKind::Params => AstFragment::Params(smallvec![ - ast::Param { - attrs: Default::default(), - id, - pat, - span, - ty, - is_placeholder: true, - } - ]), - AstFragmentKind::StructFields => AstFragment::StructFields(smallvec![ - ast::StructField { - attrs: Default::default(), - id, - ident: None, - span, - ty, - vis, - is_placeholder: true, - } - ]), - AstFragmentKind::Variants => AstFragment::Variants(smallvec![ - ast::Variant { - attrs: Default::default(), - data: ast::VariantData::Struct(Default::default(), false), - disr_expr: None, - id, - ident, - span, - is_placeholder: true, - } - ]) - } -} - -pub struct PlaceholderExpander<'a, 'b> { - expanded_fragments: FxHashMap<ast::NodeId, AstFragment>, - cx: &'a mut ExtCtxt<'b>, - monotonic: bool, -} - -impl<'a, 'b> PlaceholderExpander<'a, 'b> { - pub fn new(cx: &'a mut ExtCtxt<'b>, monotonic: bool) -> Self { - PlaceholderExpander { - cx, - expanded_fragments: FxHashMap::default(), - monotonic, - } - } - - pub fn add(&mut self, id: ast::NodeId, mut fragment: AstFragment, placeholders: Vec<NodeId>) { - fragment.mut_visit_with(self); - if let AstFragment::Items(mut items) = fragment { - for placeholder in placeholders { - match self.remove(placeholder) { - AstFragment::Items(derived_items) => items.extend(derived_items), - _ => unreachable!(), - } - } - fragment = AstFragment::Items(items); - } - self.expanded_fragments.insert(id, fragment); - } - - fn remove(&mut self, id: ast::NodeId) -> AstFragment { - self.expanded_fragments.remove(&id).unwrap() - } -} - -impl<'a, 'b> MutVisitor for PlaceholderExpander<'a, 'b> { - fn flat_map_arm(&mut self, arm: ast::Arm) -> SmallVec<[ast::Arm; 1]> { - if arm.is_placeholder { - self.remove(arm.id).make_arms() - } else { - noop_flat_map_arm(arm, self) - } - } - - fn flat_map_field(&mut self, field: ast::Field) -> SmallVec<[ast::Field; 1]> { - if field.is_placeholder { - self.remove(field.id).make_fields() - } else { - noop_flat_map_field(field, self) - } - } - - fn flat_map_field_pattern(&mut self, fp: ast::FieldPat) -> SmallVec<[ast::FieldPat; 1]> { - if fp.is_placeholder { - self.remove(fp.id).make_field_patterns() - } else { - noop_flat_map_field_pattern(fp, self) - } - } - - fn flat_map_generic_param( - &mut self, - param: ast::GenericParam - ) -> SmallVec<[ast::GenericParam; 1]> - { - if param.is_placeholder { - self.remove(param.id).make_generic_params() - } else { - noop_flat_map_generic_param(param, self) - } - } - - fn flat_map_param(&mut self, p: ast::Param) -> SmallVec<[ast::Param; 1]> { - if p.is_placeholder { - self.remove(p.id).make_params() - } else { - noop_flat_map_param(p, self) - } - } - - fn flat_map_struct_field(&mut self, sf: ast::StructField) -> SmallVec<[ast::StructField; 1]> { - if sf.is_placeholder { - self.remove(sf.id).make_struct_fields() - } else { - noop_flat_map_struct_field(sf, self) - } - } - - fn flat_map_variant(&mut self, variant: ast::Variant) -> SmallVec<[ast::Variant; 1]> { - if variant.is_placeholder { - self.remove(variant.id).make_variants() - } else { - noop_flat_map_variant(variant, self) - } - } - - fn flat_map_item(&mut self, item: P<ast::Item>) -> SmallVec<[P<ast::Item>; 1]> { - match item.kind { - ast::ItemKind::Mac(_) => return self.remove(item.id).make_items(), - ast::ItemKind::MacroDef(_) => return smallvec![item], - _ => {} - } - - noop_flat_map_item(item, self) - } - - fn flat_map_trait_item(&mut self, item: ast::TraitItem) -> SmallVec<[ast::TraitItem; 1]> { - match item.kind { - ast::TraitItemKind::Macro(_) => self.remove(item.id).make_trait_items(), - _ => noop_flat_map_trait_item(item, self), - } - } - - fn flat_map_impl_item(&mut self, item: ast::ImplItem) -> SmallVec<[ast::ImplItem; 1]> { - match item.kind { - ast::ImplItemKind::Macro(_) => self.remove(item.id).make_impl_items(), - _ => noop_flat_map_impl_item(item, self), - } - } - - fn flat_map_foreign_item(&mut self, item: ast::ForeignItem) -> SmallVec<[ast::ForeignItem; 1]> { - match item.kind { - ast::ForeignItemKind::Macro(_) => self.remove(item.id).make_foreign_items(), - _ => noop_flat_map_foreign_item(item, self), - } - } - - fn visit_expr(&mut self, expr: &mut P<ast::Expr>) { - match expr.kind { - ast::ExprKind::Mac(_) => *expr = self.remove(expr.id).make_expr(), - _ => noop_visit_expr(expr, self), - } - } - - fn filter_map_expr(&mut self, expr: P<ast::Expr>) -> Option<P<ast::Expr>> { - match expr.kind { - ast::ExprKind::Mac(_) => self.remove(expr.id).make_opt_expr(), - _ => noop_filter_map_expr(expr, self), - } - } - - fn flat_map_stmt(&mut self, stmt: ast::Stmt) -> SmallVec<[ast::Stmt; 1]> { - let (style, mut stmts) = match stmt.kind { - ast::StmtKind::Mac(mac) => (mac.1, self.remove(stmt.id).make_stmts()), - _ => return noop_flat_map_stmt(stmt, self), - }; - - if style == ast::MacStmtStyle::Semicolon { - if let Some(stmt) = stmts.pop() { - stmts.push(stmt.add_trailing_semicolon()); - } - } - - stmts - } - - fn visit_pat(&mut self, pat: &mut P<ast::Pat>) { - match pat.kind { - ast::PatKind::Mac(_) => *pat = self.remove(pat.id).make_pat(), - _ => noop_visit_pat(pat, self), - } - } - - fn visit_ty(&mut self, ty: &mut P<ast::Ty>) { - match ty.kind { - ast::TyKind::Mac(_) => *ty = self.remove(ty.id).make_ty(), - _ => noop_visit_ty(ty, self), - } - } - - fn visit_block(&mut self, block: &mut P<ast::Block>) { - noop_visit_block(block, self); - - for stmt in block.stmts.iter_mut() { - if self.monotonic { - assert_eq!(stmt.id, ast::DUMMY_NODE_ID); - stmt.id = self.cx.resolver.next_node_id(); - } - } - } - - fn visit_mod(&mut self, module: &mut ast::Mod) { - noop_visit_mod(module, self); - module.items.retain(|item| match item.kind { - ast::ItemKind::Mac(_) if !self.cx.ecfg.keep_macs => false, // remove macro definitions - _ => true, - }); - } - - fn visit_mac(&mut self, _mac: &mut ast::Mac) { - // Do nothing. - } -} diff --git a/src/libsyntax/ext/proc_macro.rs b/src/libsyntax/ext/proc_macro.rs deleted file mode 100644 index e17bbf79fd5..00000000000 --- a/src/libsyntax/ext/proc_macro.rs +++ /dev/null @@ -1,221 +0,0 @@ -use crate::ast::{self, ItemKind, Attribute, Mac}; -use crate::attr::{mark_used, mark_known}; -use crate::errors::{Applicability, FatalError}; -use crate::ext::base::{self, *}; -use crate::ext::proc_macro_server; -use crate::parse::{self, token}; -use crate::parse::parser::PathStyle; -use crate::symbol::sym; -use crate::tokenstream::{self, TokenStream}; -use crate::visit::Visitor; - -use rustc_data_structures::sync::Lrc; -use syntax_pos::{Span, DUMMY_SP}; - -const EXEC_STRATEGY: proc_macro::bridge::server::SameThread = - proc_macro::bridge::server::SameThread; - -pub struct BangProcMacro { - pub client: proc_macro::bridge::client::Client< - fn(proc_macro::TokenStream) -> proc_macro::TokenStream, - >, -} - -impl base::ProcMacro for BangProcMacro { - fn expand<'cx>(&self, - ecx: &'cx mut ExtCtxt<'_>, - span: Span, - input: TokenStream) - -> TokenStream { - let server = proc_macro_server::Rustc::new(ecx); - match self.client.run(&EXEC_STRATEGY, server, input) { - Ok(stream) => stream, - Err(e) => { - let msg = "proc macro panicked"; - let mut err = ecx.struct_span_fatal(span, msg); - if let Some(s) = e.as_str() { - err.help(&format!("message: {}", s)); - } - - err.emit(); - FatalError.raise(); - } - } - } -} - -pub struct AttrProcMacro { - pub client: proc_macro::bridge::client::Client< - fn(proc_macro::TokenStream, proc_macro::TokenStream) -> proc_macro::TokenStream, - >, -} - -impl base::AttrProcMacro for AttrProcMacro { - fn expand<'cx>(&self, - ecx: &'cx mut ExtCtxt<'_>, - span: Span, - annotation: TokenStream, - annotated: TokenStream) - -> TokenStream { - let server = proc_macro_server::Rustc::new(ecx); - match self.client.run(&EXEC_STRATEGY, server, annotation, annotated) { - Ok(stream) => stream, - Err(e) => { - let msg = "custom attribute panicked"; - let mut err = ecx.struct_span_fatal(span, msg); - if let Some(s) = e.as_str() { - err.help(&format!("message: {}", s)); - } - - err.emit(); - FatalError.raise(); - } - } - } -} - -pub struct ProcMacroDerive { - pub client: proc_macro::bridge::client::Client< - fn(proc_macro::TokenStream) -> proc_macro::TokenStream, - >, -} - -impl MultiItemModifier for ProcMacroDerive { - fn expand(&self, - ecx: &mut ExtCtxt<'_>, - span: Span, - _meta_item: &ast::MetaItem, - item: Annotatable) - -> Vec<Annotatable> { - let item = match item { - Annotatable::Arm(..) | - Annotatable::Field(..) | - Annotatable::FieldPat(..) | - Annotatable::GenericParam(..) | - Annotatable::Param(..) | - Annotatable::StructField(..) | - Annotatable::Variant(..) - => panic!("unexpected annotatable"), - Annotatable::Item(item) => item, - Annotatable::ImplItem(_) | - Annotatable::TraitItem(_) | - Annotatable::ForeignItem(_) | - Annotatable::Stmt(_) | - Annotatable::Expr(_) => { - ecx.span_err(span, "proc-macro derives may only be \ - applied to a struct, enum, or union"); - return Vec::new() - } - }; - match item.kind { - ItemKind::Struct(..) | - ItemKind::Enum(..) | - ItemKind::Union(..) => {}, - _ => { - ecx.span_err(span, "proc-macro derives may only be \ - applied to a struct, enum, or union"); - return Vec::new() - } - } - - let token = token::Interpolated(Lrc::new(token::NtItem(item))); - let input = tokenstream::TokenTree::token(token, DUMMY_SP).into(); - - let server = proc_macro_server::Rustc::new(ecx); - let stream = match self.client.run(&EXEC_STRATEGY, server, input) { - Ok(stream) => stream, - Err(e) => { - let msg = "proc-macro derive panicked"; - let mut err = ecx.struct_span_fatal(span, msg); - if let Some(s) = e.as_str() { - err.help(&format!("message: {}", s)); - } - - err.emit(); - FatalError.raise(); - } - }; - - let error_count_before = ecx.parse_sess.span_diagnostic.err_count(); - let msg = "proc-macro derive produced unparseable tokens"; - - let mut parser = parse::stream_to_parser(ecx.parse_sess, stream, Some("proc-macro derive")); - let mut items = vec![]; - - loop { - match parser.parse_item() { - Ok(None) => break, - Ok(Some(item)) => { - items.push(Annotatable::Item(item)) - } - Err(mut err) => { - // FIXME: handle this better - err.cancel(); - ecx.struct_span_fatal(span, msg).emit(); - FatalError.raise(); - } - } - } - - - // fail if there have been errors emitted - if ecx.parse_sess.span_diagnostic.err_count() > error_count_before { - ecx.struct_span_fatal(span, msg).emit(); - FatalError.raise(); - } - - items - } -} - -crate struct MarkAttrs<'a>(crate &'a [ast::Name]); - -impl<'a> Visitor<'a> for MarkAttrs<'a> { - fn visit_attribute(&mut self, attr: &Attribute) { - if let Some(ident) = attr.ident() { - if self.0.contains(&ident.name) { - mark_used(attr); - mark_known(attr); - } - } - } - - fn visit_mac(&mut self, _mac: &Mac) {} -} - -pub fn is_proc_macro_attr(attr: &Attribute) -> bool { - [sym::proc_macro, sym::proc_macro_attribute, sym::proc_macro_derive] - .iter().any(|kind| attr.check_name(*kind)) -} - -crate fn collect_derives(cx: &mut ExtCtxt<'_>, attrs: &mut Vec<ast::Attribute>) -> Vec<ast::Path> { - let mut result = Vec::new(); - attrs.retain(|attr| { - if attr.path != sym::derive { - return true; - } - if !attr.is_meta_item_list() { - cx.struct_span_err(attr.span, "malformed `derive` attribute input") - .span_suggestion( - attr.span, - "missing traits to be derived", - "#[derive(Trait1, Trait2, ...)]".to_owned(), - Applicability::HasPlaceholders, - ).emit(); - return false; - } - - match attr.parse_list(cx.parse_sess, - |parser| parser.parse_path_allowing_meta(PathStyle::Mod)) { - Ok(traits) => { - result.extend(traits); - true - } - Err(mut e) => { - e.emit(); - false - } - } - }); - result -} diff --git a/src/libsyntax/ext/proc_macro_server.rs b/src/libsyntax/ext/proc_macro_server.rs deleted file mode 100644 index 021ec46d987..00000000000 --- a/src/libsyntax/ext/proc_macro_server.rs +++ /dev/null @@ -1,711 +0,0 @@ -use crate::ast; -use crate::ext::base::ExtCtxt; -use crate::parse::{self, token, ParseSess}; -use crate::parse::lexer::comments; -use crate::tokenstream::{self, DelimSpan, IsJoint::*, TokenStream, TreeAndJoint}; - -use errors::Diagnostic; -use rustc_data_structures::sync::Lrc; -use syntax_pos::{BytePos, FileName, MultiSpan, Pos, SourceFile, Span}; -use syntax_pos::symbol::{kw, sym, Symbol}; - -use proc_macro::{Delimiter, Level, LineColumn, Spacing}; -use proc_macro::bridge::{server, TokenTree}; -use std::{ascii, panic}; -use std::ops::Bound; - -trait FromInternal<T> { - fn from_internal(x: T) -> Self; -} - -trait ToInternal<T> { - fn to_internal(self) -> T; -} - -impl FromInternal<token::DelimToken> for Delimiter { - fn from_internal(delim: token::DelimToken) -> Delimiter { - match delim { - token::Paren => Delimiter::Parenthesis, - token::Brace => Delimiter::Brace, - token::Bracket => Delimiter::Bracket, - token::NoDelim => Delimiter::None, - } - } -} - -impl ToInternal<token::DelimToken> for Delimiter { - fn to_internal(self) -> token::DelimToken { - match self { - Delimiter::Parenthesis => token::Paren, - Delimiter::Brace => token::Brace, - Delimiter::Bracket => token::Bracket, - Delimiter::None => token::NoDelim, - } - } -} - -impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec<Self>)> - for TokenTree<Group, Punct, Ident, Literal> -{ - fn from_internal(((tree, is_joint), sess, stack): (TreeAndJoint, &ParseSess, &mut Vec<Self>)) - -> Self { - use crate::parse::token::*; - - let joint = is_joint == Joint; - let Token { kind, span } = match tree { - tokenstream::TokenTree::Delimited(span, delim, tts) => { - let delimiter = Delimiter::from_internal(delim); - return TokenTree::Group(Group { - delimiter, - stream: tts.into(), - span, - }); - } - tokenstream::TokenTree::Token(token) => token, - }; - - macro_rules! tt { - ($ty:ident { $($field:ident $(: $value:expr)*),+ $(,)? }) => ( - TokenTree::$ty(self::$ty { - $($field $(: $value)*,)+ - span, - }) - ); - ($ty:ident::$method:ident($($value:expr),*)) => ( - TokenTree::$ty(self::$ty::$method($($value,)* span)) - ); - } - macro_rules! op { - ($a:expr) => { - tt!(Punct::new($a, joint)) - }; - ($a:expr, $b:expr) => {{ - stack.push(tt!(Punct::new($b, joint))); - tt!(Punct::new($a, true)) - }}; - ($a:expr, $b:expr, $c:expr) => {{ - stack.push(tt!(Punct::new($c, joint))); - stack.push(tt!(Punct::new($b, true))); - tt!(Punct::new($a, true)) - }}; - } - - match kind { - Eq => op!('='), - Lt => op!('<'), - Le => op!('<', '='), - EqEq => op!('=', '='), - Ne => op!('!', '='), - Ge => op!('>', '='), - Gt => op!('>'), - AndAnd => op!('&', '&'), - OrOr => op!('|', '|'), - Not => op!('!'), - Tilde => op!('~'), - BinOp(Plus) => op!('+'), - BinOp(Minus) => op!('-'), - BinOp(Star) => op!('*'), - BinOp(Slash) => op!('/'), - BinOp(Percent) => op!('%'), - BinOp(Caret) => op!('^'), - BinOp(And) => op!('&'), - BinOp(Or) => op!('|'), - BinOp(Shl) => op!('<', '<'), - BinOp(Shr) => op!('>', '>'), - BinOpEq(Plus) => op!('+', '='), - BinOpEq(Minus) => op!('-', '='), - BinOpEq(Star) => op!('*', '='), - BinOpEq(Slash) => op!('/', '='), - BinOpEq(Percent) => op!('%', '='), - BinOpEq(Caret) => op!('^', '='), - BinOpEq(And) => op!('&', '='), - BinOpEq(Or) => op!('|', '='), - BinOpEq(Shl) => op!('<', '<', '='), - BinOpEq(Shr) => op!('>', '>', '='), - At => op!('@'), - Dot => op!('.'), - DotDot => op!('.', '.'), - DotDotDot => op!('.', '.', '.'), - DotDotEq => op!('.', '.', '='), - Comma => op!(','), - Semi => op!(';'), - Colon => op!(':'), - ModSep => op!(':', ':'), - RArrow => op!('-', '>'), - LArrow => op!('<', '-'), - FatArrow => op!('=', '>'), - Pound => op!('#'), - Dollar => op!('$'), - Question => op!('?'), - SingleQuote => op!('\''), - - Ident(name, false) if name == kw::DollarCrate => tt!(Ident::dollar_crate()), - Ident(name, is_raw) => tt!(Ident::new(name, is_raw)), - Lifetime(name) => { - let ident = ast::Ident::new(name, span).without_first_quote(); - stack.push(tt!(Ident::new(ident.name, false))); - tt!(Punct::new('\'', true)) - } - Literal(lit) => tt!(Literal { lit }), - DocComment(c) => { - let style = comments::doc_comment_style(&c.as_str()); - let stripped = comments::strip_doc_comment_decoration(&c.as_str()); - let mut escaped = String::new(); - for ch in stripped.chars() { - escaped.extend(ch.escape_debug()); - } - let stream = vec![ - Ident(sym::doc, false), - Eq, - TokenKind::lit(token::Str, Symbol::intern(&escaped), None), - ] - .into_iter() - .map(|kind| tokenstream::TokenTree::token(kind, span)) - .collect(); - stack.push(TokenTree::Group(Group { - delimiter: Delimiter::Bracket, - stream, - span: DelimSpan::from_single(span), - })); - if style == ast::AttrStyle::Inner { - stack.push(tt!(Punct::new('!', false))); - } - tt!(Punct::new('#', false)) - } - - Interpolated(nt) => { - let stream = nt.to_tokenstream(sess, span); - TokenTree::Group(Group { - delimiter: Delimiter::None, - stream, - span: DelimSpan::from_single(span), - }) - } - - OpenDelim(..) | CloseDelim(..) => unreachable!(), - Whitespace | Comment | Shebang(..) | Unknown(..) | Eof => unreachable!(), - } - } -} - -impl ToInternal<TokenStream> for TokenTree<Group, Punct, Ident, Literal> { - fn to_internal(self) -> TokenStream { - use crate::parse::token::*; - - let (ch, joint, span) = match self { - TokenTree::Punct(Punct { ch, joint, span }) => (ch, joint, span), - TokenTree::Group(Group { - delimiter, - stream, - span, - }) => { - return tokenstream::TokenTree::Delimited( - span, - delimiter.to_internal(), - stream.into(), - ) - .into(); - } - TokenTree::Ident(self::Ident { sym, is_raw, span }) => { - return tokenstream::TokenTree::token(Ident(sym, is_raw), span).into(); - } - TokenTree::Literal(self::Literal { - lit: token::Lit { kind: token::Integer, symbol, suffix }, - span, - }) if symbol.as_str().starts_with("-") => { - let minus = BinOp(BinOpToken::Minus); - let symbol = Symbol::intern(&symbol.as_str()[1..]); - let integer = TokenKind::lit(token::Integer, symbol, suffix); - let a = tokenstream::TokenTree::token(minus, span); - let b = tokenstream::TokenTree::token(integer, span); - return vec![a, b].into_iter().collect(); - } - TokenTree::Literal(self::Literal { - lit: token::Lit { kind: token::Float, symbol, suffix }, - span, - }) if symbol.as_str().starts_with("-") => { - let minus = BinOp(BinOpToken::Minus); - let symbol = Symbol::intern(&symbol.as_str()[1..]); - let float = TokenKind::lit(token::Float, symbol, suffix); - let a = tokenstream::TokenTree::token(minus, span); - let b = tokenstream::TokenTree::token(float, span); - return vec![a, b].into_iter().collect(); - } - TokenTree::Literal(self::Literal { lit, span }) => { - return tokenstream::TokenTree::token(Literal(lit), span).into() - } - }; - - let kind = match ch { - '=' => Eq, - '<' => Lt, - '>' => Gt, - '!' => Not, - '~' => Tilde, - '+' => BinOp(Plus), - '-' => BinOp(Minus), - '*' => BinOp(Star), - '/' => BinOp(Slash), - '%' => BinOp(Percent), - '^' => BinOp(Caret), - '&' => BinOp(And), - '|' => BinOp(Or), - '@' => At, - '.' => Dot, - ',' => Comma, - ';' => Semi, - ':' => Colon, - '#' => Pound, - '$' => Dollar, - '?' => Question, - '\'' => SingleQuote, - _ => unreachable!(), - }; - - let tree = tokenstream::TokenTree::token(kind, span); - TokenStream::new(vec![(tree, if joint { Joint } else { NonJoint })]) - } -} - -impl ToInternal<errors::Level> for Level { - fn to_internal(self) -> errors::Level { - match self { - Level::Error => errors::Level::Error, - Level::Warning => errors::Level::Warning, - Level::Note => errors::Level::Note, - Level::Help => errors::Level::Help, - _ => unreachable!("unknown proc_macro::Level variant: {:?}", self), - } - } -} - -#[derive(Clone)] -pub struct TokenStreamIter { - cursor: tokenstream::Cursor, - stack: Vec<TokenTree<Group, Punct, Ident, Literal>>, -} - -#[derive(Clone)] -pub struct Group { - delimiter: Delimiter, - stream: TokenStream, - span: DelimSpan, -} - -#[derive(Copy, Clone, PartialEq, Eq, Hash)] -pub struct Punct { - ch: char, - // NB. not using `Spacing` here because it doesn't implement `Hash`. - joint: bool, - span: Span, -} - -impl Punct { - fn new(ch: char, joint: bool, span: Span) -> Punct { - const LEGAL_CHARS: &[char] = &['=', '<', '>', '!', '~', '+', '-', '*', '/', '%', '^', - '&', '|', '@', '.', ',', ';', ':', '#', '$', '?', '\'']; - if !LEGAL_CHARS.contains(&ch) { - panic!("unsupported character `{:?}`", ch) - } - Punct { ch, joint, span } - } -} - -#[derive(Copy, Clone, PartialEq, Eq, Hash)] -pub struct Ident { - sym: Symbol, - is_raw: bool, - span: Span, -} - -impl Ident { - fn is_valid(string: &str) -> bool { - let mut chars = string.chars(); - if let Some(start) = chars.next() { - rustc_lexer::is_id_start(start) && chars.all(rustc_lexer::is_id_continue) - } else { - false - } - } - fn new(sym: Symbol, is_raw: bool, span: Span) -> Ident { - let string = sym.as_str(); - if !Self::is_valid(&string) { - panic!("`{:?}` is not a valid identifier", string) - } - // Get rid of gensyms to conservatively check rawness on the string contents only. - if is_raw && !sym.as_interned_str().as_symbol().can_be_raw() { - panic!("`{}` cannot be a raw identifier", string); - } - Ident { sym, is_raw, span } - } - fn dollar_crate(span: Span) -> Ident { - // `$crate` is accepted as an ident only if it comes from the compiler. - Ident { sym: kw::DollarCrate, is_raw: false, span } - } -} - -// FIXME(eddyb) `Literal` should not expose internal `Debug` impls. -#[derive(Clone, Debug)] -pub struct Literal { - lit: token::Lit, - span: Span, -} - -pub(crate) struct Rustc<'a> { - sess: &'a ParseSess, - def_site: Span, - call_site: Span, - mixed_site: Span, -} - -impl<'a> Rustc<'a> { - pub fn new(cx: &'a ExtCtxt<'_>) -> Self { - let expn_data = cx.current_expansion.id.expn_data(); - Rustc { - sess: cx.parse_sess, - def_site: cx.with_def_site_ctxt(expn_data.def_site), - call_site: cx.with_call_site_ctxt(expn_data.call_site), - mixed_site: cx.with_mixed_site_ctxt(expn_data.call_site), - } - } - - fn lit(&mut self, kind: token::LitKind, symbol: Symbol, suffix: Option<Symbol>) -> Literal { - Literal { - lit: token::Lit::new(kind, symbol, suffix), - span: server::Span::call_site(self), - } - } -} - -impl server::Types for Rustc<'_> { - type TokenStream = TokenStream; - type TokenStreamBuilder = tokenstream::TokenStreamBuilder; - type TokenStreamIter = TokenStreamIter; - type Group = Group; - type Punct = Punct; - type Ident = Ident; - type Literal = Literal; - type SourceFile = Lrc<SourceFile>; - type MultiSpan = Vec<Span>; - type Diagnostic = Diagnostic; - type Span = Span; -} - -impl server::TokenStream for Rustc<'_> { - fn new(&mut self) -> Self::TokenStream { - TokenStream::empty() - } - fn is_empty(&mut self, stream: &Self::TokenStream) -> bool { - stream.is_empty() - } - fn from_str(&mut self, src: &str) -> Self::TokenStream { - parse::parse_stream_from_source_str( - FileName::proc_macro_source_code(src), - src.to_string(), - self.sess, - Some(self.call_site), - ) - } - fn to_string(&mut self, stream: &Self::TokenStream) -> String { - stream.to_string() - } - fn from_token_tree( - &mut self, - tree: TokenTree<Self::Group, Self::Punct, Self::Ident, Self::Literal>, - ) -> Self::TokenStream { - tree.to_internal() - } - fn into_iter(&mut self, stream: Self::TokenStream) -> Self::TokenStreamIter { - TokenStreamIter { - cursor: stream.trees(), - stack: vec![], - } - } -} - -impl server::TokenStreamBuilder for Rustc<'_> { - fn new(&mut self) -> Self::TokenStreamBuilder { - tokenstream::TokenStreamBuilder::new() - } - fn push(&mut self, builder: &mut Self::TokenStreamBuilder, stream: Self::TokenStream) { - builder.push(stream); - } - fn build(&mut self, builder: Self::TokenStreamBuilder) -> Self::TokenStream { - builder.build() - } -} - -impl server::TokenStreamIter for Rustc<'_> { - fn next( - &mut self, - iter: &mut Self::TokenStreamIter, - ) -> Option<TokenTree<Self::Group, Self::Punct, Self::Ident, Self::Literal>> { - loop { - let tree = iter.stack.pop().or_else(|| { - let next = iter.cursor.next_with_joint()?; - Some(TokenTree::from_internal((next, self.sess, &mut iter.stack))) - })?; - // HACK: The condition "dummy span + group with empty delimiter" represents an AST - // fragment approximately converted into a token stream. This may happen, for - // example, with inputs to proc macro attributes, including derives. Such "groups" - // need to flattened during iteration over stream's token trees. - // Eventually this needs to be removed in favor of keeping original token trees - // and not doing the roundtrip through AST. - if let TokenTree::Group(ref group) = tree { - if group.delimiter == Delimiter::None && group.span.entire().is_dummy() { - iter.cursor.append(group.stream.clone()); - continue; - } - } - return Some(tree); - } - } -} - -impl server::Group for Rustc<'_> { - fn new(&mut self, delimiter: Delimiter, stream: Self::TokenStream) -> Self::Group { - Group { - delimiter, - stream, - span: DelimSpan::from_single(server::Span::call_site(self)), - } - } - fn delimiter(&mut self, group: &Self::Group) -> Delimiter { - group.delimiter - } - fn stream(&mut self, group: &Self::Group) -> Self::TokenStream { - group.stream.clone() - } - fn span(&mut self, group: &Self::Group) -> Self::Span { - group.span.entire() - } - fn span_open(&mut self, group: &Self::Group) -> Self::Span { - group.span.open - } - fn span_close(&mut self, group: &Self::Group) -> Self::Span { - group.span.close - } - fn set_span(&mut self, group: &mut Self::Group, span: Self::Span) { - group.span = DelimSpan::from_single(span); - } -} - -impl server::Punct for Rustc<'_> { - fn new(&mut self, ch: char, spacing: Spacing) -> Self::Punct { - Punct::new(ch, spacing == Spacing::Joint, server::Span::call_site(self)) - } - fn as_char(&mut self, punct: Self::Punct) -> char { - punct.ch - } - fn spacing(&mut self, punct: Self::Punct) -> Spacing { - if punct.joint { - Spacing::Joint - } else { - Spacing::Alone - } - } - fn span(&mut self, punct: Self::Punct) -> Self::Span { - punct.span - } - fn with_span(&mut self, punct: Self::Punct, span: Self::Span) -> Self::Punct { - Punct { span, ..punct } - } -} - -impl server::Ident for Rustc<'_> { - fn new(&mut self, string: &str, span: Self::Span, is_raw: bool) -> Self::Ident { - Ident::new(Symbol::intern(string), is_raw, span) - } - fn span(&mut self, ident: Self::Ident) -> Self::Span { - ident.span - } - fn with_span(&mut self, ident: Self::Ident, span: Self::Span) -> Self::Ident { - Ident { span, ..ident } - } -} - -impl server::Literal for Rustc<'_> { - // FIXME(eddyb) `Literal` should not expose internal `Debug` impls. - fn debug(&mut self, literal: &Self::Literal) -> String { - format!("{:?}", literal) - } - fn integer(&mut self, n: &str) -> Self::Literal { - self.lit(token::Integer, Symbol::intern(n), None) - } - fn typed_integer(&mut self, n: &str, kind: &str) -> Self::Literal { - self.lit(token::Integer, Symbol::intern(n), Some(Symbol::intern(kind))) - } - fn float(&mut self, n: &str) -> Self::Literal { - self.lit(token::Float, Symbol::intern(n), None) - } - fn f32(&mut self, n: &str) -> Self::Literal { - self.lit(token::Float, Symbol::intern(n), Some(sym::f32)) - } - fn f64(&mut self, n: &str) -> Self::Literal { - self.lit(token::Float, Symbol::intern(n), Some(sym::f64)) - } - fn string(&mut self, string: &str) -> Self::Literal { - let mut escaped = String::new(); - for ch in string.chars() { - escaped.extend(ch.escape_debug()); - } - self.lit(token::Str, Symbol::intern(&escaped), None) - } - fn character(&mut self, ch: char) -> Self::Literal { - let mut escaped = String::new(); - escaped.extend(ch.escape_unicode()); - self.lit(token::Char, Symbol::intern(&escaped), None) - } - fn byte_string(&mut self, bytes: &[u8]) -> Self::Literal { - let string = bytes - .iter() - .cloned() - .flat_map(ascii::escape_default) - .map(Into::<char>::into) - .collect::<String>(); - self.lit(token::ByteStr, Symbol::intern(&string), None) - } - fn span(&mut self, literal: &Self::Literal) -> Self::Span { - literal.span - } - fn set_span(&mut self, literal: &mut Self::Literal, span: Self::Span) { - literal.span = span; - } - fn subspan( - &mut self, - literal: &Self::Literal, - start: Bound<usize>, - end: Bound<usize>, - ) -> Option<Self::Span> { - let span = literal.span; - let length = span.hi().to_usize() - span.lo().to_usize(); - - let start = match start { - Bound::Included(lo) => lo, - Bound::Excluded(lo) => lo + 1, - Bound::Unbounded => 0, - }; - - let end = match end { - Bound::Included(hi) => hi + 1, - Bound::Excluded(hi) => hi, - Bound::Unbounded => length, - }; - - // Bounds check the values, preventing addition overflow and OOB spans. - if start > u32::max_value() as usize - || end > u32::max_value() as usize - || (u32::max_value() - start as u32) < span.lo().to_u32() - || (u32::max_value() - end as u32) < span.lo().to_u32() - || start >= end - || end > length - { - return None; - } - - let new_lo = span.lo() + BytePos::from_usize(start); - let new_hi = span.lo() + BytePos::from_usize(end); - Some(span.with_lo(new_lo).with_hi(new_hi)) - } -} - -impl server::SourceFile for Rustc<'_> { - fn eq(&mut self, file1: &Self::SourceFile, file2: &Self::SourceFile) -> bool { - Lrc::ptr_eq(file1, file2) - } - fn path(&mut self, file: &Self::SourceFile) -> String { - match file.name { - FileName::Real(ref path) => path - .to_str() - .expect("non-UTF8 file path in `proc_macro::SourceFile::path`") - .to_string(), - _ => file.name.to_string(), - } - } - fn is_real(&mut self, file: &Self::SourceFile) -> bool { - file.is_real_file() - } -} - -impl server::MultiSpan for Rustc<'_> { - fn new(&mut self) -> Self::MultiSpan { - vec![] - } - fn push(&mut self, spans: &mut Self::MultiSpan, span: Self::Span) { - spans.push(span) - } -} - -impl server::Diagnostic for Rustc<'_> { - fn new(&mut self, level: Level, msg: &str, spans: Self::MultiSpan) -> Self::Diagnostic { - let mut diag = Diagnostic::new(level.to_internal(), msg); - diag.set_span(MultiSpan::from_spans(spans)); - diag - } - fn sub( - &mut self, - diag: &mut Self::Diagnostic, - level: Level, - msg: &str, - spans: Self::MultiSpan, - ) { - diag.sub(level.to_internal(), msg, MultiSpan::from_spans(spans), None); - } - fn emit(&mut self, diag: Self::Diagnostic) { - self.sess.span_diagnostic.emit_diagnostic(&diag); - } -} - -impl server::Span for Rustc<'_> { - fn debug(&mut self, span: Self::Span) -> String { - format!("{:?} bytes({}..{})", span.ctxt(), span.lo().0, span.hi().0) - } - fn def_site(&mut self) -> Self::Span { - self.def_site - } - fn call_site(&mut self) -> Self::Span { - self.call_site - } - fn mixed_site(&mut self) -> Self::Span { - self.mixed_site - } - fn source_file(&mut self, span: Self::Span) -> Self::SourceFile { - self.sess.source_map().lookup_char_pos(span.lo()).file - } - fn parent(&mut self, span: Self::Span) -> Option<Self::Span> { - span.parent() - } - fn source(&mut self, span: Self::Span) -> Self::Span { - span.source_callsite() - } - fn start(&mut self, span: Self::Span) -> LineColumn { - let loc = self.sess.source_map().lookup_char_pos(span.lo()); - LineColumn { - line: loc.line, - column: loc.col.to_usize(), - } - } - fn end(&mut self, span: Self::Span) -> LineColumn { - let loc = self.sess.source_map().lookup_char_pos(span.hi()); - LineColumn { - line: loc.line, - column: loc.col.to_usize(), - } - } - fn join(&mut self, first: Self::Span, second: Self::Span) -> Option<Self::Span> { - let self_loc = self.sess.source_map().lookup_char_pos(first.lo()); - let other_loc = self.sess.source_map().lookup_char_pos(second.lo()); - - if self_loc.file.name != other_loc.file.name { - return None; - } - - Some(first.to(second)) - } - fn resolved_at(&mut self, span: Self::Span, at: Self::Span) -> Self::Span { - span.with_ctxt(at.ctxt()) - } - fn source_text(&mut self, span: Self::Span) -> Option<String> { - self.sess.source_map().span_to_snippet(span).ok() - } -} diff --git a/src/libsyntax/feature_gate/builtin_attrs.rs b/src/libsyntax/feature_gate/builtin_attrs.rs index ab1620b9524..7dd6ae90d9a 100644 --- a/src/libsyntax/feature_gate/builtin_attrs.rs +++ b/src/libsyntax/feature_gate/builtin_attrs.rs @@ -9,8 +9,8 @@ use super::active::Features; use crate::ast; use crate::attr::AttributeTemplate; +use crate::sess::ParseSess; use crate::symbol::{Symbol, sym}; -use crate::parse::ParseSess; use syntax_pos::Span; use rustc_data_structures::fx::FxHashMap; @@ -286,7 +286,10 @@ pub const BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[ ( sym::plugin_registrar, Normal, template!(Word), Gated( - Stability::Deprecated("https://github.com/rust-lang/rust/issues/29597", None), + Stability::Deprecated( + "https://github.com/rust-lang/rust/pull/64675", + Some("may be removed in a future compiler version"), + ), sym::plugin_registrar, "compiler plugins are deprecated", cfg_fn!(plugin_registrar) @@ -295,7 +298,10 @@ pub const BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[ ( sym::plugin, CrateLevel, template!(List: "name|name(args)"), Gated( - Stability::Deprecated("https://github.com/rust-lang/rust/issues/29597", None), + Stability::Deprecated( + "https://github.com/rust-lang/rust/pull/64675", + Some("may be removed in a future compiler version"), + ), sym::plugin, "compiler plugins are deprecated", cfg_fn!(plugin) diff --git a/src/libsyntax/feature_gate/check.rs b/src/libsyntax/feature_gate/check.rs index 6008f8f3005..172511f0f09 100644 --- a/src/libsyntax/feature_gate/check.rs +++ b/src/libsyntax/feature_gate/check.rs @@ -5,14 +5,14 @@ use super::builtin_attrs::{AttributeGate, BUILTIN_ATTRIBUTE_MAP}; use crate::ast::{ self, AssocTyConstraint, AssocTyConstraintKind, NodeId, GenericParam, GenericParamKind, - PatKind, RangeEnd, + PatKind, RangeEnd, VariantData, }; use crate::attr::{self, check_builtin_attribute}; use crate::source_map::Spanned; use crate::edition::{ALL_EDITIONS, Edition}; use crate::visit::{self, FnKind, Visitor}; -use crate::parse::{token, ParseSess}; -use crate::parse::parser::Parser; +use crate::parse::token; +use crate::sess::ParseSess; use crate::symbol::{Symbol, sym}; use crate::tokenstream::TokenTree; @@ -56,7 +56,7 @@ macro_rules! gate_feature { }; } -crate fn check_attribute(attr: &ast::Attribute, parse_sess: &ParseSess, features: &Features) { +pub fn check_attribute(attr: &ast::Attribute, parse_sess: &ParseSess, features: &Features) { PostExpansionVisitor { parse_sess, features }.visit_attribute(attr) } @@ -246,6 +246,51 @@ impl<'a> PostExpansionVisitor<'a> { Abi::System => {} } } + + fn maybe_report_invalid_custom_discriminants(&self, variants: &[ast::Variant]) { + let has_fields = variants.iter().any(|variant| match variant.data { + VariantData::Tuple(..) | VariantData::Struct(..) => true, + VariantData::Unit(..) => false, + }); + + let discriminant_spans = variants.iter().filter(|variant| match variant.data { + VariantData::Tuple(..) | VariantData::Struct(..) => false, + VariantData::Unit(..) => true, + }) + .filter_map(|variant| variant.disr_expr.as_ref().map(|c| c.value.span)) + .collect::<Vec<_>>(); + + if !discriminant_spans.is_empty() && has_fields { + let mut err = feature_err( + self.parse_sess, + sym::arbitrary_enum_discriminant, + discriminant_spans.clone(), + crate::feature_gate::GateIssue::Language, + "custom discriminant values are not allowed in enums with tuple or struct variants", + ); + for sp in discriminant_spans { + err.span_label(sp, "disallowed custom discriminant"); + } + for variant in variants.iter() { + match &variant.data { + VariantData::Struct(..) => { + err.span_label( + variant.span, + "struct variant defined here", + ); + } + VariantData::Tuple(..) => { + err.span_label( + variant.span, + "tuple variant defined here", + ); + } + VariantData::Unit(..) => {} + } + } + err.emit(); + } + } } impl<'a> Visitor<'a> for PostExpansionVisitor<'a> { @@ -353,7 +398,7 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> { let has_feature = self.features.arbitrary_enum_discriminant; if !has_feature && !i.span.allows_unstable(sym::arbitrary_enum_discriminant) { - Parser::maybe_report_invalid_custom_discriminants(self.parse_sess, &variants); + self.maybe_report_invalid_custom_discriminants(&variants); } } @@ -769,7 +814,7 @@ pub fn get_features(span_handler: &Handler, krate_attrs: &[ast::Attribute], } if let Some(allowed) = allow_features.as_ref() { - if allowed.iter().find(|f| *f == name.as_str()).is_none() { + if allowed.iter().find(|&f| f == &name.as_str() as &str).is_none() { span_err!(span_handler, mi.span(), E0725, "the feature `{}` is not in the list of allowed features", name); diff --git a/src/libsyntax/feature_gate/mod.rs b/src/libsyntax/feature_gate/mod.rs index ca13ab36205..ba970618c0e 100644 --- a/src/libsyntax/feature_gate/mod.rs +++ b/src/libsyntax/feature_gate/mod.rs @@ -58,8 +58,7 @@ pub use builtin_attrs::{ deprecated_attributes, is_builtin_attr, is_builtin_attr_name, }; pub use check::{ - check_crate, get_features, feature_err, emit_feature_err, + check_crate, check_attribute, get_features, feature_err, emit_feature_err, Stability, GateIssue, UnstableFeatures, EXPLAIN_STMT_ATTR_SYNTAX, EXPLAIN_UNSIZED_TUPLE_COERCION, }; -crate use check::check_attribute; diff --git a/src/libsyntax/json.rs b/src/libsyntax/json.rs index 2423e1070fc..e3296788d9f 100644 --- a/src/libsyntax/json.rs +++ b/src/libsyntax/json.rs @@ -12,7 +12,7 @@ use crate::source_map::{SourceMap, FilePathMapping}; use errors::registry::Registry; -use errors::{SubDiagnostic, CodeSuggestion, SourceMapper}; +use errors::{SubDiagnostic, CodeSuggestion, SourceMapper, SourceMapperDyn}; use errors::{DiagnosticId, Applicability}; use errors::emitter::{Emitter, HumanReadableErrorType}; @@ -89,8 +89,8 @@ impl JsonEmitter { } impl Emitter for JsonEmitter { - fn emit_diagnostic(&mut self, db: &errors::Diagnostic) { - let data = Diagnostic::from_errors_diagnostic(db, self); + fn emit_diagnostic(&mut self, diag: &errors::Diagnostic) { + let data = Diagnostic::from_errors_diagnostic(diag, self); let result = if self.pretty { writeln!(&mut self.dst, "{}", as_pretty_json(&data)) } else { @@ -113,6 +113,10 @@ impl Emitter for JsonEmitter { } } + fn source_map(&self) -> Option<&Lrc<SourceMapperDyn>> { + Some(&self.sm) + } + fn should_show_explain(&self) -> bool { match self.json_rendered { HumanReadableErrorType::Short(_) => false, @@ -205,10 +209,10 @@ struct ArtifactNotification<'a> { } impl Diagnostic { - fn from_errors_diagnostic(db: &errors::Diagnostic, + fn from_errors_diagnostic(diag: &errors::Diagnostic, je: &JsonEmitter) -> Diagnostic { - let sugg = db.suggestions.iter().map(|sugg| { + let sugg = diag.suggestions.iter().map(|sugg| { Diagnostic { message: sugg.msg.clone(), code: None, @@ -237,30 +241,30 @@ impl Diagnostic { let output = buf.clone(); je.json_rendered.new_emitter( Box::new(buf), Some(je.sm.clone()), false, None, je.external_macro_backtrace - ).ui_testing(je.ui_testing).emit_diagnostic(db); + ).ui_testing(je.ui_testing).emit_diagnostic(diag); let output = Arc::try_unwrap(output.0).unwrap().into_inner().unwrap(); let output = String::from_utf8(output).unwrap(); Diagnostic { - message: db.message(), - code: DiagnosticCode::map_opt_string(db.code.clone(), je), - level: db.level.to_str(), - spans: DiagnosticSpan::from_multispan(&db.span, je), - children: db.children.iter().map(|c| { + message: diag.message(), + code: DiagnosticCode::map_opt_string(diag.code.clone(), je), + level: diag.level.to_str(), + spans: DiagnosticSpan::from_multispan(&diag.span, je), + children: diag.children.iter().map(|c| { Diagnostic::from_sub_diagnostic(c, je) }).chain(sugg).collect(), rendered: Some(output), } } - fn from_sub_diagnostic(db: &SubDiagnostic, je: &JsonEmitter) -> Diagnostic { + fn from_sub_diagnostic(diag: &SubDiagnostic, je: &JsonEmitter) -> Diagnostic { Diagnostic { - message: db.message(), + message: diag.message(), code: None, - level: db.level.to_str(), - spans: db.render_span.as_ref() + level: diag.level.to_str(), + spans: diag.render_span.as_ref() .map(|sp| DiagnosticSpan::from_multispan(sp, je)) - .unwrap_or_else(|| DiagnosticSpan::from_multispan(&db.span, je)), + .unwrap_or_else(|| DiagnosticSpan::from_multispan(&diag.span, je)), children: vec![], rendered: None, } diff --git a/src/libsyntax/lib.rs b/src/libsyntax/lib.rs index 09a47795a82..3fa13f08d3a 100644 --- a/src/libsyntax/lib.rs +++ b/src/libsyntax/lib.rs @@ -13,17 +13,12 @@ #![feature(crate_visibility_modifier)] #![feature(label_break_value)] #![feature(nll)] -#![feature(proc_macro_diagnostic)] -#![feature(proc_macro_internals)] -#![feature(proc_macro_span)] #![feature(try_trait)] #![feature(slice_patterns)] #![feature(unicode_internals)] #![recursion_limit="256"] -extern crate proc_macro; - pub use errors; use rustc_data_structures::sync::Lock; use rustc_index::bit_set::GrowableBitSet; @@ -34,26 +29,7 @@ use syntax_pos::edition::Edition; #[cfg(test)] mod tests; -const MACRO_ARGUMENTS: Option<&'static str> = Some("macro arguments"); - -// A variant of 'try!' that panics on an Err. This is used as a crutch on the -// way towards a non-panic!-prone parser. It should be used for fatal parsing -// errors; eventually we plan to convert all code using panictry to just use -// normal try. -#[macro_export] -macro_rules! panictry { - ($e:expr) => ({ - use std::result::Result::{Ok, Err}; - use errors::FatalError; - match $e { - Ok(e) => e, - Err(mut e) => { - e.emit(); - FatalError.raise() - } - } - }) -} +pub const MACRO_ARGUMENTS: Option<&'static str> = Some("macro arguments"); // A variant of 'panictry!' that works on a Vec<Diagnostic> instead of a single DiagnosticBuilder. macro_rules! panictry_buffer { @@ -91,7 +67,7 @@ pub struct Globals { impl Globals { fn new(edition: Edition) -> Globals { Globals { - // We have no idea how many attributes their will be, so just + // We have no idea how many attributes there will be, so just // initiate the vectors with 0 bits. We'll grow them as necessary. used_attrs: Lock::new(GrowableBitSet::new_empty()), known_attrs: Lock::new(GrowableBitSet::new_empty()), @@ -147,6 +123,7 @@ pub mod ptr; pub mod show_span; pub use syntax_pos::edition; pub use syntax_pos::symbol; +pub mod sess; pub mod tokenstream; pub mod visit; @@ -156,19 +133,4 @@ pub mod print { mod helpers; } -pub mod ext { - mod placeholders; - mod proc_macro_server; - - pub use syntax_pos::hygiene; - pub use mbe::macro_rules::compile_declarative_macro; - pub mod allocator; - pub mod base; - pub mod build; - pub mod expand; - pub mod proc_macro; - - crate mod mbe; -} - pub mod early_buffered_lints; diff --git a/src/libsyntax/mut_visit.rs b/src/libsyntax/mut_visit.rs index 3923b9f297b..60ee17d09b7 100644 --- a/src/libsyntax/mut_visit.rs +++ b/src/libsyntax/mut_visit.rs @@ -610,10 +610,8 @@ pub fn noop_visit_tt<T: MutVisitor>(tt: &mut TokenTree, vis: &mut T) { } pub fn noop_visit_tts<T: MutVisitor>(TokenStream(tts): &mut TokenStream, vis: &mut T) { - visit_opt(tts, |tts| { - let tts = Lrc::make_mut(tts); - visit_vec(tts, |(tree, _is_joint)| vis.visit_tt(tree)); - }) + let tts = Lrc::make_mut(tts); + visit_vec(tts, |(tree, _is_joint)| vis.visit_tt(tree)); } // Applies ident visitor if it's an ident; applies other visits to interpolated nodes. diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs index ac3feadce3a..e6dc9a4c134 100644 --- a/src/libsyntax/parse/lexer/mod.rs +++ b/src/libsyntax/parse/lexer/mod.rs @@ -1,5 +1,5 @@ -use crate::parse::ParseSess; use crate::parse::token::{self, Token, TokenKind}; +use crate::sess::ParseSess; use crate::symbol::{sym, Symbol}; use crate::parse::unescape_error_reporting::{emit_unescape_error, push_escaped_char}; diff --git a/src/libsyntax/parse/lexer/tokentrees.rs b/src/libsyntax/parse/lexer/tokentrees.rs index e5ba7e45309..b4dd23c9f9b 100644 --- a/src/libsyntax/parse/lexer/tokentrees.rs +++ b/src/libsyntax/parse/lexer/tokentrees.rs @@ -1,3 +1,4 @@ +use rustc_data_structures::fx::FxHashMap; use syntax_pos::Span; use crate::print::pprust::token_to_string; @@ -16,6 +17,7 @@ impl<'a> StringReader<'a> { unmatched_braces: Vec::new(), matching_delim_spans: Vec::new(), last_unclosed_found_span: None, + last_delim_empty_block_spans: FxHashMap::default() }; let res = tt_reader.parse_all_token_trees(); (res, tt_reader.unmatched_braces) @@ -34,6 +36,7 @@ struct TokenTreesReader<'a> { /// Used only for error recovery when arriving to EOF with mismatched braces. matching_delim_spans: Vec<(token::DelimToken, Span, Span)>, last_unclosed_found_span: Option<Span>, + last_delim_empty_block_spans: FxHashMap<token::DelimToken, Span> } impl<'a> TokenTreesReader<'a> { @@ -121,13 +124,20 @@ impl<'a> TokenTreesReader<'a> { // Correct delimiter. token::CloseDelim(d) if d == delim => { let (open_brace, open_brace_span) = self.open_braces.pop().unwrap(); + let close_brace_span = self.token.span; + + if tts.is_empty() { + let empty_block_span = open_brace_span.to(close_brace_span); + self.last_delim_empty_block_spans.insert(delim, empty_block_span); + } + if self.open_braces.len() == 0 { // Clear up these spans to avoid suggesting them as we've found // properly matched delimiters so far for an entire block. self.matching_delim_spans.clear(); } else { self.matching_delim_spans.push( - (open_brace, open_brace_span, self.token.span), + (open_brace, open_brace_span, close_brace_span), ); } // Parse the close delimiter. @@ -193,13 +203,20 @@ impl<'a> TokenTreesReader<'a> { tts.into() ).into()) }, - token::CloseDelim(_) => { + token::CloseDelim(delim) => { // An unexpected closing delimiter (i.e., there is no // matching opening delimiter). let token_str = token_to_string(&self.token); let msg = format!("unexpected close delimiter: `{}`", token_str); let mut err = self.string_reader.sess.span_diagnostic .struct_span_err(self.token.span, &msg); + + if let Some(span) = self.last_delim_empty_block_spans.remove(&delim) { + err.span_label( + span, + "this block is empty, you might have not meant to close it" + ); + } err.span_label(self.token.span, "unexpected close delimiter"); Err(err) }, diff --git a/src/libsyntax/parse/literal.rs b/src/libsyntax/parse/literal.rs index 56a79bfe5d5..7952e293a53 100644 --- a/src/libsyntax/parse/literal.rs +++ b/src/libsyntax/parse/literal.rs @@ -1,14 +1,10 @@ //! Code related to parsing literals. use crate::ast::{self, Lit, LitKind}; -use crate::parse::parser::Parser; -use crate::parse::PResult; -use crate::parse::token::{self, Token, TokenKind}; -use crate::print::pprust; +use crate::parse::token::{self, Token}; use crate::symbol::{kw, sym, Symbol}; -use crate::tokenstream::{TokenStream, TokenTree}; +use crate::tokenstream::TokenTree; -use errors::{Applicability, Handler}; use log::debug; use rustc_data_structures::sync::Lrc; use syntax_pos::Span; @@ -28,72 +24,6 @@ crate enum LitError { IntTooLarge, } -impl LitError { - fn report(&self, diag: &Handler, lit: token::Lit, span: Span) { - let token::Lit { kind, suffix, .. } = lit; - match *self { - // `NotLiteral` is not an error by itself, so we don't report - // it and give the parser opportunity to try something else. - LitError::NotLiteral => {} - // `LexerError` *is* an error, but it was already reported - // by lexer, so here we don't report it the second time. - LitError::LexerError => {} - LitError::InvalidSuffix => { - expect_no_suffix( - diag, span, &format!("{} {} literal", kind.article(), kind.descr()), suffix - ); - } - LitError::InvalidIntSuffix => { - let suf = suffix.expect("suffix error with no suffix").as_str(); - if looks_like_width_suffix(&['i', 'u'], &suf) { - // If it looks like a width, try to be helpful. - let msg = format!("invalid width `{}` for integer literal", &suf[1..]); - diag.struct_span_err(span, &msg) - .help("valid widths are 8, 16, 32, 64 and 128") - .emit(); - } else { - let msg = format!("invalid suffix `{}` for integer literal", suf); - diag.struct_span_err(span, &msg) - .span_label(span, format!("invalid suffix `{}`", suf)) - .help("the suffix must be one of the integral types (`u32`, `isize`, etc)") - .emit(); - } - } - LitError::InvalidFloatSuffix => { - let suf = suffix.expect("suffix error with no suffix").as_str(); - if looks_like_width_suffix(&['f'], &suf) { - // If it looks like a width, try to be helpful. - let msg = format!("invalid width `{}` for float literal", &suf[1..]); - diag.struct_span_err(span, &msg) - .help("valid widths are 32 and 64") - .emit(); - } else { - let msg = format!("invalid suffix `{}` for float literal", suf); - diag.struct_span_err(span, &msg) - .span_label(span, format!("invalid suffix `{}`", suf)) - .help("valid suffixes are `f32` and `f64`") - .emit(); - } - } - LitError::NonDecimalFloat(base) => { - let descr = match base { - 16 => "hexadecimal", - 8 => "octal", - 2 => "binary", - _ => unreachable!(), - }; - diag.struct_span_err(span, &format!("{} float literal is not supported", descr)) - .span_label(span, "not supported") - .emit(); - } - LitError::IntTooLarge => { - diag.struct_span_err(span, "integer literal is too large") - .emit(); - } - } - } -} - impl LitKind { /// Converts literal token into a semantic literal. fn from_lit_token(lit: token::Lit) -> Result<LitKind, LitError> { @@ -204,7 +134,7 @@ impl LitKind { let (kind, symbol, suffix) = match *self { LitKind::Str(symbol, ast::StrStyle::Cooked) => { // Don't re-intern unless the escaped string is different. - let s = &symbol.as_str(); + let s: &str = &symbol.as_str(); let escaped = s.escape_default().to_string(); let symbol = if escaped == *s { symbol } else { Symbol::intern(&escaped) }; (token::Str, symbol, None) @@ -254,7 +184,7 @@ impl LitKind { impl Lit { /// Converts literal token into an AST literal. - fn from_lit_token(token: token::Lit, span: Span) -> Result<Lit, LitError> { + crate fn from_lit_token(token: token::Lit, span: Span) -> Result<Lit, LitError> { Ok(Lit { token, kind: LitKind::from_lit_token(token)?, span }) } @@ -286,109 +216,16 @@ impl Lit { Lit { token: kind.to_lit_token(), kind, span } } - /// Losslessly convert an AST literal into a token stream. - crate fn tokens(&self) -> TokenStream { + /// Losslessly convert an AST literal into a token tree. + crate fn token_tree(&self) -> TokenTree { let token = match self.token.kind { token::Bool => token::Ident(self.token.symbol, false), _ => token::Literal(self.token), }; - TokenTree::token(token, self.span).into() - } -} - -impl<'a> Parser<'a> { - /// Matches `lit = true | false | token_lit`. - crate fn parse_lit(&mut self) -> PResult<'a, Lit> { - let mut recovered = None; - if self.token == token::Dot { - // Attempt to recover `.4` as `0.4`. - recovered = self.look_ahead(1, |next_token| { - if let token::Literal(token::Lit { kind: token::Integer, symbol, suffix }) - = next_token.kind { - if self.token.span.hi() == next_token.span.lo() { - let s = String::from("0.") + &symbol.as_str(); - let kind = TokenKind::lit(token::Float, Symbol::intern(&s), suffix); - return Some(Token::new(kind, self.token.span.to(next_token.span))); - } - } - None - }); - if let Some(token) = &recovered { - self.bump(); - self.diagnostic() - .struct_span_err(token.span, "float literals must have an integer part") - .span_suggestion( - token.span, - "must have an integer part", - pprust::token_to_string(token), - Applicability::MachineApplicable, - ) - .emit(); - } - } - - let token = recovered.as_ref().unwrap_or(&self.token); - match Lit::from_token(token) { - Ok(lit) => { - self.bump(); - Ok(lit) - } - Err(LitError::NotLiteral) => { - let msg = format!("unexpected token: {}", self.this_token_descr()); - Err(self.span_fatal(token.span, &msg)) - } - Err(err) => { - let (lit, span) = (token.expect_lit(), token.span); - self.bump(); - err.report(&self.sess.span_diagnostic, lit, span); - // Pack possible quotes and prefixes from the original literal into - // the error literal's symbol so they can be pretty-printed faithfully. - let suffixless_lit = token::Lit::new(lit.kind, lit.symbol, None); - let symbol = Symbol::intern(&suffixless_lit.to_string()); - let lit = token::Lit::new(token::Err, symbol, lit.suffix); - Lit::from_lit_token(lit, span).map_err(|_| unreachable!()) - } - } - } -} - -crate fn expect_no_suffix(diag: &Handler, sp: Span, kind: &str, suffix: Option<Symbol>) { - if let Some(suf) = suffix { - let mut err = if kind == "a tuple index" && - [sym::i32, sym::u32, sym::isize, sym::usize].contains(&suf) { - // #59553: warn instead of reject out of hand to allow the fix to percolate - // through the ecosystem when people fix their macros - let mut err = diag.struct_span_warn( - sp, - &format!("suffixes on {} are invalid", kind), - ); - err.note(&format!( - "`{}` is *temporarily* accepted on tuple index fields as it was \ - incorrectly accepted on stable for a few releases", - suf, - )); - err.help( - "on proc macros, you'll want to use `syn::Index::from` or \ - `proc_macro::Literal::*_unsuffixed` for code that will desugar \ - to tuple field access", - ); - err.note( - "for more context, see https://github.com/rust-lang/rust/issues/60210", - ); - err - } else { - diag.struct_span_err(sp, &format!("suffixes on {} are invalid", kind)) - }; - err.span_label(sp, format!("invalid suffix `{}`", suf)); - err.emit(); + TokenTree::token(token, self.span) } } -// Checks if `s` looks like i32 or u1234 etc. -fn looks_like_width_suffix(first_chars: &[char], s: &str) -> bool { - s.len() > 1 && s.starts_with(first_chars) && s[1..].chars().all(|c| c.is_ascii_digit()) -} - fn strip_underscores(symbol: Symbol) -> Symbol { // Do not allocate a new string unless necessary. let s = symbol.as_str(); diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index 1518da23b09..e6b794a6a99 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -1,40 +1,33 @@ //! The main parser interface. -use crate::ast::{self, CrateConfig, NodeId}; -use crate::early_buffered_lints::{BufferedEarlyLint, BufferedEarlyLintId}; -use crate::source_map::{SourceMap, FilePathMapping}; -use crate::feature_gate::UnstableFeatures; -use crate::parse::parser::Parser; -use crate::parse::parser::emit_unclosed_delims; -use crate::parse::token::TokenKind; -use crate::tokenstream::{TokenStream, TokenTree}; +use crate::ast; +use crate::parse::parser::{Parser, emit_unclosed_delims}; +use crate::parse::token::Nonterminal; +use crate::tokenstream::{self, TokenStream, TokenTree}; use crate::print::pprust; -use crate::symbol::Symbol; +use crate::sess::ParseSess; -use errors::{Applicability, FatalError, Level, Handler, ColorConfig, Diagnostic, DiagnosticBuilder}; -use rustc_data_structures::fx::{FxHashSet, FxHashMap}; +use errors::{FatalError, Level, Diagnostic, DiagnosticBuilder}; #[cfg(target_arch = "x86_64")] use rustc_data_structures::static_assert_size; -use rustc_data_structures::sync::{Lrc, Lock, Once}; -use syntax_pos::{Span, SourceFile, FileName, MultiSpan}; -use syntax_pos::edition::Edition; -use syntax_pos::hygiene::ExpnId; +use rustc_data_structures::sync::Lrc; +use syntax_pos::{Span, SourceFile, FileName}; use std::borrow::Cow; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::str; +use log::info; + #[cfg(test)] mod tests; #[macro_use] pub mod parser; -pub mod attr; pub mod lexer; pub mod token; crate mod classify; -crate mod diagnostics; crate mod literal; crate mod unescape_error_reporting; @@ -45,112 +38,6 @@ pub type PResult<'a, T> = Result<T, DiagnosticBuilder<'a>>; #[cfg(target_arch = "x86_64")] static_assert_size!(PResult<'_, bool>, 16); -/// Collected spans during parsing for places where a certain feature was -/// used and should be feature gated accordingly in `check_crate`. -#[derive(Default)] -pub struct GatedSpans { - /// Spans collected for gating `let_chains`, e.g. `if a && let b = c {}`. - pub let_chains: Lock<Vec<Span>>, - /// Spans collected for gating `async_closure`, e.g. `async || ..`. - pub async_closure: Lock<Vec<Span>>, - /// Spans collected for gating `yield e?` expressions (`generators` gate). - pub yields: Lock<Vec<Span>>, - /// Spans collected for gating `or_patterns`, e.g. `Some(Foo | Bar)`. - pub or_patterns: Lock<Vec<Span>>, - /// Spans collected for gating `const_extern_fn`, e.g. `const extern fn foo`. - pub const_extern_fn: Lock<Vec<Span>>, -} - -/// Info about a parsing session. -pub struct ParseSess { - pub span_diagnostic: Handler, - pub unstable_features: UnstableFeatures, - pub config: CrateConfig, - pub edition: Edition, - pub missing_fragment_specifiers: Lock<FxHashSet<Span>>, - /// Places where raw identifiers were used. This is used for feature-gating raw identifiers. - pub raw_identifier_spans: Lock<Vec<Span>>, - /// Used to determine and report recursive module inclusions. - included_mod_stack: Lock<Vec<PathBuf>>, - source_map: Lrc<SourceMap>, - pub buffered_lints: Lock<Vec<BufferedEarlyLint>>, - /// Contains the spans of block expressions that could have been incomplete based on the - /// operation token that followed it, but that the parser cannot identify without further - /// analysis. - pub ambiguous_block_expr_parse: Lock<FxHashMap<Span, Span>>, - pub injected_crate_name: Once<Symbol>, - pub gated_spans: GatedSpans, -} - -impl ParseSess { - pub fn new(file_path_mapping: FilePathMapping) -> Self { - let cm = Lrc::new(SourceMap::new(file_path_mapping)); - let handler = Handler::with_tty_emitter( - ColorConfig::Auto, - true, - None, - Some(cm.clone()), - ); - ParseSess::with_span_handler(handler, cm) - } - - pub fn with_span_handler(handler: Handler, source_map: Lrc<SourceMap>) -> Self { - Self { - span_diagnostic: handler, - unstable_features: UnstableFeatures::from_environment(), - config: FxHashSet::default(), - edition: ExpnId::root().expn_data().edition, - missing_fragment_specifiers: Lock::new(FxHashSet::default()), - raw_identifier_spans: Lock::new(Vec::new()), - included_mod_stack: Lock::new(vec![]), - source_map, - buffered_lints: Lock::new(vec![]), - ambiguous_block_expr_parse: Lock::new(FxHashMap::default()), - injected_crate_name: Once::new(), - gated_spans: GatedSpans::default(), - } - } - - #[inline] - pub fn source_map(&self) -> &SourceMap { - &self.source_map - } - - pub fn buffer_lint<S: Into<MultiSpan>>(&self, - lint_id: BufferedEarlyLintId, - span: S, - id: NodeId, - msg: &str, - ) { - self.buffered_lints.with_lock(|buffered_lints| { - buffered_lints.push(BufferedEarlyLint{ - span: span.into(), - id, - msg: msg.into(), - lint_id, - }); - }); - } - - /// Extend an error with a suggestion to wrap an expression with parentheses to allow the - /// parser to continue parsing the following operation as part of the same expression. - pub fn expr_parentheses_needed( - &self, - err: &mut DiagnosticBuilder<'_>, - span: Span, - alt_snippet: Option<String>, - ) { - if let Some(snippet) = self.source_map().span_to_snippet(span).ok().or(alt_snippet) { - err.span_suggestion( - span, - "parentheses are required to parse this as an expression", - format!("({})", snippet), - Applicability::MachineApplicable, - ); - } - } -} - #[derive(Clone)] pub struct Directory<'a> { pub path: Cow<'a, Path>, @@ -384,26 +271,131 @@ pub fn stream_to_parser_with_base_dir<'a>( Parser::new(sess, stream, Some(base_dir), true, false, None) } -/// A sequence separator. -pub struct SeqSep { - /// The separator token. - pub sep: Option<TokenKind>, - /// `true` if a trailing separator is allowed. - pub trailing_sep_allowed: bool, +// NOTE(Centril): The following probably shouldn't be here but it acknowledges the +// fact that architecturally, we are using parsing (read on below to understand why). + +pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> TokenStream { + // A `Nonterminal` is often a parsed AST item. At this point we now + // need to convert the parsed AST to an actual token stream, e.g. + // un-parse it basically. + // + // Unfortunately there's not really a great way to do that in a + // guaranteed lossless fashion right now. The fallback here is to just + // stringify the AST node and reparse it, but this loses all span + // information. + // + // As a result, some AST nodes are annotated with the token stream they + // came from. Here we attempt to extract these lossless token streams + // before we fall back to the stringification. + let tokens = match *nt { + Nonterminal::NtItem(ref item) => { + prepend_attrs(sess, &item.attrs, item.tokens.as_ref(), span) + } + Nonterminal::NtTraitItem(ref item) => { + prepend_attrs(sess, &item.attrs, item.tokens.as_ref(), span) + } + Nonterminal::NtImplItem(ref item) => { + prepend_attrs(sess, &item.attrs, item.tokens.as_ref(), span) + } + Nonterminal::NtIdent(ident, is_raw) => { + Some(tokenstream::TokenTree::token(token::Ident(ident.name, is_raw), ident.span).into()) + } + Nonterminal::NtLifetime(ident) => { + Some(tokenstream::TokenTree::token(token::Lifetime(ident.name), ident.span).into()) + } + Nonterminal::NtTT(ref tt) => { + Some(tt.clone().into()) + } + _ => None, + }; + + // FIXME(#43081): Avoid this pretty-print + reparse hack + let source = pprust::nonterminal_to_string(nt); + let filename = FileName::macro_expansion_source_code(&source); + let tokens_for_real = parse_stream_from_source_str(filename, source, sess, Some(span)); + + // During early phases of the compiler the AST could get modified + // directly (e.g., attributes added or removed) and the internal cache + // of tokens my not be invalidated or updated. Consequently if the + // "lossless" token stream disagrees with our actual stringification + // (which has historically been much more battle-tested) then we go + // with the lossy stream anyway (losing span information). + // + // Note that the comparison isn't `==` here to avoid comparing spans, + // but it *also* is a "probable" equality which is a pretty weird + // definition. We mostly want to catch actual changes to the AST + // like a `#[cfg]` being processed or some weird `macro_rules!` + // expansion. + // + // What we *don't* want to catch is the fact that a user-defined + // literal like `0xf` is stringified as `15`, causing the cached token + // stream to not be literal `==` token-wise (ignoring spans) to the + // token stream we got from stringification. + // + // Instead the "probably equal" check here is "does each token + // recursively have the same discriminant?" We basically don't look at + // the token values here and assume that such fine grained token stream + // modifications, including adding/removing typically non-semantic + // tokens such as extra braces and commas, don't happen. + if let Some(tokens) = tokens { + if tokens.probably_equal_for_proc_macro(&tokens_for_real) { + return tokens + } + info!("cached tokens found, but they're not \"probably equal\", \ + going with stringified version"); + } + return tokens_for_real } -impl SeqSep { - pub fn trailing_allowed(t: TokenKind) -> SeqSep { - SeqSep { - sep: Some(t), - trailing_sep_allowed: true, - } +fn prepend_attrs( + sess: &ParseSess, + attrs: &[ast::Attribute], + tokens: Option<&tokenstream::TokenStream>, + span: syntax_pos::Span +) -> Option<tokenstream::TokenStream> { + let tokens = tokens?; + if attrs.len() == 0 { + return Some(tokens.clone()) } + let mut builder = tokenstream::TokenStreamBuilder::new(); + for attr in attrs { + assert_eq!(attr.style, ast::AttrStyle::Outer, + "inner attributes should prevent cached tokens from existing"); + + let source = pprust::attribute_to_string(attr); + let macro_filename = FileName::macro_expansion_source_code(&source); + if attr.is_sugared_doc { + let stream = parse_stream_from_source_str(macro_filename, source, sess, Some(span)); + builder.push(stream); + continue + } - pub fn none() -> SeqSep { - SeqSep { - sep: None, - trailing_sep_allowed: false, + // synthesize # [ $path $tokens ] manually here + let mut brackets = tokenstream::TokenStreamBuilder::new(); + + // For simple paths, push the identifier directly + if attr.path.segments.len() == 1 && attr.path.segments[0].args.is_none() { + let ident = attr.path.segments[0].ident; + let token = token::Ident(ident.name, ident.as_str().starts_with("r#")); + brackets.push(tokenstream::TokenTree::token(token, ident.span)); + + // ... and for more complicated paths, fall back to a reparse hack that + // should eventually be removed. + } else { + let stream = parse_stream_from_source_str(macro_filename, source, sess, Some(span)); + brackets.push(stream); } + + brackets.push(attr.tokens.clone()); + + // The span we list here for `#` and for `[ ... ]` are both wrong in + // that it encompasses more than each token, but it hopefully is "good + // enough" for now at least. + builder.push(tokenstream::TokenTree::token(token::Pound, attr.span)); + let delim_span = tokenstream::DelimSpan::from_single(attr.span); + builder.push(tokenstream::TokenTree::Delimited( + delim_span, token::DelimToken::Bracket, brackets.build().into())); } + builder.push(tokens.clone()); + Some(builder.build()) } diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index 4a457f5a43c..2ce0046ca27 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -1,28 +1,28 @@ +pub mod attr; mod expr; mod pat; mod item; -pub use item::AliasKind; mod module; -pub use module::{ModulePath, ModulePathSuccess}; mod ty; mod path; pub use path::PathStyle; mod stmt; mod generics; +mod diagnostics; +use diagnostics::Error; use crate::ast::{ - self, DUMMY_NODE_ID, AttrStyle, Attribute, BindingMode, CrateSugar, Ident, - IsAsync, MacDelimiter, Mutability, Param, StrStyle, SelfKind, TyKind, Visibility, - VisibilityKind, Unsafety, + self, DUMMY_NODE_ID, AttrStyle, Attribute, CrateSugar, Ident, + IsAsync, MacDelimiter, Mutability, StrStyle, Visibility, VisibilityKind, Unsafety, }; -use crate::parse::{ParseSess, PResult, Directory, DirectoryOwnership, SeqSep, literal, token}; -use crate::parse::diagnostics::{Error, dummy_arg}; +use crate::parse::{PResult, Directory, DirectoryOwnership}; use crate::parse::lexer::UnmatchedBrace; use crate::parse::lexer::comments::{doc_comment_style, strip_doc_comment_decoration}; -use crate::parse::token::{Token, TokenKind, DelimToken}; +use crate::parse::token::{self, Token, TokenKind, DelimToken}; use crate::print::pprust; use crate::ptr::P; -use crate::source_map::{self, respan}; +use crate::sess::ParseSess; +use crate::source_map::respan; use crate::symbol::{kw, sym, Symbol}; use crate::tokenstream::{self, DelimSpan, TokenTree, TokenStream, TreeAndJoint}; use crate::ThinVec; @@ -44,29 +44,18 @@ bitflags::bitflags! { } #[derive(Clone, Copy, PartialEq, Debug)] -crate enum SemiColonMode { +enum SemiColonMode { Break, Ignore, Comma, } #[derive(Clone, Copy, PartialEq, Debug)] -crate enum BlockMode { +enum BlockMode { Break, Ignore, } -/// The parsing configuration used to parse a parameter list (see `parse_fn_params`). -struct ParamCfg { - /// Is `self` is allowed as the first parameter? - is_self_allowed: bool, - /// Is `...` allowed as the tail of the parameter list? - allow_c_variadic: bool, - /// `is_name_required` decides if, per-parameter, - /// the parameter must have a pattern or just a type. - is_name_required: fn(&token::Token) -> bool, -} - /// Like `maybe_whole_expr`, but for things other than expressions. #[macro_export] macro_rules! maybe_whole { @@ -97,13 +86,6 @@ macro_rules! maybe_recover_from_interpolated_ty_qpath { } } -fn maybe_append(mut lhs: Vec<Attribute>, mut rhs: Option<Vec<Attribute>>) -> Vec<Attribute> { - if let Some(ref mut rhs) = rhs { - lhs.append(rhs); - } - lhs -} - #[derive(Debug, Clone, Copy, PartialEq)] enum PrevTokenKind { DocComment, @@ -135,33 +117,33 @@ pub struct Parser<'a> { prev_token_kind: PrevTokenKind, restrictions: Restrictions, /// Used to determine the path to externally loaded source files. - crate directory: Directory<'a>, + pub(super) directory: Directory<'a>, /// `true` to parse sub-modules in other files. - pub recurse_into_file_modules: bool, + pub(super) recurse_into_file_modules: bool, /// Name of the root module this parser originated from. If `None`, then the /// name is not known. This does not change while the parser is descending /// into modules, and sub-parsers have new values for this name. pub root_module_name: Option<String>, - crate expected_tokens: Vec<TokenType>, + expected_tokens: Vec<TokenType>, token_cursor: TokenCursor, desugar_doc_comments: bool, /// `true` we should configure out of line modules as we parse. - pub cfg_mods: bool, + cfg_mods: bool, /// This field is used to keep track of how many left angle brackets we have seen. This is /// required in order to detect extra leading left angle brackets (`<` characters) and error /// appropriately. /// /// See the comments in the `parse_path_segment` function for more details. - crate unmatched_angle_bracket_count: u32, - crate max_angle_bracket_count: u32, + unmatched_angle_bracket_count: u32, + max_angle_bracket_count: u32, /// A list of all unclosed delimiters found by the lexer. If an entry is used for error recovery /// it gets removed from here. Every entry left at the end gets emitted as an independent /// error. - crate unclosed_delims: Vec<UnmatchedBrace>, - crate last_unexpected_token_span: Option<Span>, - crate last_type_ascription: Option<(Span, bool /* likely path typo */)>, + pub(super) unclosed_delims: Vec<UnmatchedBrace>, + last_unexpected_token_span: Option<Span>, + pub last_type_ascription: Option<(Span, bool /* likely path typo */)>, /// If present, this `Parser` is not parsing Rust code but rather a macro call. - crate subparser_name: Option<&'static str>, + subparser_name: Option<&'static str>, } impl<'a> Drop for Parser<'a> { @@ -205,7 +187,7 @@ struct TokenCursorFrame { /// You can find some more example usage of this in the `collect_tokens` method /// on the parser. #[derive(Clone)] -crate enum LastToken { +enum LastToken { Collecting(Vec<TreeAndJoint>), Was(Option<TreeAndJoint>), } @@ -296,10 +278,10 @@ impl TokenCursor { token::NoDelim, &if doc_comment_style(&name.as_str()) == AttrStyle::Inner { [TokenTree::token(token::Pound, sp), TokenTree::token(token::Not, sp), body] - .iter().cloned().collect::<TokenStream>().into() + .iter().cloned().collect::<TokenStream>() } else { [TokenTree::token(token::Pound, sp), body] - .iter().cloned().collect::<TokenStream>().into() + .iter().cloned().collect::<TokenStream>() }, ))); @@ -308,7 +290,7 @@ impl TokenCursor { } #[derive(Clone, PartialEq)] -crate enum TokenType { +enum TokenType { Token(TokenKind), Keyword(Symbol), Operator, @@ -320,7 +302,7 @@ crate enum TokenType { } impl TokenType { - crate fn to_string(&self) -> String { + fn to_string(&self) -> String { match *self { TokenType::Token(ref t) => format!("`{}`", pprust::token_kind_to_string(t)), TokenType::Keyword(kw) => format!("`{}`", kw), @@ -335,11 +317,35 @@ impl TokenType { } #[derive(Copy, Clone, Debug)] -crate enum TokenExpectType { +enum TokenExpectType { Expect, NoExpect, } +/// A sequence separator. +struct SeqSep { + /// The separator token. + sep: Option<TokenKind>, + /// `true` if a trailing separator is allowed. + trailing_sep_allowed: bool, +} + +impl SeqSep { + fn trailing_allowed(t: TokenKind) -> SeqSep { + SeqSep { + sep: Some(t), + trailing_sep_allowed: true, + } + } + + fn none() -> SeqSep { + SeqSep { + sep: None, + trailing_sep_allowed: false, + } + } +} + impl<'a> Parser<'a> { pub fn new( sess: &'a ParseSess, @@ -416,7 +422,7 @@ impl<'a> Parser<'a> { pprust::token_to_string(&self.token) } - crate fn token_descr(&self) -> Option<&'static str> { + fn token_descr(&self) -> Option<&'static str> { Some(match &self.token.kind { _ if self.token.is_special_ident() => "reserved identifier", _ if self.token.is_used_keyword() => "keyword", @@ -426,7 +432,7 @@ impl<'a> Parser<'a> { }) } - crate fn this_token_descr(&self) -> String { + pub(super) fn this_token_descr(&self) -> String { if let Some(prefix) = self.token_descr() { format!("{} `{}`", prefix, self.this_token_to_string()) } else { @@ -476,7 +482,7 @@ impl<'a> Parser<'a> { } } - pub fn parse_ident(&mut self) -> PResult<'a, ast::Ident> { + fn parse_ident(&mut self) -> PResult<'a, ast::Ident> { self.parse_ident_common(true) } @@ -509,7 +515,7 @@ impl<'a> Parser<'a> { /// /// This method will automatically add `tok` to `expected_tokens` if `tok` is not /// encountered. - crate fn check(&mut self, tok: &TokenKind) -> bool { + fn check(&mut self, tok: &TokenKind) -> bool { let is_present = self.token == *tok; if !is_present { self.expected_tokens.push(TokenType::Token(tok.clone())); } is_present @@ -531,7 +537,7 @@ impl<'a> Parser<'a> { /// If the next token is the given keyword, eats it and returns `true`. /// Otherwise, returns `false`. An expectation is also added for diagnostics purposes. - pub fn eat_keyword(&mut self, kw: Symbol) -> bool { + fn eat_keyword(&mut self, kw: Symbol) -> bool { if self.check_keyword(kw) { self.bump(); true @@ -569,7 +575,7 @@ impl<'a> Parser<'a> { } } - crate fn check_ident(&mut self) -> bool { + fn check_ident(&mut self) -> bool { self.check_or_expected(self.token.is_ident(), TokenType::Ident) } @@ -649,10 +655,6 @@ impl<'a> Parser<'a> { } } - fn expect_no_suffix(&self, sp: Span, kind: &str, suffix: Option<ast::Name>) { - literal::expect_no_suffix(&self.sess.span_diagnostic, sp, kind, suffix) - } - /// Attempts to consume a `<`. If `<<` is seen, replaces it with a single /// `<` and continue. If `<-` is seen, replaces it with a single `<` /// and continue. If a `<` is not seen, returns false. @@ -738,7 +740,7 @@ impl<'a> Parser<'a> { /// Parses a sequence, including the closing delimiter. The function /// `f` must consume tokens until reaching the next separator or /// closing bracket. - pub fn parse_seq_to_end<T>( + fn parse_seq_to_end<T>( &mut self, ket: &TokenKind, sep: SeqSep, @@ -754,7 +756,7 @@ impl<'a> Parser<'a> { /// Parses a sequence, not including the closing delimiter. The function /// `f` must consume tokens until reaching the next separator or /// closing bracket. - pub fn parse_seq_to_before_end<T>( + fn parse_seq_to_before_end<T>( &mut self, ket: &TokenKind, sep: SeqSep, @@ -772,7 +774,7 @@ impl<'a> Parser<'a> { }) } - crate fn parse_seq_to_before_tokens<T>( + fn parse_seq_to_before_tokens<T>( &mut self, kets: &[&TokenKind], sep: SeqSep, @@ -1018,7 +1020,7 @@ impl<'a> Parser<'a> { } } - crate fn process_potential_macro_variable(&mut self) { + pub fn process_potential_macro_variable(&mut self) { self.token = match self.token.kind { token::Dollar if self.token.span.from_expansion() && self.look_ahead(1, |t| t.is_ident()) => { @@ -1052,7 +1054,7 @@ impl<'a> Parser<'a> { } /// Parses a single token tree from the input. - crate fn parse_token_tree(&mut self) -> TokenTree { + pub fn parse_token_tree(&mut self) -> TokenTree { match self.token.kind { token::OpenDelim(..) => { let frame = mem::replace(&mut self.token_cursor.frame, @@ -1105,271 +1107,6 @@ impl<'a> Parser<'a> { res } - /// Parses the parameter list of a function, including the `(` and `)` delimiters. - fn parse_fn_params(&mut self, mut cfg: ParamCfg) -> PResult<'a, Vec<Param>> { - let sp = self.token.span; - let is_trait_item = cfg.is_self_allowed; - let mut c_variadic = false; - // Parse the arguments, starting out with `self` being possibly allowed... - let (params, _) = self.parse_paren_comma_seq(|p| { - let param = p.parse_param_general(&cfg, is_trait_item); - // ...now that we've parsed the first argument, `self` is no longer allowed. - cfg.is_self_allowed = false; - - match param { - Ok(param) => Ok( - if let TyKind::CVarArgs = param.ty.kind { - c_variadic = true; - if p.token != token::CloseDelim(token::Paren) { - p.span_err( - p.token.span, - "`...` must be the last argument of a C-variadic function", - ); - // FIXME(eddyb) this should probably still push `CVarArgs`. - // Maybe AST validation/HIR lowering should emit the above error? - None - } else { - Some(param) - } - } else { - Some(param) - } - ), - Err(mut e) => { - e.emit(); - let lo = p.prev_span; - // Skip every token until next possible arg or end. - p.eat_to_tokens(&[&token::Comma, &token::CloseDelim(token::Paren)]); - // Create a placeholder argument for proper arg count (issue #34264). - let span = lo.to(p.prev_span); - Ok(Some(dummy_arg(Ident::new(kw::Invalid, span)))) - } - } - })?; - - let mut params: Vec<_> = params.into_iter().filter_map(|x| x).collect(); - - // Replace duplicated recovered params with `_` pattern to avoid unecessary errors. - self.deduplicate_recovered_params_names(&mut params); - - if c_variadic && params.len() <= 1 { - self.span_err( - sp, - "C-variadic function must be declared with at least one named argument", - ); - } - - Ok(params) - } - - /// Skips unexpected attributes and doc comments in this position and emits an appropriate - /// error. - /// This version of parse param doesn't necessarily require identifier names. - fn parse_param_general(&mut self, cfg: &ParamCfg, is_trait_item: bool) -> PResult<'a, Param> { - let lo = self.token.span; - let attrs = self.parse_outer_attributes()?; - - // Possibly parse `self`. Recover if we parsed it and it wasn't allowed here. - if let Some(mut param) = self.parse_self_param()? { - param.attrs = attrs.into(); - return if cfg.is_self_allowed { - Ok(param) - } else { - self.recover_bad_self_param(param, is_trait_item) - }; - } - - let is_name_required = match self.token.kind { - token::DotDotDot => false, - _ => (cfg.is_name_required)(&self.token), - }; - let (pat, ty) = if is_name_required || self.is_named_param() { - debug!("parse_param_general parse_pat (is_name_required:{})", is_name_required); - - let pat = self.parse_fn_param_pat()?; - if let Err(mut err) = self.expect(&token::Colon) { - return if let Some(ident) = self.parameter_without_type( - &mut err, - pat, - is_name_required, - cfg.is_self_allowed, - is_trait_item, - ) { - err.emit(); - Ok(dummy_arg(ident)) - } else { - Err(err) - }; - } - - self.eat_incorrect_doc_comment_for_param_type(); - (pat, self.parse_ty_common(true, true, cfg.allow_c_variadic)?) - } else { - debug!("parse_param_general ident_to_pat"); - let parser_snapshot_before_ty = self.clone(); - self.eat_incorrect_doc_comment_for_param_type(); - let mut ty = self.parse_ty_common(true, true, cfg.allow_c_variadic); - if ty.is_ok() && self.token != token::Comma && - self.token != token::CloseDelim(token::Paren) { - // This wasn't actually a type, but a pattern looking like a type, - // so we are going to rollback and re-parse for recovery. - ty = self.unexpected(); - } - match ty { - Ok(ty) => { - let ident = Ident::new(kw::Invalid, self.prev_span); - let bm = BindingMode::ByValue(Mutability::Immutable); - let pat = self.mk_pat_ident(ty.span, bm, ident); - (pat, ty) - } - // If this is a C-variadic argument and we hit an error, return the error. - Err(err) if self.token == token::DotDotDot => return Err(err), - // Recover from attempting to parse the argument as a type without pattern. - Err(mut err) => { - err.cancel(); - mem::replace(self, parser_snapshot_before_ty); - self.recover_arg_parse()? - } - } - }; - - let span = lo.to(self.token.span); - - Ok(Param { - attrs: attrs.into(), - id: ast::DUMMY_NODE_ID, - is_placeholder: false, - pat, - span, - ty, - }) - } - - /// Returns the parsed optional self parameter and whether a self shortcut was used. - /// - /// See `parse_self_param_with_attrs` to collect attributes. - fn parse_self_param(&mut self) -> PResult<'a, Option<Param>> { - // Extract an identifier *after* having confirmed that the token is one. - let expect_self_ident = |this: &mut Self| { - match this.token.kind { - // Preserve hygienic context. - token::Ident(name, _) => { - let span = this.token.span; - this.bump(); - Ident::new(name, span) - } - _ => unreachable!(), - } - }; - // Is `self` `n` tokens ahead? - let is_isolated_self = |this: &Self, n| { - this.is_keyword_ahead(n, &[kw::SelfLower]) - && this.look_ahead(n + 1, |t| t != &token::ModSep) - }; - // Is `mut self` `n` tokens ahead? - let is_isolated_mut_self = |this: &Self, n| { - this.is_keyword_ahead(n, &[kw::Mut]) - && is_isolated_self(this, n + 1) - }; - // Parse `self` or `self: TYPE`. We already know the current token is `self`. - let parse_self_possibly_typed = |this: &mut Self, m| { - let eself_ident = expect_self_ident(this); - let eself_hi = this.prev_span; - let eself = if this.eat(&token::Colon) { - SelfKind::Explicit(this.parse_ty()?, m) - } else { - SelfKind::Value(m) - }; - Ok((eself, eself_ident, eself_hi)) - }; - // Recover for the grammar `*self`, `*const self`, and `*mut self`. - let recover_self_ptr = |this: &mut Self| { - let msg = "cannot pass `self` by raw pointer"; - let span = this.token.span; - this.struct_span_err(span, msg) - .span_label(span, msg) - .emit(); - - Ok((SelfKind::Value(Mutability::Immutable), expect_self_ident(this), this.prev_span)) - }; - - // Parse optional `self` parameter of a method. - // Only a limited set of initial token sequences is considered `self` parameters; anything - // else is parsed as a normal function parameter list, so some lookahead is required. - let eself_lo = self.token.span; - let (eself, eself_ident, eself_hi) = match self.token.kind { - token::BinOp(token::And) => { - let eself = if is_isolated_self(self, 1) { - // `&self` - self.bump(); - SelfKind::Region(None, Mutability::Immutable) - } else if is_isolated_mut_self(self, 1) { - // `&mut self` - self.bump(); - self.bump(); - SelfKind::Region(None, Mutability::Mutable) - } else if self.look_ahead(1, |t| t.is_lifetime()) && is_isolated_self(self, 2) { - // `&'lt self` - self.bump(); - let lt = self.expect_lifetime(); - SelfKind::Region(Some(lt), Mutability::Immutable) - } else if self.look_ahead(1, |t| t.is_lifetime()) && is_isolated_mut_self(self, 2) { - // `&'lt mut self` - self.bump(); - let lt = self.expect_lifetime(); - self.bump(); - SelfKind::Region(Some(lt), Mutability::Mutable) - } else { - // `¬_self` - return Ok(None); - }; - (eself, expect_self_ident(self), self.prev_span) - } - // `*self` - token::BinOp(token::Star) if is_isolated_self(self, 1) => { - self.bump(); - recover_self_ptr(self)? - } - // `*mut self` and `*const self` - token::BinOp(token::Star) if - self.look_ahead(1, |t| t.is_mutability()) - && is_isolated_self(self, 2) => - { - self.bump(); - self.bump(); - recover_self_ptr(self)? - } - // `self` and `self: TYPE` - token::Ident(..) if is_isolated_self(self, 0) => { - parse_self_possibly_typed(self, Mutability::Immutable)? - } - // `mut self` and `mut self: TYPE` - token::Ident(..) if is_isolated_mut_self(self, 0) => { - self.bump(); - parse_self_possibly_typed(self, Mutability::Mutable)? - } - _ => return Ok(None), - }; - - let eself = source_map::respan(eself_lo.to(eself_hi), eself); - Ok(Some(Param::from_self(ThinVec::default(), eself, eself_ident))) - } - - fn is_named_param(&self) -> bool { - let offset = match self.token.kind { - token::Interpolated(ref nt) => match **nt { - token::NtPat(..) => return self.look_ahead(1, |t| t == &token::Colon), - _ => 0, - } - token::BinOp(token::And) | token::AndAnd => 1, - _ if self.token.is_keyword(kw::Mut) => 1, - _ => 0, - }; - - self.look_ahead(offset, |t| t.is_ident()) && - self.look_ahead(offset + 1, |t| t == &token::Colon) - } - fn is_crate_vis(&self) -> bool { self.token.is_keyword(kw::Crate) && self.look_ahead(1, |t| t != &token::ModSep) } @@ -1454,12 +1191,14 @@ impl<'a> Parser<'a> { `pub(super)`: visible only in the current module's parent `pub(in path::to::module)`: visible only on the specified path"##; + let path_str = pprust::path_to_string(&path); + struct_span_err!(self.sess.span_diagnostic, path.span, E0704, "{}", msg) .help(suggestion) .span_suggestion( path.span, - &format!("make this visible only to module `{}` with `in`", path), - format!("in {}", path), + &format!("make this visible only to module `{}` with `in`", path_str), + format!("in {}", path_str), Applicability::MachineApplicable, ) .emit(); @@ -1548,7 +1287,7 @@ impl<'a> Parser<'a> { // This can happen due to a bad interaction of two unrelated recovery mechanisms with // mismatched delimiters *and* recovery lookahead on the likely typo `pub ident(` // (#62881). - return Ok((ret?, TokenStream::new(vec![]))); + return Ok((ret?, TokenStream::default())); } else { &mut self.token_cursor.stack[prev].last_token }; @@ -1563,7 +1302,7 @@ impl<'a> Parser<'a> { // This can happen due to a bad interaction of two unrelated recovery mechanisms // with mismatched delimiters *and* recovery lookahead on the likely typo // `pub ident(` (#62895, different but similar to the case above). - return Ok((ret?, TokenStream::new(vec![]))); + return Ok((ret?, TokenStream::default())); } }; @@ -1601,7 +1340,7 @@ impl<'a> Parser<'a> { *t == token::BinOp(token::Star)) } - pub fn parse_optional_str(&mut self) -> Option<(Symbol, ast::StrStyle, Option<ast::Name>)> { + fn parse_optional_str(&mut self) -> Option<(Symbol, ast::StrStyle, Option<ast::Name>)> { let ret = match self.token.kind { token::Literal(token::Lit { kind: token::Str, symbol, suffix }) => (symbol, ast::StrStyle::Cooked, suffix), @@ -1641,7 +1380,7 @@ impl<'a> Parser<'a> { ], Applicability::MaybeIncorrect, ).span_suggestion( - self.sess.source_map.next_point(self.prev_span), + self.sess.source_map().next_point(self.prev_span), "add a semicolon", ';'.to_string(), Applicability::MaybeIncorrect, diff --git a/src/libsyntax/parse/attr.rs b/src/libsyntax/parse/parser/attr.rs index e74f3045db8..188a144cac9 100644 --- a/src/libsyntax/parse/attr.rs +++ b/src/libsyntax/parse/parser/attr.rs @@ -1,13 +1,11 @@ +use super::{SeqSep, PResult, Parser, TokenType, PathStyle}; use crate::attr; use crate::ast; -use crate::parse::{SeqSep, PResult}; use crate::parse::token::{self, Nonterminal, DelimToken}; -use crate::parse::parser::{Parser, TokenType, PathStyle}; use crate::tokenstream::{TokenStream, TokenTree}; use crate::source_map::Span; use log::debug; -use smallvec::smallvec; #[derive(Debug)] enum InnerAttributeParsePolicy<'a> { @@ -20,7 +18,7 @@ const DEFAULT_UNEXPECTED_INNER_ATTR_ERR_MSG: &str = "an inner attribute is not \ impl<'a> Parser<'a> { /// Parses attributes that appear before an item. - crate fn parse_outer_attributes(&mut self) -> PResult<'a, Vec<ast::Attribute>> { + pub(super) fn parse_outer_attributes(&mut self) -> PResult<'a, Vec<ast::Attribute>> { let mut attrs: Vec<ast::Attribute> = Vec::new(); let mut just_parsed_doc_comment = false; loop { @@ -84,9 +82,10 @@ impl<'a> Parser<'a> { /// The same as `parse_attribute`, except it takes in an `InnerAttributeParsePolicy` /// that prescribes how to handle inner attributes. - fn parse_attribute_with_inner_parse_policy(&mut self, - inner_parse_policy: InnerAttributeParsePolicy<'_>) - -> PResult<'a, ast::Attribute> { + fn parse_attribute_with_inner_parse_policy( + &mut self, + inner_parse_policy: InnerAttributeParsePolicy<'_> + ) -> PResult<'a, ast::Attribute> { debug!("parse_attribute_with_inner_parse_policy: inner_parse_policy={:?} self.token={:?}", inner_parse_policy, self.token); @@ -193,17 +192,17 @@ impl<'a> Parser<'a> { is_interpolated_expr = true; } } - let tokens = if is_interpolated_expr { + let token_tree = if is_interpolated_expr { // We need to accept arbitrary interpolated expressions to continue // supporting things like `doc = $expr` that work on stable. // Non-literal interpolated expressions are rejected after expansion. - self.parse_token_tree().into() + self.parse_token_tree() } else { - self.parse_unsuffixed_lit()?.tokens() + self.parse_unsuffixed_lit()?.token_tree() }; - TokenStream::from_streams(smallvec![eq.into(), tokens]) + TokenStream::new(vec![eq.into(), token_tree.into()]) } else { - TokenStream::empty() + TokenStream::default() }; ast::AttrItem { path, tokens } }) @@ -260,6 +259,27 @@ impl<'a> Parser<'a> { Ok(lit) } + /// Parses `cfg_attr(pred, attr_item_list)` where `attr_item_list` is comma-delimited. + crate fn parse_cfg_attr(&mut self) -> PResult<'a, (ast::MetaItem, Vec<(ast::AttrItem, Span)>)> { + self.expect(&token::OpenDelim(token::Paren))?; + + let cfg_predicate = self.parse_meta_item()?; + self.expect(&token::Comma)?; + + // Presumably, the majority of the time there will only be one attr. + let mut expanded_attrs = Vec::with_capacity(1); + + while !self.check(&token::CloseDelim(token::Paren)) { + let lo = self.token.span.lo(); + let item = self.parse_attr_item()?; + expanded_attrs.push((item, self.prev_span.with_lo(lo))); + self.expect_one_of(&[token::Comma], &[token::CloseDelim(token::Paren)])?; + } + + self.expect(&token::CloseDelim(token::Paren))?; + Ok((cfg_predicate, expanded_attrs)) + } + /// Matches the following grammar (per RFC 1559). /// /// meta_item : PATH ( '=' UNSUFFIXED_LIT | '(' meta_item_inner? ')' )? ; diff --git a/src/libsyntax/parse/diagnostics.rs b/src/libsyntax/parse/parser/diagnostics.rs index 42cbe28fc17..06982c789db 100644 --- a/src/libsyntax/parse/diagnostics.rs +++ b/src/libsyntax/parse/parser/diagnostics.rs @@ -1,10 +1,11 @@ +use super::{ + BlockMode, PathStyle, SemiColonMode, TokenType, TokenExpectType, + SeqSep, PResult, Parser +}; use crate::ast::{ self, Param, BinOpKind, BindingMode, BlockCheckMode, Expr, ExprKind, Ident, Item, ItemKind, - Mutability, Pat, PatKind, PathSegment, QSelf, Ty, TyKind, VariantData, + Mutability, Pat, PatKind, PathSegment, QSelf, Ty, TyKind, }; -use crate::feature_gate::feature_err; -use crate::parse::{SeqSep, PResult, Parser, ParseSess}; -use crate::parse::parser::{BlockMode, PathStyle, SemiColonMode, TokenType, TokenExpectType}; use crate::parse::token::{self, TokenKind}; use crate::print::pprust; use crate::ptr::P; @@ -18,8 +19,9 @@ use log::{debug, trace}; use std::mem; const TURBOFISH: &'static str = "use `::<...>` instead of `<...>` to specify type arguments"; + /// Creates a placeholder argument. -crate fn dummy_arg(ident: Ident) -> Param { +pub(super) fn dummy_arg(ident: Ident) -> Param { let pat = P(Pat { id: ast::DUMMY_NODE_ID, kind: PatKind::Ident(BindingMode::ByValue(Mutability::Immutable), ident, None), @@ -122,7 +124,7 @@ impl Error { } } -pub trait RecoverQPath: Sized + 'static { +pub(super) trait RecoverQPath: Sized + 'static { const PATH_STYLE: PathStyle = PathStyle::Expr; fn to_ty(&self) -> Option<P<Ty>>; fn recovered(qself: Option<QSelf>, path: ast::Path) -> Self; @@ -174,39 +176,43 @@ impl<'a> Parser<'a> { self.span_fatal(self.token.span, m) } - pub fn span_fatal<S: Into<MultiSpan>>(&self, sp: S, m: &str) -> DiagnosticBuilder<'a> { + crate fn span_fatal<S: Into<MultiSpan>>(&self, sp: S, m: &str) -> DiagnosticBuilder<'a> { self.sess.span_diagnostic.struct_span_fatal(sp, m) } - pub fn span_fatal_err<S: Into<MultiSpan>>(&self, sp: S, err: Error) -> DiagnosticBuilder<'a> { + pub(super) fn span_fatal_err<S: Into<MultiSpan>>( + &self, + sp: S, + err: Error, + ) -> DiagnosticBuilder<'a> { err.span_err(sp, self.diagnostic()) } - pub fn bug(&self, m: &str) -> ! { + pub(super) fn bug(&self, m: &str) -> ! { self.sess.span_diagnostic.span_bug(self.token.span, m) } - pub fn span_err<S: Into<MultiSpan>>(&self, sp: S, m: &str) { + pub(super) fn span_err<S: Into<MultiSpan>>(&self, sp: S, m: &str) { self.sess.span_diagnostic.span_err(sp, m) } - crate fn struct_span_err<S: Into<MultiSpan>>(&self, sp: S, m: &str) -> DiagnosticBuilder<'a> { + pub fn struct_span_err<S: Into<MultiSpan>>(&self, sp: S, m: &str) -> DiagnosticBuilder<'a> { self.sess.span_diagnostic.struct_span_err(sp, m) } - crate fn span_bug<S: Into<MultiSpan>>(&self, sp: S, m: &str) -> ! { + pub fn span_bug<S: Into<MultiSpan>>(&self, sp: S, m: &str) -> ! { self.sess.span_diagnostic.span_bug(sp, m) } - crate fn diagnostic(&self) -> &'a errors::Handler { + pub(super) fn diagnostic(&self) -> &'a errors::Handler { &self.sess.span_diagnostic } - crate fn span_to_snippet(&self, span: Span) -> Result<String, SpanSnippetError> { + pub(super) fn span_to_snippet(&self, span: Span) -> Result<String, SpanSnippetError> { self.sess.source_map().span_to_snippet(span) } - crate fn expected_ident_found(&self) -> DiagnosticBuilder<'a> { + pub(super) fn expected_ident_found(&self) -> DiagnosticBuilder<'a> { let mut err = self.struct_span_err( self.token.span, &format!("expected identifier, found {}", self.this_token_descr()), @@ -237,7 +243,7 @@ impl<'a> Parser<'a> { err } - pub fn expected_one_of_not_found( + pub(super) fn expected_one_of_not_found( &mut self, edible: &[TokenKind], inedible: &[TokenKind], @@ -424,7 +430,7 @@ impl<'a> Parser<'a> { /// Eats and discards tokens until one of `kets` is encountered. Respects token trees, /// passes through any errors encountered. Used for error recovery. - crate fn eat_to_tokens(&mut self, kets: &[&TokenKind]) { + pub(super) fn eat_to_tokens(&mut self, kets: &[&TokenKind]) { if let Err(ref mut err) = self.parse_seq_to_before_tokens( kets, SeqSep::none(), @@ -442,7 +448,7 @@ impl<'a> Parser<'a> { /// let _ = vec![1, 2, 3].into_iter().collect::<Vec<usize>>>>(); /// ^^ help: remove extra angle brackets /// ``` - crate fn check_trailing_angle_brackets(&mut self, segment: &PathSegment, end: TokenKind) { + pub(super) fn check_trailing_angle_brackets(&mut self, segment: &PathSegment, end: TokenKind) { // This function is intended to be invoked after parsing a path segment where there are two // cases: // @@ -561,7 +567,7 @@ impl<'a> Parser<'a> { /// inner_op r2 /// / \ /// l1 r1 - crate fn check_no_chained_comparison( + pub(super) fn check_no_chained_comparison( &mut self, lhs: &Expr, outer_op: &AssocOp, @@ -696,7 +702,7 @@ impl<'a> Parser<'a> { } } - crate fn maybe_report_ambiguous_plus( + pub(super) fn maybe_report_ambiguous_plus( &mut self, allow_plus: bool, impl_dyn_multi: bool, @@ -715,55 +721,7 @@ impl<'a> Parser<'a> { } } - crate fn maybe_report_invalid_custom_discriminants( - sess: &ParseSess, - variants: &[ast::Variant], - ) { - let has_fields = variants.iter().any(|variant| match variant.data { - VariantData::Tuple(..) | VariantData::Struct(..) => true, - VariantData::Unit(..) => false, - }); - - let discriminant_spans = variants.iter().filter(|variant| match variant.data { - VariantData::Tuple(..) | VariantData::Struct(..) => false, - VariantData::Unit(..) => true, - }) - .filter_map(|variant| variant.disr_expr.as_ref().map(|c| c.value.span)) - .collect::<Vec<_>>(); - - if !discriminant_spans.is_empty() && has_fields { - let mut err = feature_err( - sess, - sym::arbitrary_enum_discriminant, - discriminant_spans.clone(), - crate::feature_gate::GateIssue::Language, - "custom discriminant values are not allowed in enums with tuple or struct variants", - ); - for sp in discriminant_spans { - err.span_label(sp, "disallowed custom discriminant"); - } - for variant in variants.iter() { - match &variant.data { - VariantData::Struct(..) => { - err.span_label( - variant.span, - "struct variant defined here", - ); - } - VariantData::Tuple(..) => { - err.span_label( - variant.span, - "tuple variant defined here", - ); - } - VariantData::Unit(..) => {} - } - } - err.emit(); - } - } - - crate fn maybe_recover_from_bad_type_plus( + pub(super) fn maybe_recover_from_bad_type_plus( &mut self, allow_plus: bool, ty: &Ty, @@ -817,7 +775,7 @@ impl<'a> Parser<'a> { /// Tries to recover from associated item paths like `[T]::AssocItem` / `(T, U)::AssocItem`. /// Attempts to convert the base expression/pattern/type into a type, parses the `::AssocItem` /// tail, and combines them into a `<Ty>::AssocItem` expression/pattern/type. - crate fn maybe_recover_from_bad_qpath<T: RecoverQPath>( + pub(super) fn maybe_recover_from_bad_qpath<T: RecoverQPath>( &mut self, base: P<T>, allow_recovery: bool, @@ -833,7 +791,7 @@ impl<'a> Parser<'a> { /// Given an already parsed `Ty`, parses the `::AssocItem` tail and /// combines them into a `<Ty>::AssocItem` expression/pattern/type. - crate fn maybe_recover_from_bad_qpath_stage_2<T: RecoverQPath>( + pub(super) fn maybe_recover_from_bad_qpath_stage_2<T: RecoverQPath>( &mut self, ty_span: Span, ty: P<Ty>, @@ -856,7 +814,7 @@ impl<'a> Parser<'a> { // This is a best-effort recovery. path.span, "try", - format!("<{}>::{}", ty_str, path), + format!("<{}>::{}", ty_str, pprust::path_to_string(&path)), Applicability::MaybeIncorrect, ) .emit(); @@ -872,7 +830,7 @@ impl<'a> Parser<'a> { ))) } - crate fn maybe_consume_incorrect_semicolon(&mut self, items: &[P<Item>]) -> bool { + pub(super) fn maybe_consume_incorrect_semicolon(&mut self, items: &[P<Item>]) -> bool { if self.eat(&token::Semi) { let mut err = self.struct_span_err(self.prev_span, "expected item, found `;`"); err.span_suggestion_short( @@ -908,7 +866,7 @@ impl<'a> Parser<'a> { /// Creates a `DiagnosticBuilder` for an unexpected token `t` and tries to recover if it is a /// closing delimiter. - pub fn unexpected_try_recover( + pub(super) fn unexpected_try_recover( &mut self, t: &TokenKind, ) -> PResult<'a, bool /* recovered */> { @@ -958,7 +916,7 @@ impl<'a> Parser<'a> { Err(err) } - crate fn parse_semi_or_incorrect_foreign_fn_body( + pub(super) fn parse_semi_or_incorrect_foreign_fn_body( &mut self, ident: &Ident, extern_sp: Span, @@ -996,7 +954,7 @@ impl<'a> Parser<'a> { /// Consumes alternative await syntaxes like `await!(<expr>)`, `await <expr>`, /// `await? <expr>`, `await(<expr>)`, and `await { <expr> }`. - crate fn parse_incorrect_await_syntax( + pub(super) fn parse_incorrect_await_syntax( &mut self, lo: Span, await_sp: Span, @@ -1048,7 +1006,7 @@ impl<'a> Parser<'a> { } /// If encountering `future.await()`, consumes and emits an error. - crate fn recover_from_await_method_call(&mut self) { + pub(super) fn recover_from_await_method_call(&mut self) { if self.token == token::OpenDelim(token::Paren) && self.look_ahead(1, |t| t == &token::CloseDelim(token::Paren)) { @@ -1071,7 +1029,7 @@ impl<'a> Parser<'a> { /// and suggest writing `for $pat in $expr` instead. /// /// This should be called before parsing the `$block`. - crate fn recover_parens_around_for_head( + pub(super) fn recover_parens_around_for_head( &mut self, pat: P<Pat>, expr: &Expr, @@ -1109,7 +1067,7 @@ impl<'a> Parser<'a> { } } - crate fn could_ascription_be_path(&self, node: &ast::ExprKind) -> bool { + pub(super) fn could_ascription_be_path(&self, node: &ast::ExprKind) -> bool { self.token.is_ident() && if let ast::ExprKind::Path(..) = node { true } else { false } && !self.token.is_reserved_ident() && // v `foo:bar(baz)` @@ -1123,7 +1081,7 @@ impl<'a> Parser<'a> { self.look_ahead(2, |t| t == &token::Lt)) // `foo:bar::<baz>` } - crate fn recover_seq_parse_error( + pub(super) fn recover_seq_parse_error( &mut self, delim: token::DelimToken, lo: Span, @@ -1140,7 +1098,7 @@ impl<'a> Parser<'a> { } } - crate fn recover_closing_delimiter( + pub(super) fn recover_closing_delimiter( &mut self, tokens: &[TokenKind], mut err: DiagnosticBuilder<'a>, @@ -1191,7 +1149,7 @@ impl<'a> Parser<'a> { } /// Recovers from `pub` keyword in places where it seems _reasonable_ but isn't valid. - crate fn eat_bad_pub(&mut self) { + pub(super) fn eat_bad_pub(&mut self) { if self.token.is_keyword(kw::Pub) { match self.parse_visibility(false) { Ok(vis) => { @@ -1209,7 +1167,7 @@ impl<'a> Parser<'a> { /// statement. This is something of a best-effort heuristic. /// /// We terminate when we find an unmatched `}` (without consuming it). - crate fn recover_stmt(&mut self) { + pub(super) fn recover_stmt(&mut self) { self.recover_stmt_(SemiColonMode::Ignore, BlockMode::Ignore) } @@ -1220,7 +1178,11 @@ impl<'a> Parser<'a> { /// /// If `break_on_block` is `Break`, then we will stop consuming tokens /// after finding (and consuming) a brace-delimited block. - crate fn recover_stmt_(&mut self, break_on_semi: SemiColonMode, break_on_block: BlockMode) { + pub(super) fn recover_stmt_( + &mut self, + break_on_semi: SemiColonMode, + break_on_block: BlockMode, + ) { let mut brace_depth = 0; let mut bracket_depth = 0; let mut in_block = false; @@ -1288,7 +1250,7 @@ impl<'a> Parser<'a> { } } - crate fn check_for_for_in_in_typo(&mut self, in_span: Span) { + pub(super) fn check_for_for_in_in_typo(&mut self, in_span: Span) { if self.eat_keyword(kw::In) { // a common typo: `for _ in in bar {}` self.struct_span_err(self.prev_span, "expected iterable, found keyword `in`") @@ -1302,14 +1264,14 @@ impl<'a> Parser<'a> { } } - crate fn expected_semi_or_open_brace<T>(&mut self) -> PResult<'a, T> { + pub(super) fn expected_semi_or_open_brace<T>(&mut self) -> PResult<'a, T> { let token_str = self.this_token_descr(); let mut err = self.fatal(&format!("expected `;` or `{{`, found {}", token_str)); err.span_label(self.token.span, "expected `;` or `{`"); Err(err) } - crate fn eat_incorrect_doc_comment_for_param_type(&mut self) { + pub(super) fn eat_incorrect_doc_comment_for_param_type(&mut self) { if let token::DocComment(_) = self.token.kind { self.struct_span_err( self.token.span, @@ -1337,7 +1299,7 @@ impl<'a> Parser<'a> { } } - crate fn parameter_without_type( + pub(super) fn parameter_without_type( &mut self, err: &mut DiagnosticBuilder<'_>, pat: P<ast::Pat>, @@ -1400,7 +1362,7 @@ impl<'a> Parser<'a> { None } - crate fn recover_arg_parse(&mut self) -> PResult<'a, (P<ast::Pat>, P<ast::Ty>)> { + pub(super) fn recover_arg_parse(&mut self) -> PResult<'a, (P<ast::Pat>, P<ast::Ty>)> { let pat = self.parse_pat(Some("argument name"))?; self.expect(&token::Colon)?; let ty = self.parse_ty()?; @@ -1428,7 +1390,7 @@ impl<'a> Parser<'a> { Ok((pat, ty)) } - crate fn recover_bad_self_param( + pub(super) fn recover_bad_self_param( &mut self, mut param: ast::Param, is_trait_item: bool, @@ -1446,7 +1408,7 @@ impl<'a> Parser<'a> { Ok(param) } - crate fn consume_block(&mut self, delim: token::DelimToken) { + pub(super) fn consume_block(&mut self, delim: token::DelimToken) { let mut brace_depth = 0; loop { if self.eat(&token::OpenDelim(delim)) { @@ -1466,7 +1428,7 @@ impl<'a> Parser<'a> { } } - crate fn expected_expression_found(&self) -> DiagnosticBuilder<'a> { + pub(super) fn expected_expression_found(&self) -> DiagnosticBuilder<'a> { let (span, msg) = match (&self.token.kind, self.subparser_name) { (&token::Eof, Some(origin)) => { let sp = self.sess.source_map().next_point(self.token.span); @@ -1511,7 +1473,7 @@ impl<'a> Parser<'a> { /// the parameters are *names* (so we don't emit errors about not being able to find `b` in /// the local scope), but if we find the same name multiple times, like in `fn foo(i8, i8)`, /// we deduplicate them to not complain about duplicated parameter names. - crate fn deduplicate_recovered_params_names(&self, fn_inputs: &mut Vec<Param>) { + pub(super) fn deduplicate_recovered_params_names(&self, fn_inputs: &mut Vec<Param>) { let mut seen_inputs = FxHashSet::default(); for input in fn_inputs.iter_mut() { let opt_ident = if let (PatKind::Ident(_, ident, _), TyKind::Err) = ( diff --git a/src/libsyntax/parse/parser/expr.rs b/src/libsyntax/parse/parser/expr.rs index b459782d237..67a530ec683 100644 --- a/src/libsyntax/parse/parser/expr.rs +++ b/src/libsyntax/parse/parser/expr.rs @@ -1,18 +1,18 @@ -use super::{ - Parser, PResult, Restrictions, PrevTokenKind, TokenType, PathStyle, BlockMode, SemiColonMode, - SeqSep, TokenExpectType, -}; +use super::{Parser, PResult, Restrictions, PrevTokenKind, TokenType, PathStyle, BlockMode}; +use super::{SemiColonMode, SeqSep, TokenExpectType}; use super::pat::{GateOr, PARAM_EXPECTED}; +use super::diagnostics::Error; + +use crate::parse::literal::LitError; use crate::ast::{ self, DUMMY_NODE_ID, Attribute, AttrStyle, Ident, CaptureBy, BlockCheckMode, Expr, ExprKind, RangeLimits, Label, Movability, IsAsync, Arm, Ty, TyKind, - FunctionRetTy, Param, FnDecl, BinOpKind, BinOp, UnOp, Mac, AnonConst, Field, + FunctionRetTy, Param, FnDecl, BinOpKind, BinOp, UnOp, Mac, AnonConst, Field, Lit, }; use crate::maybe_recover_from_interpolated_ty_qpath; use crate::parse::classify; -use crate::parse::token::{self, Token}; -use crate::parse::diagnostics::Error; +use crate::parse::token::{self, Token, TokenKind}; use crate::print::pprust; use crate::ptr::P; use crate::source_map::{self, Span}; @@ -20,6 +20,7 @@ use crate::symbol::{kw, sym}; use crate::util::parser::{AssocOp, Fixity, prec_let_scrutinee_needs_par}; use errors::Applicability; +use syntax_pos::Symbol; use std::mem; use rustc_data_structures::thin_vec::ThinVec; @@ -422,7 +423,7 @@ impl<'a> Parser<'a> { self.struct_span_err(span_of_tilde, "`~` cannot be used as a unary operator") .span_suggestion_short( span_of_tilde, - "use `!` to perform bitwise negation", + "use `!` to perform bitwise not", "!".to_owned(), Applicability::MachineApplicable ) @@ -552,8 +553,11 @@ impl<'a> Parser<'a> { // Report non-fatal diagnostics, keep `x as usize` as an expression // in AST and continue parsing. - let msg = format!("`<` is interpreted as a start of generic \ - arguments for `{}`, not a {}", path, op_noun); + let msg = format!( + "`<` is interpreted as a start of generic arguments for `{}`, not a {}", + pprust::path_to_string(&path), + op_noun, + ); let span_after_type = parser_snapshot_after_type.token.span; let expr = mk_expr(self, P(Ty { span: path.span, @@ -1069,8 +1073,167 @@ impl<'a> Parser<'a> { self.maybe_recover_from_bad_qpath(expr, true) } + /// Matches `lit = true | false | token_lit`. + pub(super) fn parse_lit(&mut self) -> PResult<'a, Lit> { + let mut recovered = None; + if self.token == token::Dot { + // Attempt to recover `.4` as `0.4`. + recovered = self.look_ahead(1, |next_token| { + if let token::Literal(token::Lit { kind: token::Integer, symbol, suffix }) + = next_token.kind { + if self.token.span.hi() == next_token.span.lo() { + let s = String::from("0.") + &symbol.as_str(); + let kind = TokenKind::lit(token::Float, Symbol::intern(&s), suffix); + return Some(Token::new(kind, self.token.span.to(next_token.span))); + } + } + None + }); + if let Some(token) = &recovered { + self.bump(); + self.struct_span_err(token.span, "float literals must have an integer part") + .span_suggestion( + token.span, + "must have an integer part", + pprust::token_to_string(token), + Applicability::MachineApplicable, + ) + .emit(); + } + } + + let token = recovered.as_ref().unwrap_or(&self.token); + match Lit::from_token(token) { + Ok(lit) => { + self.bump(); + Ok(lit) + } + Err(LitError::NotLiteral) => { + let msg = format!("unexpected token: {}", self.this_token_descr()); + Err(self.span_fatal(token.span, &msg)) + } + Err(err) => { + let (lit, span) = (token.expect_lit(), token.span); + self.bump(); + self.error_literal_from_token(err, lit, span); + // Pack possible quotes and prefixes from the original literal into + // the error literal's symbol so they can be pretty-printed faithfully. + let suffixless_lit = token::Lit::new(lit.kind, lit.symbol, None); + let symbol = Symbol::intern(&suffixless_lit.to_string()); + let lit = token::Lit::new(token::Err, symbol, lit.suffix); + Lit::from_lit_token(lit, span).map_err(|_| unreachable!()) + } + } + } + + fn error_literal_from_token(&self, err: LitError, lit: token::Lit, span: Span) { + // Checks if `s` looks like i32 or u1234 etc. + fn looks_like_width_suffix(first_chars: &[char], s: &str) -> bool { + s.len() > 1 + && s.starts_with(first_chars) + && s[1..].chars().all(|c| c.is_ascii_digit()) + } + + let token::Lit { kind, suffix, .. } = lit; + match err { + // `NotLiteral` is not an error by itself, so we don't report + // it and give the parser opportunity to try something else. + LitError::NotLiteral => {} + // `LexerError` *is* an error, but it was already reported + // by lexer, so here we don't report it the second time. + LitError::LexerError => {} + LitError::InvalidSuffix => { + self.expect_no_suffix( + span, + &format!("{} {} literal", kind.article(), kind.descr()), + suffix, + ); + } + LitError::InvalidIntSuffix => { + let suf = suffix.expect("suffix error with no suffix").as_str(); + if looks_like_width_suffix(&['i', 'u'], &suf) { + // If it looks like a width, try to be helpful. + let msg = format!("invalid width `{}` for integer literal", &suf[1..]); + self.struct_span_err(span, &msg) + .help("valid widths are 8, 16, 32, 64 and 128") + .emit(); + } else { + let msg = format!("invalid suffix `{}` for integer literal", suf); + self.struct_span_err(span, &msg) + .span_label(span, format!("invalid suffix `{}`", suf)) + .help("the suffix must be one of the integral types (`u32`, `isize`, etc)") + .emit(); + } + } + LitError::InvalidFloatSuffix => { + let suf = suffix.expect("suffix error with no suffix").as_str(); + if looks_like_width_suffix(&['f'], &suf) { + // If it looks like a width, try to be helpful. + let msg = format!("invalid width `{}` for float literal", &suf[1..]); + self.struct_span_err(span, &msg) + .help("valid widths are 32 and 64") + .emit(); + } else { + let msg = format!("invalid suffix `{}` for float literal", suf); + self.struct_span_err(span, &msg) + .span_label(span, format!("invalid suffix `{}`", suf)) + .help("valid suffixes are `f32` and `f64`") + .emit(); + } + } + LitError::NonDecimalFloat(base) => { + let descr = match base { + 16 => "hexadecimal", + 8 => "octal", + 2 => "binary", + _ => unreachable!(), + }; + self.struct_span_err(span, &format!("{} float literal is not supported", descr)) + .span_label(span, "not supported") + .emit(); + } + LitError::IntTooLarge => { + self.struct_span_err(span, "integer literal is too large") + .emit(); + } + } + } + + pub(super) fn expect_no_suffix(&self, sp: Span, kind: &str, suffix: Option<Symbol>) { + if let Some(suf) = suffix { + let mut err = if kind == "a tuple index" + && [sym::i32, sym::u32, sym::isize, sym::usize].contains(&suf) + { + // #59553: warn instead of reject out of hand to allow the fix to percolate + // through the ecosystem when people fix their macros + let mut err = self.sess.span_diagnostic.struct_span_warn( + sp, + &format!("suffixes on {} are invalid", kind), + ); + err.note(&format!( + "`{}` is *temporarily* accepted on tuple index fields as it was \ + incorrectly accepted on stable for a few releases", + suf, + )); + err.help( + "on proc macros, you'll want to use `syn::Index::from` or \ + `proc_macro::Literal::*_unsuffixed` for code that will desugar \ + to tuple field access", + ); + err.note( + "for more context, see https://github.com/rust-lang/rust/issues/60210", + ); + err + } else { + self.struct_span_err(sp, &format!("suffixes on {} are invalid", kind)) + }; + err.span_label(sp, format!("invalid suffix `{}`", suf)); + err.emit(); + } + } + /// Matches `'-' lit | lit` (cf. `ast_validation::AstValidator::check_expr_within_pat`). - crate fn parse_literal_maybe_minus(&mut self) -> PResult<'a, P<Expr>> { + pub fn parse_literal_maybe_minus(&mut self) -> PResult<'a, P<Expr>> { maybe_whole_expr!(self); let minus_lo = self.token.span; @@ -1090,7 +1253,7 @@ impl<'a> Parser<'a> { } /// Parses a block or unsafe block. - crate fn parse_block_expr( + pub(super) fn parse_block_expr( &mut self, opt_label: Option<Label>, lo: Span, @@ -1395,7 +1558,7 @@ impl<'a> Parser<'a> { return Ok(self.mk_expr(lo.to(hi), ExprKind::Match(discriminant, arms), attrs)); } - crate fn parse_arm(&mut self) -> PResult<'a, Arm> { + pub(super) fn parse_arm(&mut self) -> PResult<'a, Arm> { let attrs = self.parse_outer_attributes()?; let lo = self.token.span; let pat = self.parse_top_pat(GateOr::No)?; @@ -1503,7 +1666,7 @@ impl<'a> Parser<'a> { } /// Parses an `async move? {...}` expression. - pub fn parse_async_block(&mut self, mut attrs: ThinVec<Attribute>) -> PResult<'a, P<Expr>> { + fn parse_async_block(&mut self, mut attrs: ThinVec<Attribute>) -> PResult<'a, P<Expr>> { let span_lo = self.token.span; self.expect_keyword(kw::Async)?; let capture_clause = self.parse_capture_clause(); @@ -1783,4 +1946,8 @@ impl<'a> Parser<'a> { crate fn mk_expr(&self, span: Span, kind: ExprKind, attrs: ThinVec<Attribute>) -> P<Expr> { P(Expr { kind, span, attrs, id: DUMMY_NODE_ID }) } + + pub(super) fn mk_expr_err(&self, span: Span) -> P<Expr> { + self.mk_expr(span, ExprKind::Err, ThinVec::new()) + } } diff --git a/src/libsyntax/parse/parser/generics.rs b/src/libsyntax/parse/parser/generics.rs index 2ecd9cca3c6..bfcb0042a75 100644 --- a/src/libsyntax/parse/parser/generics.rs +++ b/src/libsyntax/parse/parser/generics.rs @@ -74,7 +74,7 @@ impl<'a> Parser<'a> { /// Parses a (possibly empty) list of lifetime and type parameters, possibly including /// a trailing comma and erroneous trailing attributes. - crate fn parse_generic_params(&mut self) -> PResult<'a, Vec<ast::GenericParam>> { + pub(super) fn parse_generic_params(&mut self) -> PResult<'a, Vec<ast::GenericParam>> { let mut params = Vec::new(); loop { let attrs = self.parse_outer_attributes()?; diff --git a/src/libsyntax/parse/parser/item.rs b/src/libsyntax/parse/parser/item.rs index 3c60c88e2aa..73bd80e2a21 100644 --- a/src/libsyntax/parse/parser/item.rs +++ b/src/libsyntax/parse/parser/item.rs @@ -1,34 +1,28 @@ -use super::{Parser, PResult, PathStyle, SemiColonMode, BlockMode, ParamCfg}; +use super::{Parser, PResult, PathStyle, SemiColonMode, BlockMode}; +use super::diagnostics::{Error, dummy_arg}; use crate::maybe_whole; use crate::ptr::P; -use crate::ast::{ - self, DUMMY_NODE_ID, Ident, Attribute, AttrStyle, - Item, ItemKind, ImplItem, ImplItemKind, TraitItem, TraitItemKind, - UseTree, UseTreeKind, PathSegment, - IsAuto, Constness, IsAsync, Unsafety, Defaultness, - Visibility, VisibilityKind, Mutability, FnDecl, FnHeader, MethodSig, Block, - ForeignItem, ForeignItemKind, - Ty, TyKind, Generics, GenericBounds, TraitRef, - EnumDef, VariantData, StructField, AnonConst, - Mac, MacDelimiter, -}; -use crate::ext::base::DummyResult; +use crate::ast::{self, DUMMY_NODE_ID, Ident, Attribute, AttrStyle, AnonConst, Item, ItemKind}; +use crate::ast::{ImplItem, ImplItemKind, TraitItem, TraitItemKind, UseTree, UseTreeKind}; +use crate::ast::{PathSegment, IsAuto, Constness, IsAsync, Unsafety, Defaultness}; +use crate::ast::{Visibility, VisibilityKind, Mutability, FnHeader, ForeignItem, ForeignItemKind}; +use crate::ast::{Ty, TyKind, Generics, GenericBounds, TraitRef, EnumDef, VariantData, StructField}; +use crate::ast::{Mac, MacDelimiter, Block, BindingMode, FnDecl, MethodSig, SelfKind, Param}; use crate::parse::token; -use crate::parse::parser::maybe_append; -use crate::parse::diagnostics::Error; use crate::tokenstream::{TokenTree, TokenStream}; -use crate::source_map::{respan, Span}; use crate::symbol::{kw, sym}; +use crate::source_map::{self, respan, Span}; +use crate::ThinVec; -use std::mem; use log::debug; +use std::mem; use rustc_target::spec::abi::Abi; use errors::{Applicability, DiagnosticBuilder, DiagnosticId, StashKey}; /// Whether the type alias or associated type is a concrete type or an opaque type. #[derive(Debug)] -pub enum AliasKind { +pub(super) enum AliasKind { /// Just a new name for the same type. Weak(P<Ty>), /// Only trait impls of the type will be usable, not the actual type itself. @@ -412,7 +406,7 @@ impl<'a> Parser<'a> { self.parse_macro_use_or_failure(attrs, macros_allowed, attributes_allowed, lo, vis) } - fn mk_item_with_info( + pub(super) fn mk_item_with_info( &self, attrs: Vec<Attribute>, lo: Span, @@ -421,18 +415,15 @@ impl<'a> Parser<'a> { ) -> PResult<'a, Option<P<Item>>> { let (ident, item, extra_attrs) = info; let span = lo.to(self.prev_span); - let attrs = maybe_append(attrs, extra_attrs); + let attrs = Self::maybe_append(attrs, extra_attrs); Ok(Some(self.mk_item(span, ident, item, vis, attrs))) } - fn recover_first_param(&mut self) -> &'static str { - match self.parse_outer_attributes() - .and_then(|_| self.parse_self_param()) - .map_err(|mut e| e.cancel()) - { - Ok(Some(_)) => "method", - _ => "function", + fn maybe_append<T>(mut lhs: Vec<T>, mut rhs: Option<Vec<T>>) -> Vec<T> { + if let Some(ref mut rhs) = rhs { + lhs.append(rhs); } + lhs } /// This is the fall-through for parsing items. @@ -620,7 +611,7 @@ impl<'a> Parser<'a> { let ty_second = if self.token == token::DotDot { // We need to report this error after `cfg` expansion for compatibility reasons self.bump(); // `..`, do not add it to expected tokens - Some(DummyResult::raw_ty(self.prev_span, true)) + Some(self.mk_ty(self.prev_span, TyKind::Err)) } else if has_for || self.token.can_begin_type() { Some(self.parse_ty()?) } else { @@ -707,9 +698,11 @@ impl<'a> Parser<'a> { Ok(item) } - fn parse_impl_item_(&mut self, - at_end: &mut bool, - mut attrs: Vec<Attribute>) -> PResult<'a, ImplItem> { + fn parse_impl_item_( + &mut self, + at_end: &mut bool, + mut attrs: Vec<Attribute>, + ) -> PResult<'a, ImplItem> { let lo = self.token.span; let vis = self.parse_visibility(false)?; let defaultness = self.parse_defaultness(); @@ -722,8 +715,11 @@ impl<'a> Parser<'a> { (name, kind, generics) } else if self.is_const_item() { self.parse_impl_const()? + } else if let Some(mac) = self.parse_assoc_macro_invoc("impl", Some(&vis), at_end)? { + // FIXME: code copied from `parse_macro_use_or_failure` -- use abstraction! + (Ident::invalid(), ast::ImplItemKind::Macro(mac), Generics::default()) } else { - let (name, inner_attrs, generics, kind) = self.parse_impl_method(&vis, at_end)?; + let (name, inner_attrs, generics, kind) = self.parse_impl_method(at_end)?; attrs.extend(inner_attrs); (name, kind, generics) }; @@ -783,71 +779,6 @@ impl<'a> Parser<'a> { Ok((name, ImplItemKind::Const(typ, expr), Generics::default())) } - /// Parses a method or a macro invocation in a trait impl. - fn parse_impl_method( - &mut self, - vis: &Visibility, - at_end: &mut bool - ) -> PResult<'a, (Ident, Vec<Attribute>, Generics, ImplItemKind)> { - // FIXME: code copied from `parse_macro_use_or_failure` -- use abstraction! - if let Some(mac) = self.parse_assoc_macro_invoc("impl", Some(vis), at_end)? { - // method macro - Ok((Ident::invalid(), vec![], Generics::default(), ast::ImplItemKind::Macro(mac))) - } else { - let (ident, sig, generics) = self.parse_method_sig(|_| true)?; - *at_end = true; - let (inner_attrs, body) = self.parse_inner_attrs_and_block()?; - Ok((ident, inner_attrs, generics, ast::ImplItemKind::Method(sig, body))) - } - } - - /// Parse the "signature", including the identifier, parameters, and generics - /// of a method. The body is not parsed as that differs between `trait`s and `impl`s. - fn parse_method_sig( - &mut self, - is_name_required: fn(&token::Token) -> bool, - ) -> PResult<'a, (Ident, MethodSig, Generics)> { - let header = self.parse_fn_front_matter()?; - let (ident, decl, generics) = self.parse_fn_sig(ParamCfg { - is_self_allowed: true, - allow_c_variadic: false, - is_name_required, - })?; - Ok((ident, MethodSig { header, decl }, generics)) - } - - /// Parses all the "front matter" for a `fn` declaration, up to - /// and including the `fn` keyword: - /// - /// - `const fn` - /// - `unsafe fn` - /// - `const unsafe fn` - /// - `extern fn` - /// - etc. - fn parse_fn_front_matter(&mut self) -> PResult<'a, FnHeader> { - let is_const_fn = self.eat_keyword(kw::Const); - let const_span = self.prev_span; - let asyncness = self.parse_asyncness(); - if let IsAsync::Async { .. } = asyncness { - self.ban_async_in_2015(self.prev_span); - } - let asyncness = respan(self.prev_span, asyncness); - let unsafety = self.parse_unsafety(); - let (constness, unsafety, abi) = if is_const_fn { - (respan(const_span, Constness::Const), unsafety, Abi::Rust) - } else { - let abi = self.parse_extern_abi()?; - (respan(self.prev_span, Constness::NotConst), unsafety, abi) - }; - if !self.eat_keyword(kw::Fn) { - // It is possible for `expect_one_of` to recover given the contents of - // `self.expected_tokens`, therefore, do not use `self.unexpected()` which doesn't - // account for this. - if !self.expect_one_of(&[], &[])? { unreachable!() } - } - Ok(FnHeader { constness, unsafety, asyncness, abi }) - } - /// Parses `auto? trait Foo { ... }` or `trait Foo = Bar;`. fn parse_item_trait(&mut self, unsafety: Unsafety) -> PResult<'a, ItemInfo> { // Parse optional `auto` prefix. @@ -957,13 +888,7 @@ impl<'a> Parser<'a> { // trait item macro. (Ident::invalid(), TraitItemKind::Macro(mac), Generics::default()) } else { - // This is somewhat dubious; We don't want to allow - // argument names to be left off if there is a definition... - // - // We don't allow argument names to be left off in edition 2018. - let (ident, sig, generics) = self.parse_method_sig(|t| t.span.rust_2018())?; - let body = self.parse_trait_method_body(at_end, &mut attrs)?; - (ident, TraitItemKind::Method(sig, body), generics) + self.parse_trait_item_method(at_end, &mut attrs)? }; Ok(TraitItem { @@ -991,43 +916,6 @@ impl<'a> Parser<'a> { Ok((ident, TraitItemKind::Const(ty, default), Generics::default())) } - /// Parse the "body" of a method in a trait item definition. - /// This can either be `;` when there's no body, - /// or e.g. a block when the method is a provided one. - fn parse_trait_method_body( - &mut self, - at_end: &mut bool, - attrs: &mut Vec<Attribute>, - ) -> PResult<'a, Option<P<Block>>> { - Ok(match self.token.kind { - token::Semi => { - debug!("parse_trait_method_body(): parsing required method"); - self.bump(); - *at_end = true; - None - } - token::OpenDelim(token::Brace) => { - debug!("parse_trait_method_body(): parsing provided method"); - *at_end = true; - let (inner_attrs, body) = self.parse_inner_attrs_and_block()?; - attrs.extend(inner_attrs.iter().cloned()); - Some(body) - } - token::Interpolated(ref nt) => { - match **nt { - token::NtBlock(..) => { - *at_end = true; - let (inner_attrs, body) = self.parse_inner_attrs_and_block()?; - attrs.extend(inner_attrs.iter().cloned()); - Some(body) - } - _ => return self.expected_semi_or_open_brace(), - } - } - _ => return self.expected_semi_or_open_brace(), - }) - } - /// Parses the following grammar: /// /// TraitItemAssocTy = Ident ["<"...">"] [":" [GenericBounds]] ["where" ...] ["=" Ty] @@ -1194,45 +1082,6 @@ impl<'a> Parser<'a> { Ok(ident) } - /// Parses an item-position function declaration. - fn parse_item_fn( - &mut self, - lo: Span, - vis: Visibility, - attrs: Vec<Attribute>, - header: FnHeader, - ) -> PResult<'a, Option<P<Item>>> { - let (ident, decl, generics) = self.parse_fn_sig(ParamCfg { - is_self_allowed: false, - allow_c_variadic: header.abi == Abi::C && header.unsafety == Unsafety::Unsafe, - is_name_required: |_| true, - })?; - let (inner_attrs, body) = self.parse_inner_attrs_and_block()?; - let kind = ItemKind::Fn(decl, header, generics, body); - self.mk_item_with_info(attrs, lo, vis, (ident, kind, Some(inner_attrs))) - } - - /// Parse the "signature", including the identifier, parameters, and generics of a function. - fn parse_fn_sig(&mut self, cfg: ParamCfg) -> PResult<'a, (Ident, P<FnDecl>, Generics)> { - let ident = self.parse_ident()?; - let mut generics = self.parse_generics()?; - let decl = self.parse_fn_decl(cfg, true)?; - generics.where_clause = self.parse_where_clause()?; - Ok((ident, decl, generics)) - } - - /// Parses the parameter list and result type of a function declaration. - pub(super) fn parse_fn_decl( - &mut self, - cfg: ParamCfg, - ret_allow_plus: bool, - ) -> PResult<'a, P<FnDecl>> { - Ok(P(FnDecl { - inputs: self.parse_fn_params(cfg)?, - output: self.parse_ret_ty(ret_allow_plus)?, - })) - } - /// Parses `extern` for foreign ABIs modules. /// /// `extern` is expected to have been @@ -1273,7 +1122,7 @@ impl<'a> Parser<'a> { } /// Parses a foreign item. - crate fn parse_foreign_item(&mut self, extern_sp: Span) -> PResult<'a, ForeignItem> { + pub fn parse_foreign_item(&mut self, extern_sp: Span) -> PResult<'a, ForeignItem> { maybe_whole!(self, NtForeignItem, |ni| ni); let attrs = self.parse_outer_attributes()?; @@ -1344,32 +1193,6 @@ impl<'a> Parser<'a> { } } - /// Parses a function declaration from a foreign module. - fn parse_item_foreign_fn( - &mut self, - vis: ast::Visibility, - lo: Span, - attrs: Vec<Attribute>, - extern_sp: Span, - ) -> PResult<'a, ForeignItem> { - self.expect_keyword(kw::Fn)?; - let (ident, decl, generics) = self.parse_fn_sig(super::ParamCfg { - is_self_allowed: false, - allow_c_variadic: true, - is_name_required: |_| true, - })?; - let span = lo.to(self.token.span); - self.parse_semi_or_incorrect_foreign_fn_body(&ident, extern_sp)?; - Ok(ast::ForeignItem { - ident, - attrs, - kind: ForeignItemKind::Fn(decl, generics), - id: DUMMY_NODE_ID, - span, - vis, - }) - } - /// Parses a static item from a foreign module. /// Assumes that the `static` keyword is already parsed. fn parse_item_foreign_static(&mut self, vis: ast::Visibility, lo: Span, attrs: Vec<Attribute>) @@ -1910,3 +1733,466 @@ impl<'a> Parser<'a> { }) } } + +/// The parsing configuration used to parse a parameter list (see `parse_fn_params`). +pub(super) struct ParamCfg { + /// Is `self` is allowed as the first parameter? + pub is_self_allowed: bool, + /// Is `...` allowed as the tail of the parameter list? + pub allow_c_variadic: bool, + /// `is_name_required` decides if, per-parameter, + /// the parameter must have a pattern or just a type. + pub is_name_required: fn(&token::Token) -> bool, +} + +/// Parsing of functions and methods. +impl<'a> Parser<'a> { + /// Parses an item-position function declaration. + fn parse_item_fn( + &mut self, + lo: Span, + vis: Visibility, + attrs: Vec<Attribute>, + header: FnHeader, + ) -> PResult<'a, Option<P<Item>>> { + let (ident, decl, generics) = self.parse_fn_sig(ParamCfg { + is_self_allowed: false, + allow_c_variadic: header.abi == Abi::C && header.unsafety == Unsafety::Unsafe, + is_name_required: |_| true, + })?; + let (inner_attrs, body) = self.parse_inner_attrs_and_block()?; + let kind = ItemKind::Fn(decl, header, generics, body); + self.mk_item_with_info(attrs, lo, vis, (ident, kind, Some(inner_attrs))) + } + + /// Parses a function declaration from a foreign module. + fn parse_item_foreign_fn( + &mut self, + vis: ast::Visibility, + lo: Span, + attrs: Vec<Attribute>, + extern_sp: Span, + ) -> PResult<'a, ForeignItem> { + self.expect_keyword(kw::Fn)?; + let (ident, decl, generics) = self.parse_fn_sig(ParamCfg { + is_self_allowed: false, + allow_c_variadic: true, + is_name_required: |_| true, + })?; + let span = lo.to(self.token.span); + self.parse_semi_or_incorrect_foreign_fn_body(&ident, extern_sp)?; + Ok(ast::ForeignItem { + ident, + attrs, + kind: ForeignItemKind::Fn(decl, generics), + id: DUMMY_NODE_ID, + span, + vis, + }) + } + + /// Parses a method or a macro invocation in a trait impl. + fn parse_impl_method( + &mut self, + at_end: &mut bool, + ) -> PResult<'a, (Ident, Vec<Attribute>, Generics, ImplItemKind)> { + let (ident, sig, generics) = self.parse_method_sig(|_| true)?; + *at_end = true; + let (inner_attrs, body) = self.parse_inner_attrs_and_block()?; + Ok((ident, inner_attrs, generics, ast::ImplItemKind::Method(sig, body))) + } + + fn parse_trait_item_method( + &mut self, + at_end: &mut bool, + attrs: &mut Vec<Attribute>, + ) -> PResult<'a, (Ident, TraitItemKind, Generics)> { + // This is somewhat dubious; We don't want to allow + // argument names to be left off if there is a definition... + // + // We don't allow argument names to be left off in edition 2018. + let (ident, sig, generics) = self.parse_method_sig(|t| t.span.rust_2018())?; + let body = self.parse_trait_method_body(at_end, attrs)?; + Ok((ident, TraitItemKind::Method(sig, body), generics)) + } + + /// Parse the "body" of a method in a trait item definition. + /// This can either be `;` when there's no body, + /// or e.g. a block when the method is a provided one. + fn parse_trait_method_body( + &mut self, + at_end: &mut bool, + attrs: &mut Vec<Attribute>, + ) -> PResult<'a, Option<P<Block>>> { + Ok(match self.token.kind { + token::Semi => { + debug!("parse_trait_method_body(): parsing required method"); + self.bump(); + *at_end = true; + None + } + token::OpenDelim(token::Brace) => { + debug!("parse_trait_method_body(): parsing provided method"); + *at_end = true; + let (inner_attrs, body) = self.parse_inner_attrs_and_block()?; + attrs.extend(inner_attrs.iter().cloned()); + Some(body) + } + token::Interpolated(ref nt) => { + match **nt { + token::NtBlock(..) => { + *at_end = true; + let (inner_attrs, body) = self.parse_inner_attrs_and_block()?; + attrs.extend(inner_attrs.iter().cloned()); + Some(body) + } + _ => return self.expected_semi_or_open_brace(), + } + } + _ => return self.expected_semi_or_open_brace(), + }) + } + + /// Parse the "signature", including the identifier, parameters, and generics + /// of a method. The body is not parsed as that differs between `trait`s and `impl`s. + fn parse_method_sig( + &mut self, + is_name_required: fn(&token::Token) -> bool, + ) -> PResult<'a, (Ident, MethodSig, Generics)> { + let header = self.parse_fn_front_matter()?; + let (ident, decl, generics) = self.parse_fn_sig(ParamCfg { + is_self_allowed: true, + allow_c_variadic: false, + is_name_required, + })?; + Ok((ident, MethodSig { header, decl }, generics)) + } + + /// Parses all the "front matter" for a `fn` declaration, up to + /// and including the `fn` keyword: + /// + /// - `const fn` + /// - `unsafe fn` + /// - `const unsafe fn` + /// - `extern fn` + /// - etc. + fn parse_fn_front_matter(&mut self) -> PResult<'a, FnHeader> { + let is_const_fn = self.eat_keyword(kw::Const); + let const_span = self.prev_span; + let asyncness = self.parse_asyncness(); + if let IsAsync::Async { .. } = asyncness { + self.ban_async_in_2015(self.prev_span); + } + let asyncness = respan(self.prev_span, asyncness); + let unsafety = self.parse_unsafety(); + let (constness, unsafety, abi) = if is_const_fn { + (respan(const_span, Constness::Const), unsafety, Abi::Rust) + } else { + let abi = self.parse_extern_abi()?; + (respan(self.prev_span, Constness::NotConst), unsafety, abi) + }; + if !self.eat_keyword(kw::Fn) { + // It is possible for `expect_one_of` to recover given the contents of + // `self.expected_tokens`, therefore, do not use `self.unexpected()` which doesn't + // account for this. + if !self.expect_one_of(&[], &[])? { unreachable!() } + } + Ok(FnHeader { constness, unsafety, asyncness, abi }) + } + + /// Parse the "signature", including the identifier, parameters, and generics of a function. + fn parse_fn_sig(&mut self, cfg: ParamCfg) -> PResult<'a, (Ident, P<FnDecl>, Generics)> { + let ident = self.parse_ident()?; + let mut generics = self.parse_generics()?; + let decl = self.parse_fn_decl(cfg, true)?; + generics.where_clause = self.parse_where_clause()?; + Ok((ident, decl, generics)) + } + + /// Parses the parameter list and result type of a function declaration. + pub(super) fn parse_fn_decl( + &mut self, + cfg: ParamCfg, + ret_allow_plus: bool, + ) -> PResult<'a, P<FnDecl>> { + Ok(P(FnDecl { + inputs: self.parse_fn_params(cfg)?, + output: self.parse_ret_ty(ret_allow_plus)?, + })) + } + + /// Parses the parameter list of a function, including the `(` and `)` delimiters. + fn parse_fn_params(&mut self, mut cfg: ParamCfg) -> PResult<'a, Vec<Param>> { + let sp = self.token.span; + let is_trait_item = cfg.is_self_allowed; + let mut c_variadic = false; + // Parse the arguments, starting out with `self` being possibly allowed... + let (params, _) = self.parse_paren_comma_seq(|p| { + let param = p.parse_param_general(&cfg, is_trait_item); + // ...now that we've parsed the first argument, `self` is no longer allowed. + cfg.is_self_allowed = false; + + match param { + Ok(param) => Ok( + if let TyKind::CVarArgs = param.ty.kind { + c_variadic = true; + if p.token != token::CloseDelim(token::Paren) { + p.span_err( + p.token.span, + "`...` must be the last argument of a C-variadic function", + ); + // FIXME(eddyb) this should probably still push `CVarArgs`. + // Maybe AST validation/HIR lowering should emit the above error? + None + } else { + Some(param) + } + } else { + Some(param) + } + ), + Err(mut e) => { + e.emit(); + let lo = p.prev_span; + // Skip every token until next possible arg or end. + p.eat_to_tokens(&[&token::Comma, &token::CloseDelim(token::Paren)]); + // Create a placeholder argument for proper arg count (issue #34264). + let span = lo.to(p.prev_span); + Ok(Some(dummy_arg(Ident::new(kw::Invalid, span)))) + } + } + })?; + + let mut params: Vec<_> = params.into_iter().filter_map(|x| x).collect(); + + // Replace duplicated recovered params with `_` pattern to avoid unecessary errors. + self.deduplicate_recovered_params_names(&mut params); + + if c_variadic && params.len() <= 1 { + self.span_err( + sp, + "C-variadic function must be declared with at least one named argument", + ); + } + + Ok(params) + } + + /// Skips unexpected attributes and doc comments in this position and emits an appropriate + /// error. + /// This version of parse param doesn't necessarily require identifier names. + fn parse_param_general(&mut self, cfg: &ParamCfg, is_trait_item: bool) -> PResult<'a, Param> { + let lo = self.token.span; + let attrs = self.parse_outer_attributes()?; + + // Possibly parse `self`. Recover if we parsed it and it wasn't allowed here. + if let Some(mut param) = self.parse_self_param()? { + param.attrs = attrs.into(); + return if cfg.is_self_allowed { + Ok(param) + } else { + self.recover_bad_self_param(param, is_trait_item) + }; + } + + let is_name_required = match self.token.kind { + token::DotDotDot => false, + _ => (cfg.is_name_required)(&self.token), + }; + let (pat, ty) = if is_name_required || self.is_named_param() { + debug!("parse_param_general parse_pat (is_name_required:{})", is_name_required); + + let pat = self.parse_fn_param_pat()?; + if let Err(mut err) = self.expect(&token::Colon) { + return if let Some(ident) = self.parameter_without_type( + &mut err, + pat, + is_name_required, + cfg.is_self_allowed, + is_trait_item, + ) { + err.emit(); + Ok(dummy_arg(ident)) + } else { + Err(err) + }; + } + + self.eat_incorrect_doc_comment_for_param_type(); + (pat, self.parse_ty_common(true, true, cfg.allow_c_variadic)?) + } else { + debug!("parse_param_general ident_to_pat"); + let parser_snapshot_before_ty = self.clone(); + self.eat_incorrect_doc_comment_for_param_type(); + let mut ty = self.parse_ty_common(true, true, cfg.allow_c_variadic); + if ty.is_ok() && self.token != token::Comma && + self.token != token::CloseDelim(token::Paren) { + // This wasn't actually a type, but a pattern looking like a type, + // so we are going to rollback and re-parse for recovery. + ty = self.unexpected(); + } + match ty { + Ok(ty) => { + let ident = Ident::new(kw::Invalid, self.prev_span); + let bm = BindingMode::ByValue(Mutability::Immutable); + let pat = self.mk_pat_ident(ty.span, bm, ident); + (pat, ty) + } + // If this is a C-variadic argument and we hit an error, return the error. + Err(err) if self.token == token::DotDotDot => return Err(err), + // Recover from attempting to parse the argument as a type without pattern. + Err(mut err) => { + err.cancel(); + mem::replace(self, parser_snapshot_before_ty); + self.recover_arg_parse()? + } + } + }; + + let span = lo.to(self.token.span); + + Ok(Param { + attrs: attrs.into(), + id: ast::DUMMY_NODE_ID, + is_placeholder: false, + pat, + span, + ty, + }) + } + + /// Returns the parsed optional self parameter and whether a self shortcut was used. + /// + /// See `parse_self_param_with_attrs` to collect attributes. + fn parse_self_param(&mut self) -> PResult<'a, Option<Param>> { + // Extract an identifier *after* having confirmed that the token is one. + let expect_self_ident = |this: &mut Self| { + match this.token.kind { + // Preserve hygienic context. + token::Ident(name, _) => { + let span = this.token.span; + this.bump(); + Ident::new(name, span) + } + _ => unreachable!(), + } + }; + // Is `self` `n` tokens ahead? + let is_isolated_self = |this: &Self, n| { + this.is_keyword_ahead(n, &[kw::SelfLower]) + && this.look_ahead(n + 1, |t| t != &token::ModSep) + }; + // Is `mut self` `n` tokens ahead? + let is_isolated_mut_self = |this: &Self, n| { + this.is_keyword_ahead(n, &[kw::Mut]) + && is_isolated_self(this, n + 1) + }; + // Parse `self` or `self: TYPE`. We already know the current token is `self`. + let parse_self_possibly_typed = |this: &mut Self, m| { + let eself_ident = expect_self_ident(this); + let eself_hi = this.prev_span; + let eself = if this.eat(&token::Colon) { + SelfKind::Explicit(this.parse_ty()?, m) + } else { + SelfKind::Value(m) + }; + Ok((eself, eself_ident, eself_hi)) + }; + // Recover for the grammar `*self`, `*const self`, and `*mut self`. + let recover_self_ptr = |this: &mut Self| { + let msg = "cannot pass `self` by raw pointer"; + let span = this.token.span; + this.struct_span_err(span, msg) + .span_label(span, msg) + .emit(); + + Ok((SelfKind::Value(Mutability::Immutable), expect_self_ident(this), this.prev_span)) + }; + + // Parse optional `self` parameter of a method. + // Only a limited set of initial token sequences is considered `self` parameters; anything + // else is parsed as a normal function parameter list, so some lookahead is required. + let eself_lo = self.token.span; + let (eself, eself_ident, eself_hi) = match self.token.kind { + token::BinOp(token::And) => { + let eself = if is_isolated_self(self, 1) { + // `&self` + self.bump(); + SelfKind::Region(None, Mutability::Immutable) + } else if is_isolated_mut_self(self, 1) { + // `&mut self` + self.bump(); + self.bump(); + SelfKind::Region(None, Mutability::Mutable) + } else if self.look_ahead(1, |t| t.is_lifetime()) && is_isolated_self(self, 2) { + // `&'lt self` + self.bump(); + let lt = self.expect_lifetime(); + SelfKind::Region(Some(lt), Mutability::Immutable) + } else if self.look_ahead(1, |t| t.is_lifetime()) && is_isolated_mut_self(self, 2) { + // `&'lt mut self` + self.bump(); + let lt = self.expect_lifetime(); + self.bump(); + SelfKind::Region(Some(lt), Mutability::Mutable) + } else { + // `¬_self` + return Ok(None); + }; + (eself, expect_self_ident(self), self.prev_span) + } + // `*self` + token::BinOp(token::Star) if is_isolated_self(self, 1) => { + self.bump(); + recover_self_ptr(self)? + } + // `*mut self` and `*const self` + token::BinOp(token::Star) if + self.look_ahead(1, |t| t.is_mutability()) + && is_isolated_self(self, 2) => + { + self.bump(); + self.bump(); + recover_self_ptr(self)? + } + // `self` and `self: TYPE` + token::Ident(..) if is_isolated_self(self, 0) => { + parse_self_possibly_typed(self, Mutability::Immutable)? + } + // `mut self` and `mut self: TYPE` + token::Ident(..) if is_isolated_mut_self(self, 0) => { + self.bump(); + parse_self_possibly_typed(self, Mutability::Mutable)? + } + _ => return Ok(None), + }; + + let eself = source_map::respan(eself_lo.to(eself_hi), eself); + Ok(Some(Param::from_self(ThinVec::default(), eself, eself_ident))) + } + + fn is_named_param(&self) -> bool { + let offset = match self.token.kind { + token::Interpolated(ref nt) => match **nt { + token::NtPat(..) => return self.look_ahead(1, |t| t == &token::Colon), + _ => 0, + } + token::BinOp(token::And) | token::AndAnd => 1, + _ if self.token.is_keyword(kw::Mut) => 1, + _ => 0, + }; + + self.look_ahead(offset, |t| t.is_ident()) && + self.look_ahead(offset + 1, |t| t == &token::Colon) + } + + fn recover_first_param(&mut self) -> &'static str { + match self.parse_outer_attributes() + .and_then(|_| self.parse_self_param()) + .map_err(|mut e| e.cancel()) + { + Ok(Some(_)) => "method", + _ => "function", + } + } +} diff --git a/src/libsyntax/parse/parser/module.rs b/src/libsyntax/parse/parser/module.rs index 2d2fb487d7d..a0e4d2bbb7a 100644 --- a/src/libsyntax/parse/parser/module.rs +++ b/src/libsyntax/parse/parser/module.rs @@ -1,24 +1,24 @@ use super::{Parser, PResult}; use super::item::ItemInfo; +use super::diagnostics::Error; use crate::attr; use crate::ast::{self, Ident, Attribute, ItemKind, Mod, Crate}; use crate::parse::{new_sub_parser_from_file, DirectoryOwnership}; use crate::parse::token::{self, TokenKind}; -use crate::parse::diagnostics::{Error}; use crate::source_map::{SourceMap, Span, DUMMY_SP, FileName}; use crate::symbol::sym; use std::path::{self, Path, PathBuf}; /// Information about the path to a module. -pub struct ModulePath { +pub(super) struct ModulePath { name: String, path_exists: bool, pub result: Result<ModulePathSuccess, Error>, } -pub struct ModulePathSuccess { +pub(super) struct ModulePathSuccess { pub path: PathBuf, pub directory_ownership: DirectoryOwnership, warn: bool, @@ -39,6 +39,8 @@ impl<'a> Parser<'a> { /// Parses a `mod <foo> { ... }` or `mod <foo>;` item. pub(super) fn parse_item_mod(&mut self, outer_attrs: &[Attribute]) -> PResult<'a, ItemInfo> { let (in_cfg, outer_attrs) = { + // FIXME(Centril): This results in a cycle between config and parsing. + // Consider using dynamic dispatch via `self.sess` to disentangle the knot. let mut strip_unconfigured = crate::config::StripUnconfigured { sess: self.sess, features: None, // Don't perform gated feature checking. @@ -198,7 +200,7 @@ impl<'a> Parser<'a> { } } - pub fn submod_path_from_attr(attrs: &[Attribute], dir_path: &Path) -> Option<PathBuf> { + pub(super) fn submod_path_from_attr(attrs: &[Attribute], dir_path: &Path) -> Option<PathBuf> { if let Some(s) = attr::first_attr_value_str_by_name(attrs, sym::path) { let s = s.as_str(); @@ -215,7 +217,7 @@ impl<'a> Parser<'a> { } /// Returns a path to a module. - pub fn default_submod_path( + pub(super) fn default_submod_path( id: ast::Ident, relative: Option<ast::Ident>, dir_path: &Path, diff --git a/src/libsyntax/parse/parser/pat.rs b/src/libsyntax/parse/parser/pat.rs index 48f9e301610..af795e51792 100644 --- a/src/libsyntax/parse/parser/pat.rs +++ b/src/libsyntax/parse/parser/pat.rs @@ -22,7 +22,7 @@ const WHILE_PARSING_OR_MSG: &str = "while parsing this or-pattern starting here" /// Whether or not an or-pattern should be gated when occurring in the current context. #[derive(PartialEq)] -pub enum GateOr { Yes, No } +pub(super) enum GateOr { Yes, No } /// Whether or not to recover a `,` when parsing or-patterns. #[derive(PartialEq, Copy, Clone)] @@ -367,6 +367,7 @@ impl<'a> Parser<'a> { let pat = self.mk_pat(lo.to(self.prev_span), pat); let pat = self.maybe_recover_from_bad_qpath(pat, true)?; + let pat = self.recover_intersection_pat(pat)?; if !allow_range_pat { self.ban_pat_range_if_ambiguous(&pat)? @@ -375,6 +376,65 @@ impl<'a> Parser<'a> { Ok(pat) } + /// Try to recover the more general form `intersect ::= $pat_lhs @ $pat_rhs`. + /// + /// Allowed binding patterns generated by `binding ::= ref? mut? $ident @ $pat_rhs` + /// should already have been parsed by now at this point, + /// if the next token is `@` then we can try to parse the more general form. + /// + /// Consult `parse_pat_ident` for the `binding` grammar. + /// + /// The notion of intersection patterns are found in + /// e.g. [F#][and] where they are called AND-patterns. + /// + /// [and]: https://docs.microsoft.com/en-us/dotnet/fsharp/language-reference/pattern-matching + fn recover_intersection_pat(&mut self, lhs: P<Pat>) -> PResult<'a, P<Pat>> { + if self.token.kind != token::At { + // Next token is not `@` so it's not going to be an intersection pattern. + return Ok(lhs); + } + + // At this point we attempt to parse `@ $pat_rhs` and emit an error. + self.bump(); // `@` + let mut rhs = self.parse_pat(None)?; + let sp = lhs.span.to(rhs.span); + + if let PatKind::Ident(_, _, ref mut sub @ None) = rhs.kind { + // The user inverted the order, so help them fix that. + let mut applicability = Applicability::MachineApplicable; + lhs.walk(&mut |p| match p.kind { + // `check_match` is unhappy if the subpattern has a binding anywhere. + PatKind::Ident(..) => { + applicability = Applicability::MaybeIncorrect; + false // Short-circuit. + }, + _ => true, + }); + + let lhs_span = lhs.span; + // Move the LHS into the RHS as a subpattern. + // The RHS is now the full pattern. + *sub = Some(lhs); + + self.struct_span_err(sp, "pattern on wrong side of `@`") + .span_label(lhs_span, "pattern on the left, should be on the right") + .span_label(rhs.span, "binding on the right, should be on the left") + .span_suggestion(sp, "switch the order", pprust::pat_to_string(&rhs), applicability) + .emit(); + } else { + // The special case above doesn't apply so we may have e.g. `A(x) @ B(y)`. + rhs.kind = PatKind::Wild; + self.struct_span_err(sp, "left-hand side of `@` must be a binding") + .span_label(lhs.span, "interpreted as a pattern, not a binding") + .span_label(rhs.span, "also a pattern") + .note("bindings are `x`, `mut x`, `ref x`, and `ref mut x`") + .emit(); + } + + rhs.span = sp; + Ok(rhs) + } + /// Ban a range pattern if it has an ambiguous interpretation. fn ban_pat_range_if_ambiguous(&self, pat: &Pat) -> PResult<'a, ()> { match pat.kind { diff --git a/src/libsyntax/parse/parser/path.rs b/src/libsyntax/parse/parser/path.rs index ca823991a2e..639d61a2b5c 100644 --- a/src/libsyntax/parse/parser/path.rs +++ b/src/libsyntax/parse/parser/path.rs @@ -111,7 +111,7 @@ impl<'a> Parser<'a> { /// Like `parse_path`, but also supports parsing `Word` meta items into paths for /// backwards-compatibility. This is used when parsing derive macro paths in `#[derive]` /// attributes. - pub fn parse_path_allowing_meta(&mut self, style: PathStyle) -> PResult<'a, Path> { + fn parse_path_allowing_meta(&mut self, style: PathStyle) -> PResult<'a, Path> { let meta_ident = match self.token.kind { token::Interpolated(ref nt) => match **nt { token::NtMeta(ref item) => match item.tokens.is_empty() { @@ -129,7 +129,22 @@ impl<'a> Parser<'a> { self.parse_path(style) } - crate fn parse_path_segments( + /// Parse a list of paths inside `#[derive(path_0, ..., path_n)]`. + crate fn parse_derive_paths(&mut self) -> PResult<'a, Vec<Path>> { + self.expect(&token::OpenDelim(token::Paren))?; + let mut list = Vec::new(); + while !self.eat(&token::CloseDelim(token::Paren)) { + let path = self.parse_path_allowing_meta(PathStyle::Mod)?; + list.push(path); + if !self.eat(&token::Comma) { + self.expect(&token::CloseDelim(token::Paren))?; + break + } + } + Ok(list) + } + + pub(super) fn parse_path_segments( &mut self, segments: &mut Vec<PathSegment>, style: PathStyle, diff --git a/src/libsyntax/parse/parser/stmt.rs b/src/libsyntax/parse/parser/stmt.rs index 855b03ddd6f..d54d9c4b8e9 100644 --- a/src/libsyntax/parse/parser/stmt.rs +++ b/src/libsyntax/parse/parser/stmt.rs @@ -2,14 +2,13 @@ use super::{Parser, PResult, Restrictions, PrevTokenKind, SemiColonMode, BlockMo use super::expr::LhsExpr; use super::path::PathStyle; use super::pat::GateOr; +use super::diagnostics::Error; use crate::ptr::P; use crate::{maybe_whole, ThinVec}; use crate::ast::{self, DUMMY_NODE_ID, Stmt, StmtKind, Local, Block, BlockCheckMode, Expr, ExprKind}; use crate::ast::{Attribute, AttrStyle, VisibilityKind, MacStmtStyle, Mac, MacDelimiter}; -use crate::ext::base::DummyResult; use crate::parse::{classify, DirectoryOwnership}; -use crate::parse::diagnostics::Error; use crate::parse::token; use crate::source_map::{respan, Span}; use crate::symbol::{kw, sym}; @@ -373,7 +372,9 @@ impl<'a> Parser<'a> { } /// Parses a block. Inner attributes are allowed. - crate fn parse_inner_attrs_and_block(&mut self) -> PResult<'a, (Vec<Attribute>, P<Block>)> { + pub(super) fn parse_inner_attrs_and_block( + &mut self + ) -> PResult<'a, (Vec<Attribute>, P<Block>)> { maybe_whole!(self, NtBlock, |x| (Vec::new(), x)); let lo = self.token.span; @@ -400,7 +401,7 @@ impl<'a> Parser<'a> { self.recover_stmt_(SemiColonMode::Ignore, BlockMode::Ignore); Some(Stmt { id: DUMMY_NODE_ID, - kind: StmtKind::Expr(DummyResult::raw_expr(self.token.span, true)), + kind: StmtKind::Expr(self.mk_expr_err(self.token.span)), span: self.token.span, }) } @@ -422,7 +423,7 @@ impl<'a> Parser<'a> { } /// Parses a statement, including the trailing semicolon. - crate fn parse_full_stmt(&mut self, macro_legacy_warnings: bool) -> PResult<'a, Option<Stmt>> { + pub fn parse_full_stmt(&mut self, macro_legacy_warnings: bool) -> PResult<'a, Option<Stmt>> { // Skip looking for a trailing semicolon when we have an interpolated statement. maybe_whole!(self, NtStmt, |x| Some(x)); @@ -443,7 +444,7 @@ impl<'a> Parser<'a> { self.recover_stmt(); // Don't complain about type errors in body tail after parse error (#57383). let sp = expr.span.to(self.prev_span); - stmt.kind = StmtKind::Expr(DummyResult::raw_expr(sp, true)); + stmt.kind = StmtKind::Expr(self.mk_expr_err(sp)); } } } diff --git a/src/libsyntax/parse/parser/ty.rs b/src/libsyntax/parse/parser/ty.rs index 018b5951e6e..86c94b680b2 100644 --- a/src/libsyntax/parse/parser/ty.rs +++ b/src/libsyntax/parse/parser/ty.rs @@ -1,4 +1,5 @@ use super::{Parser, PResult, PathStyle, PrevTokenKind, TokenType}; +use super::item::ParamCfg; use crate::{maybe_whole, maybe_recover_from_interpolated_ty_qpath}; use crate::ptr::P; @@ -209,7 +210,7 @@ impl<'a> Parser<'a> { }; let span = lo.to(self.prev_span); - let ty = P(Ty { kind, span, id: ast::DUMMY_NODE_ID }); + let ty = self.mk_ty(span, kind); // Try to recover from use of `+` with incorrect priority. self.maybe_report_ambiguous_plus(allow_plus, impl_dyn_multi, &ty); @@ -281,7 +282,7 @@ impl<'a> Parser<'a> { let unsafety = self.parse_unsafety(); let abi = self.parse_extern_abi()?; self.expect_keyword(kw::Fn)?; - let cfg = super::ParamCfg { + let cfg = ParamCfg { is_self_allowed: false, allow_c_variadic: true, is_name_required: |_| false, @@ -295,7 +296,7 @@ impl<'a> Parser<'a> { }))) } - crate fn parse_generic_bounds(&mut self, + pub(super) fn parse_generic_bounds(&mut self, colon_span: Option<Span>) -> PResult<'a, GenericBounds> { self.parse_generic_bounds_common(true, colon_span) } @@ -432,13 +433,13 @@ impl<'a> Parser<'a> { } } - crate fn check_lifetime(&mut self) -> bool { + pub fn check_lifetime(&mut self) -> bool { self.expected_tokens.push(TokenType::Lifetime); self.token.is_lifetime() } /// Parses a single lifetime `'a` or panics. - crate fn expect_lifetime(&mut self) -> Lifetime { + pub fn expect_lifetime(&mut self) -> Lifetime { if let Some(ident) = self.token.lifetime() { let span = self.token.span; self.bump(); @@ -447,4 +448,8 @@ impl<'a> Parser<'a> { self.span_bug(self.token.span, "not a lifetime") } } + + pub(super) fn mk_ty(&self, span: Span, kind: TyKind) -> P<Ty> { + P(Ty { kind, span, id: ast::DUMMY_NODE_ID }) + } } diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index fd78a2bd534..4a8b25c6107 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -4,16 +4,13 @@ pub use DelimToken::*; pub use LitKind::*; pub use TokenKind::*; -use crate::ast::{self}; -use crate::parse::{parse_stream_from_source_str, ParseSess}; -use crate::print::pprust; +use crate::ast; use crate::ptr::P; use crate::symbol::kw; -use crate::tokenstream::{self, DelimSpan, TokenStream, TokenTree}; +use crate::tokenstream::TokenTree; use syntax_pos::symbol::Symbol; -use syntax_pos::{self, Span, FileName, DUMMY_SP}; -use log::info; +use syntax_pos::{self, Span, DUMMY_SP}; use std::fmt; use std::mem; @@ -36,7 +33,7 @@ pub enum BinOpToken { } /// A delimiter token. -#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] +#[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] pub enum DelimToken { /// A round parenthesis (i.e., `(` or `)`). Paren, @@ -288,7 +285,7 @@ impl TokenKind { } impl Token { - crate fn new(kind: TokenKind, span: Span) -> Self { + pub fn new(kind: TokenKind, span: Span) -> Self { Token { kind, span } } @@ -298,12 +295,12 @@ impl Token { } /// Recovers a `Token` from an `ast::Ident`. This creates a raw identifier if necessary. - crate fn from_ast_ident(ident: ast::Ident) -> Self { + pub fn from_ast_ident(ident: ast::Ident) -> Self { Token::new(Ident(ident.name, ident.is_raw_guess()), ident.span) } /// Return this token by value and leave a dummy token in its place. - crate fn take(&mut self) -> Self { + pub fn take(&mut self) -> Self { mem::replace(self, Token::dummy()) } @@ -324,7 +321,7 @@ impl Token { } /// Returns `true` if the token can appear at the start of an expression. - crate fn can_begin_expr(&self) -> bool { + pub fn can_begin_expr(&self) -> bool { match self.kind { Ident(name, is_raw) => ident_can_begin_expr(name, self.span, is_raw), // value name or keyword @@ -356,7 +353,7 @@ impl Token { } /// Returns `true` if the token can appear at the start of a type. - crate fn can_begin_type(&self) -> bool { + pub fn can_begin_type(&self) -> bool { match self.kind { Ident(name, is_raw) => ident_can_begin_type(name, self.span, is_raw), // type name or keyword @@ -399,7 +396,7 @@ impl Token { } /// Returns `true` if the token is any literal - crate fn is_lit(&self) -> bool { + pub fn is_lit(&self) -> bool { match self.kind { Literal(..) => true, _ => false, @@ -415,7 +412,7 @@ impl Token { /// Returns `true` if the token is any literal, a minus (which can prefix a literal, /// for example a '-42', or one of the boolean idents). - crate fn can_begin_literal_or_bool(&self) -> bool { + pub fn can_begin_literal_or_bool(&self) -> bool { match self.kind { Literal(..) | BinOp(Minus) => true, Ident(name, false) if name.is_bool_lit() => true, @@ -737,131 +734,3 @@ impl fmt::Debug for Nonterminal { } } } - -impl Nonterminal { - pub fn to_tokenstream(&self, sess: &ParseSess, span: Span) -> TokenStream { - // A `Nonterminal` is often a parsed AST item. At this point we now - // need to convert the parsed AST to an actual token stream, e.g. - // un-parse it basically. - // - // Unfortunately there's not really a great way to do that in a - // guaranteed lossless fashion right now. The fallback here is to just - // stringify the AST node and reparse it, but this loses all span - // information. - // - // As a result, some AST nodes are annotated with the token stream they - // came from. Here we attempt to extract these lossless token streams - // before we fall back to the stringification. - let tokens = match *self { - Nonterminal::NtItem(ref item) => { - prepend_attrs(sess, &item.attrs, item.tokens.as_ref(), span) - } - Nonterminal::NtTraitItem(ref item) => { - prepend_attrs(sess, &item.attrs, item.tokens.as_ref(), span) - } - Nonterminal::NtImplItem(ref item) => { - prepend_attrs(sess, &item.attrs, item.tokens.as_ref(), span) - } - Nonterminal::NtIdent(ident, is_raw) => { - Some(TokenTree::token(Ident(ident.name, is_raw), ident.span).into()) - } - Nonterminal::NtLifetime(ident) => { - Some(TokenTree::token(Lifetime(ident.name), ident.span).into()) - } - Nonterminal::NtTT(ref tt) => { - Some(tt.clone().into()) - } - _ => None, - }; - - // FIXME(#43081): Avoid this pretty-print + reparse hack - let source = pprust::nonterminal_to_string(self); - let filename = FileName::macro_expansion_source_code(&source); - let tokens_for_real = parse_stream_from_source_str(filename, source, sess, Some(span)); - - // During early phases of the compiler the AST could get modified - // directly (e.g., attributes added or removed) and the internal cache - // of tokens my not be invalidated or updated. Consequently if the - // "lossless" token stream disagrees with our actual stringification - // (which has historically been much more battle-tested) then we go - // with the lossy stream anyway (losing span information). - // - // Note that the comparison isn't `==` here to avoid comparing spans, - // but it *also* is a "probable" equality which is a pretty weird - // definition. We mostly want to catch actual changes to the AST - // like a `#[cfg]` being processed or some weird `macro_rules!` - // expansion. - // - // What we *don't* want to catch is the fact that a user-defined - // literal like `0xf` is stringified as `15`, causing the cached token - // stream to not be literal `==` token-wise (ignoring spans) to the - // token stream we got from stringification. - // - // Instead the "probably equal" check here is "does each token - // recursively have the same discriminant?" We basically don't look at - // the token values here and assume that such fine grained token stream - // modifications, including adding/removing typically non-semantic - // tokens such as extra braces and commas, don't happen. - if let Some(tokens) = tokens { - if tokens.probably_equal_for_proc_macro(&tokens_for_real) { - return tokens - } - info!("cached tokens found, but they're not \"probably equal\", \ - going with stringified version"); - } - return tokens_for_real - } -} - -fn prepend_attrs(sess: &ParseSess, - attrs: &[ast::Attribute], - tokens: Option<&tokenstream::TokenStream>, - span: syntax_pos::Span) - -> Option<tokenstream::TokenStream> -{ - let tokens = tokens?; - if attrs.len() == 0 { - return Some(tokens.clone()) - } - let mut builder = tokenstream::TokenStreamBuilder::new(); - for attr in attrs { - assert_eq!(attr.style, ast::AttrStyle::Outer, - "inner attributes should prevent cached tokens from existing"); - - let source = pprust::attribute_to_string(attr); - let macro_filename = FileName::macro_expansion_source_code(&source); - if attr.is_sugared_doc { - let stream = parse_stream_from_source_str(macro_filename, source, sess, Some(span)); - builder.push(stream); - continue - } - - // synthesize # [ $path $tokens ] manually here - let mut brackets = tokenstream::TokenStreamBuilder::new(); - - // For simple paths, push the identifier directly - if attr.path.segments.len() == 1 && attr.path.segments[0].args.is_none() { - let ident = attr.path.segments[0].ident; - let token = Ident(ident.name, ident.as_str().starts_with("r#")); - brackets.push(tokenstream::TokenTree::token(token, ident.span)); - - // ... and for more complicated paths, fall back to a reparse hack that - // should eventually be removed. - } else { - let stream = parse_stream_from_source_str(macro_filename, source, sess, Some(span)); - brackets.push(stream); - } - - brackets.push(attr.tokens.clone()); - - // The span we list here for `#` and for `[ ... ]` are both wrong in - // that it encompasses more than each token, but it hopefully is "good - // enough" for now at least. - builder.push(tokenstream::TokenTree::token(Pound, attr.span)); - let delim_span = DelimSpan::from_single(attr.span); - builder.push(tokenstream::TokenTree::Delimited( - delim_span, DelimToken::Bracket, brackets.build().into())); - } - builder.push(tokens.clone()); - Some(builder.build()) -} diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs index 7d4ffe493d7..136fc355f89 100644 --- a/src/libsyntax/print/pprust.rs +++ b/src/libsyntax/print/pprust.rs @@ -6,10 +6,11 @@ use crate::attr; use crate::source_map::{self, SourceMap, Spanned}; use crate::parse::token::{self, BinOpToken, DelimToken, Nonterminal, Token, TokenKind}; use crate::parse::lexer::comments; -use crate::parse::{self, ParseSess}; +use crate::parse; use crate::print::pp::{self, Breaks}; use crate::print::pp::Breaks::{Consistent, Inconsistent}; use crate::ptr::P; +use crate::sess::ParseSess; use crate::symbol::{kw, sym}; use crate::tokenstream::{self, TokenStream, TokenTree}; @@ -2381,7 +2382,8 @@ impl<'a> State<'a> { } self.print_ident(ident); if let Some(ref p) = *sub { - self.s.word("@"); + self.s.space(); + self.s.word_space("@"); self.print_pat(p); } } diff --git a/src/libsyntax/ptr.rs b/src/libsyntax/ptr.rs index 7300ce24954..d987dc855b6 100644 --- a/src/libsyntax/ptr.rs +++ b/src/libsyntax/ptr.rs @@ -35,7 +35,6 @@ use rustc_serialize::{Encodable, Decodable, Encoder, Decoder}; use rustc_data_structures::stable_hasher::{StableHasher, HashStable}; /// An owned smart pointer. -#[derive(Hash, PartialEq, Eq)] pub struct P<T: ?Sized> { ptr: Box<T> } diff --git a/src/libsyntax/sess.rs b/src/libsyntax/sess.rs new file mode 100644 index 00000000000..e49d3954f8e --- /dev/null +++ b/src/libsyntax/sess.rs @@ -0,0 +1,124 @@ +//! Contains `ParseSess` which holds state living beyond what one `Parser` might. +//! It also serves as an input to the parser itself. + +use crate::ast::{CrateConfig, NodeId}; +use crate::early_buffered_lints::{BufferedEarlyLint, BufferedEarlyLintId}; +use crate::source_map::{SourceMap, FilePathMapping}; +use crate::feature_gate::UnstableFeatures; + +use errors::{Applicability, Handler, ColorConfig, DiagnosticBuilder}; +use rustc_data_structures::fx::{FxHashSet, FxHashMap}; +use rustc_data_structures::sync::{Lrc, Lock, Once}; +use syntax_pos::{Symbol, Span, MultiSpan}; +use syntax_pos::edition::Edition; +use syntax_pos::hygiene::ExpnId; + +use std::path::PathBuf; +use std::str; + +/// Collected spans during parsing for places where a certain feature was +/// used and should be feature gated accordingly in `check_crate`. +#[derive(Default)] +crate struct GatedSpans { + /// Spans collected for gating `let_chains`, e.g. `if a && let b = c {}`. + crate let_chains: Lock<Vec<Span>>, + /// Spans collected for gating `async_closure`, e.g. `async || ..`. + crate async_closure: Lock<Vec<Span>>, + /// Spans collected for gating `yield e?` expressions (`generators` gate). + crate yields: Lock<Vec<Span>>, + /// Spans collected for gating `or_patterns`, e.g. `Some(Foo | Bar)`. + crate or_patterns: Lock<Vec<Span>>, + /// Spans collected for gating `const_extern_fn`, e.g. `const extern fn foo`. + crate const_extern_fn: Lock<Vec<Span>>, +} + +/// Info about a parsing session. +pub struct ParseSess { + pub span_diagnostic: Handler, + crate unstable_features: UnstableFeatures, + pub config: CrateConfig, + pub edition: Edition, + pub missing_fragment_specifiers: Lock<FxHashSet<Span>>, + /// Places where raw identifiers were used. This is used for feature-gating raw identifiers. + pub raw_identifier_spans: Lock<Vec<Span>>, + /// Used to determine and report recursive module inclusions. + pub(super) included_mod_stack: Lock<Vec<PathBuf>>, + source_map: Lrc<SourceMap>, + pub buffered_lints: Lock<Vec<BufferedEarlyLint>>, + /// Contains the spans of block expressions that could have been incomplete based on the + /// operation token that followed it, but that the parser cannot identify without further + /// analysis. + pub ambiguous_block_expr_parse: Lock<FxHashMap<Span, Span>>, + pub injected_crate_name: Once<Symbol>, + crate gated_spans: GatedSpans, +} + +impl ParseSess { + pub fn new(file_path_mapping: FilePathMapping) -> Self { + let cm = Lrc::new(SourceMap::new(file_path_mapping)); + let handler = Handler::with_tty_emitter( + ColorConfig::Auto, + true, + None, + Some(cm.clone()), + ); + ParseSess::with_span_handler(handler, cm) + } + + pub fn with_span_handler(handler: Handler, source_map: Lrc<SourceMap>) -> Self { + Self { + span_diagnostic: handler, + unstable_features: UnstableFeatures::from_environment(), + config: FxHashSet::default(), + edition: ExpnId::root().expn_data().edition, + missing_fragment_specifiers: Lock::new(FxHashSet::default()), + raw_identifier_spans: Lock::new(Vec::new()), + included_mod_stack: Lock::new(vec![]), + source_map, + buffered_lints: Lock::new(vec![]), + ambiguous_block_expr_parse: Lock::new(FxHashMap::default()), + injected_crate_name: Once::new(), + gated_spans: GatedSpans::default(), + } + } + + #[inline] + pub fn source_map(&self) -> &SourceMap { + &self.source_map + } + + pub fn buffer_lint( + &self, + lint_id: BufferedEarlyLintId, + span: impl Into<MultiSpan>, + id: NodeId, + msg: &str, + ) { + self.buffered_lints.with_lock(|buffered_lints| { + buffered_lints.push(BufferedEarlyLint{ + span: span.into(), + id, + msg: msg.into(), + lint_id, + }); + }); + } + + /// Extend an error with a suggestion to wrap an expression with parentheses to allow the + /// parser to continue parsing the following operation as part of the same expression. + pub fn expr_parentheses_needed( + &self, + err: &mut DiagnosticBuilder<'_>, + span: Span, + alt_snippet: Option<String>, + ) { + if let Some(snippet) = self.source_map().span_to_snippet(span).ok().or(alt_snippet) { + err.span_suggestion( + span, + "parentheses are required to parse this as an expression", + format!("({})", snippet), + Applicability::MachineApplicable, + ); + } + } +} diff --git a/src/libsyntax/source_map.rs b/src/libsyntax/source_map.rs index 5e569f9dae3..a1d147637e2 100644 --- a/src/libsyntax/source_map.rs +++ b/src/libsyntax/source_map.rs @@ -41,7 +41,7 @@ pub fn original_sp(sp: Span, enclosing_sp: Span) -> Span { } } -#[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] +#[derive(Clone, RustcEncodable, RustcDecodable, Debug, Copy)] pub struct Spanned<T> { pub node: T, pub span: Span, @@ -970,6 +970,9 @@ impl SourceMapper for SourceMap { fn span_to_string(&self, sp: Span) -> String { self.span_to_string(sp) } + fn span_to_snippet(&self, sp: Span) -> Result<String, SpanSnippetError> { + self.span_to_snippet(sp) + } fn span_to_filename(&self, sp: Span) -> FileName { self.span_to_filename(sp) } diff --git a/src/libsyntax/tests.rs b/src/libsyntax/tests.rs index f510ac9273d..881bdaa84d0 100644 --- a/src/libsyntax/tests.rs +++ b/src/libsyntax/tests.rs @@ -1,7 +1,8 @@ -use crate::{ast, panictry}; -use crate::parse::{ParseSess, PResult, source_file_to_stream}; +use crate::ast; +use crate::parse::{PResult, source_file_to_stream}; use crate::parse::new_parser_from_source_str; use crate::parse::parser::Parser; +use crate::sess::ParseSess; use crate::source_map::{SourceMap, FilePathMapping}; use crate::tokenstream::TokenStream; use crate::with_default_globals; @@ -27,7 +28,7 @@ crate fn with_error_checking_parse<'a, T, F>(s: String, ps: &'a ParseSess, f: F) F: FnOnce(&mut Parser<'a>) -> PResult<'a, T>, { let mut p = string_to_parser(&ps, s); - let x = panictry!(f(&mut p)); + let x = f(&mut p).unwrap(); p.sess.span_diagnostic.abort_if_errors(); x } diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index bef12ed4fad..ac155556cda 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -14,7 +14,6 @@ //! ownership of the original. use crate::parse::token::{self, DelimToken, Token, TokenKind}; -use crate::print::pprust; use syntax_pos::{BytePos, Span, DUMMY_SP}; #[cfg(target_arch = "x86_64")] @@ -23,7 +22,7 @@ use rustc_data_structures::sync::Lrc; use rustc_serialize::{Decoder, Decodable, Encoder, Encodable}; use smallvec::{SmallVec, smallvec}; -use std::{fmt, iter, mem}; +use std::{iter, mem}; #[cfg(test)] mod tests; @@ -137,13 +136,8 @@ impl TokenTree { /// The goal is for procedural macros to work with `TokenStream`s and `TokenTree`s /// instead of a representation of the abstract syntax tree. /// Today's `TokenTree`s can still contain AST via `token::Interpolated` for back-compat. -/// -/// The use of `Option` is an optimization that avoids the need for an -/// allocation when the stream is empty. However, it is not guaranteed that an -/// empty stream is represented with `None`; it may be represented as a `Some` -/// around an empty `Vec`. -#[derive(Clone, Debug)] -pub struct TokenStream(pub Option<Lrc<Vec<TreeAndJoint>>>); +#[derive(Clone, Debug, Default)] +pub struct TokenStream(pub Lrc<Vec<TreeAndJoint>>); pub type TreeAndJoint = (TokenTree, IsJoint); @@ -162,38 +156,36 @@ use IsJoint::*; impl TokenStream { /// Given a `TokenStream` with a `Stream` of only two arguments, return a new `TokenStream` /// separating the two arguments with a comma for diagnostic suggestions. - pub(crate) fn add_comma(&self) -> Option<(TokenStream, Span)> { + pub fn add_comma(&self) -> Option<(TokenStream, Span)> { // Used to suggest if a user writes `foo!(a b);` - if let Some(ref stream) = self.0 { - let mut suggestion = None; - let mut iter = stream.iter().enumerate().peekable(); - while let Some((pos, ts)) = iter.next() { - if let Some((_, next)) = iter.peek() { - let sp = match (&ts, &next) { - (_, (TokenTree::Token(Token { kind: token::Comma, .. }), _)) => continue, - ((TokenTree::Token(token_left), NonJoint), - (TokenTree::Token(token_right), _)) - if ((token_left.is_ident() && !token_left.is_reserved_ident()) - || token_left.is_lit()) && - ((token_right.is_ident() && !token_right.is_reserved_ident()) - || token_right.is_lit()) => token_left.span, - ((TokenTree::Delimited(sp, ..), NonJoint), _) => sp.entire(), - _ => continue, - }; - let sp = sp.shrink_to_hi(); - let comma = (TokenTree::token(token::Comma, sp), NonJoint); - suggestion = Some((pos, comma, sp)); - } - } - if let Some((pos, comma, sp)) = suggestion { - let mut new_stream = vec![]; - let parts = stream.split_at(pos + 1); - new_stream.extend_from_slice(parts.0); - new_stream.push(comma); - new_stream.extend_from_slice(parts.1); - return Some((TokenStream::new(new_stream), sp)); + let mut suggestion = None; + let mut iter = self.0.iter().enumerate().peekable(); + while let Some((pos, ts)) = iter.next() { + if let Some((_, next)) = iter.peek() { + let sp = match (&ts, &next) { + (_, (TokenTree::Token(Token { kind: token::Comma, .. }), _)) => continue, + ((TokenTree::Token(token_left), NonJoint), + (TokenTree::Token(token_right), _)) + if ((token_left.is_ident() && !token_left.is_reserved_ident()) + || token_left.is_lit()) && + ((token_right.is_ident() && !token_right.is_reserved_ident()) + || token_right.is_lit()) => token_left.span, + ((TokenTree::Delimited(sp, ..), NonJoint), _) => sp.entire(), + _ => continue, + }; + let sp = sp.shrink_to_hi(); + let comma = (TokenTree::token(token::Comma, sp), NonJoint); + suggestion = Some((pos, comma, sp)); } } + if let Some((pos, comma, sp)) = suggestion { + let mut new_stream = vec![]; + let parts = self.0.split_at(pos + 1); + new_stream.extend_from_slice(parts.0); + new_stream.push(comma); + new_stream.extend_from_slice(parts.1); + return Some((TokenStream::new(new_stream), sp)); + } None } } @@ -210,9 +202,9 @@ impl From<TokenTree> for TreeAndJoint { } } -impl<T: Into<TokenStream>> iter::FromIterator<T> for TokenStream { - fn from_iter<I: IntoIterator<Item = T>>(iter: I) -> Self { - TokenStream::from_streams(iter.into_iter().map(Into::into).collect::<SmallVec<_>>()) +impl iter::FromIterator<TokenTree> for TokenStream { + fn from_iter<I: IntoIterator<Item = TokenTree>>(iter: I) -> Self { + TokenStream::new(iter.into_iter().map(Into::into).collect::<Vec<TreeAndJoint>>()) } } @@ -225,28 +217,21 @@ impl PartialEq<TokenStream> for TokenStream { } impl TokenStream { - pub fn len(&self) -> usize { - if let Some(ref slice) = self.0 { - slice.len() - } else { - 0 - } + pub fn new(streams: Vec<TreeAndJoint>) -> TokenStream { + TokenStream(Lrc::new(streams)) } - pub fn empty() -> TokenStream { - TokenStream(None) + pub fn is_empty(&self) -> bool { + self.0.is_empty() } - pub fn is_empty(&self) -> bool { - match self.0 { - None => true, - Some(ref stream) => stream.is_empty(), - } + pub fn len(&self) -> usize { + self.0.len() } pub(crate) fn from_streams(mut streams: SmallVec<[TokenStream; 2]>) -> TokenStream { match streams.len() { - 0 => TokenStream::empty(), + 0 => TokenStream::default(), 1 => streams.pop().unwrap(), _ => { // We are going to extend the first stream in `streams` with @@ -270,43 +255,22 @@ impl TokenStream { // Get the first stream. If it's `None`, create an empty // stream. let mut iter = streams.drain(); - let mut first_stream_lrc = match iter.next().unwrap().0 { - Some(first_stream_lrc) => first_stream_lrc, - None => Lrc::new(vec![]), - }; + let mut first_stream_lrc = iter.next().unwrap().0; // Append the elements to the first stream, after reserving // space for them. let first_vec_mut = Lrc::make_mut(&mut first_stream_lrc); first_vec_mut.reserve(num_appends); for stream in iter { - if let Some(stream) = stream.0 { - first_vec_mut.extend(stream.iter().cloned()); - } + first_vec_mut.extend(stream.0.iter().cloned()); } // Create the final `TokenStream`. - match first_vec_mut.len() { - 0 => TokenStream(None), - _ => TokenStream(Some(first_stream_lrc)), - } + TokenStream(first_stream_lrc) } } } - pub fn new(streams: Vec<TreeAndJoint>) -> TokenStream { - match streams.len() { - 0 => TokenStream(None), - _ => TokenStream(Some(Lrc::new(streams))), - } - } - - pub fn append_to_tree_and_joint_vec(self, vec: &mut Vec<TreeAndJoint>) { - if let Some(stream) = self.0 { - vec.extend(stream.iter().cloned()); - } - } - pub fn trees(&self) -> Cursor { self.clone().into_trees() } @@ -371,24 +335,22 @@ impl TokenStream { } pub fn map_enumerated<F: FnMut(usize, TokenTree) -> TokenTree>(self, mut f: F) -> TokenStream { - TokenStream(self.0.map(|stream| { - Lrc::new( - stream - .iter() - .enumerate() - .map(|(i, (tree, is_joint))| (f(i, tree.clone()), *is_joint)) - .collect()) - })) + TokenStream(Lrc::new( + self.0 + .iter() + .enumerate() + .map(|(i, (tree, is_joint))| (f(i, tree.clone()), *is_joint)) + .collect() + )) } pub fn map<F: FnMut(TokenTree) -> TokenTree>(self, mut f: F) -> TokenStream { - TokenStream(self.0.map(|stream| { - Lrc::new( - stream - .iter() - .map(|(tree, is_joint)| (f(tree.clone()), *is_joint)) - .collect()) - })) + TokenStream(Lrc::new( + self.0 + .iter() + .map(|(tree, is_joint)| (f(tree.clone()), *is_joint)) + .collect() + )) } } @@ -406,44 +368,43 @@ impl TokenStreamBuilder { // If `self` is not empty and the last tree within the last stream is a // token tree marked with `Joint`... - if let Some(TokenStream(Some(ref mut last_stream_lrc))) = self.0.last_mut() { + if let Some(TokenStream(ref mut last_stream_lrc)) = self.0.last_mut() { if let Some((TokenTree::Token(last_token), Joint)) = last_stream_lrc.last() { // ...and `stream` is not empty and the first tree within it is // a token tree... - if let TokenStream(Some(ref mut stream_lrc)) = stream { - if let Some((TokenTree::Token(token), is_joint)) = stream_lrc.first() { - - // ...and the two tokens can be glued together... - if let Some(glued_tok) = last_token.glue(&token) { - - // ...then do so, by overwriting the last token - // tree in `self` and removing the first token tree - // from `stream`. This requires using `make_mut()` - // on the last stream in `self` and on `stream`, - // and in practice this doesn't cause cloning 99.9% - // of the time. - - // Overwrite the last token tree with the merged - // token. - let last_vec_mut = Lrc::make_mut(last_stream_lrc); - *last_vec_mut.last_mut().unwrap() = - (TokenTree::Token(glued_tok), *is_joint); - - // Remove the first token tree from `stream`. (This - // is almost always the only tree in `stream`.) - let stream_vec_mut = Lrc::make_mut(stream_lrc); - stream_vec_mut.remove(0); - - // Don't push `stream` if it's empty -- that could - // block subsequent token gluing, by getting - // between two token trees that should be glued - // together. - if !stream.is_empty() { - self.0.push(stream); - } - return; + let TokenStream(ref mut stream_lrc) = stream; + if let Some((TokenTree::Token(token), is_joint)) = stream_lrc.first() { + + // ...and the two tokens can be glued together... + if let Some(glued_tok) = last_token.glue(&token) { + + // ...then do so, by overwriting the last token + // tree in `self` and removing the first token tree + // from `stream`. This requires using `make_mut()` + // on the last stream in `self` and on `stream`, + // and in practice this doesn't cause cloning 99.9% + // of the time. + + // Overwrite the last token tree with the merged + // token. + let last_vec_mut = Lrc::make_mut(last_stream_lrc); + *last_vec_mut.last_mut().unwrap() = + (TokenTree::Token(glued_tok), *is_joint); + + // Remove the first token tree from `stream`. (This + // is almost always the only tree in `stream`.) + let stream_vec_mut = Lrc::make_mut(stream_lrc); + stream_vec_mut.remove(0); + + // Don't push `stream` if it's empty -- that could + // block subsequent token gluing, by getting + // between two token trees that should be glued + // together. + if !stream.is_empty() { + self.0.push(stream); } + return; } } } @@ -476,16 +437,11 @@ impl Cursor { } pub fn next_with_joint(&mut self) -> Option<TreeAndJoint> { - match self.stream.0 { - None => None, - Some(ref stream) => { - if self.index < stream.len() { - self.index += 1; - Some(stream[self.index - 1].clone()) - } else { - None - } - } + if self.index < self.stream.len() { + self.index += 1; + Some(self.stream.0[self.index - 1].clone()) + } else { + None } } @@ -494,22 +450,13 @@ impl Cursor { return; } let index = self.index; - let stream = mem::replace(&mut self.stream, TokenStream(None)); + let stream = mem::take(&mut self.stream); *self = TokenStream::from_streams(smallvec![stream, new_stream]).into_trees(); self.index = index; } pub fn look_ahead(&self, n: usize) -> Option<TokenTree> { - match self.stream.0 { - None => None, - Some(ref stream) => stream[self.index ..].get(n).map(|(tree, _)| tree.clone()), - } - } -} - -impl fmt::Display for TokenStream { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(&pprust::tts_to_string(self.clone())) + self.stream.0[self.index ..].get(n).map(|(tree, _)| tree.clone()) } } |