diff options
Diffstat (limited to 'src/libsyntax_expand')
| -rw-r--r-- | src/libsyntax_expand/Cargo.toml | 26 | ||||
| -rw-r--r-- | src/libsyntax_expand/allocator.rs | 75 | ||||
| -rw-r--r-- | src/libsyntax_expand/base.rs | 1189 | ||||
| -rw-r--r-- | src/libsyntax_expand/build.rs | 640 | ||||
| -rw-r--r-- | src/libsyntax_expand/expand.rs | 1551 | ||||
| -rw-r--r-- | src/libsyntax_expand/lib.rs | 38 | ||||
| -rw-r--r-- | src/libsyntax_expand/mbe.rs | 166 | ||||
| -rw-r--r-- | src/libsyntax_expand/mbe/macro_check.rs | 626 | ||||
| -rw-r--r-- | src/libsyntax_expand/mbe/macro_parser.rs | 944 | ||||
| -rw-r--r-- | src/libsyntax_expand/mbe/macro_rules.rs | 1192 | ||||
| -rw-r--r-- | src/libsyntax_expand/mbe/quoted.rs | 264 | ||||
| -rw-r--r-- | src/libsyntax_expand/mbe/transcribe.rs | 399 | ||||
| -rw-r--r-- | src/libsyntax_expand/placeholders.rs | 350 | ||||
| -rw-r--r-- | src/libsyntax_expand/proc_macro.rs | 215 | ||||
| -rw-r--r-- | src/libsyntax_expand/proc_macro_server.rs | 713 |
15 files changed, 8388 insertions, 0 deletions
diff --git a/src/libsyntax_expand/Cargo.toml b/src/libsyntax_expand/Cargo.toml new file mode 100644 index 00000000000..f063753f599 --- /dev/null +++ b/src/libsyntax_expand/Cargo.toml @@ -0,0 +1,26 @@ +[package] +authors = ["The Rust Project Developers"] +name = "syntax_expand" +version = "0.0.0" +edition = "2018" +build = false + +[lib] +name = "syntax_expand" +path = "lib.rs" +doctest = false + +[dependencies] +bitflags = "1.0" +rustc_serialize = { path = "../libserialize", package = "serialize" } +log = "0.4" +scoped-tls = "1.0" +lazy_static = "1.0.0" +syntax_pos = { path = "../libsyntax_pos" } +errors = { path = "../librustc_errors", package = "rustc_errors" } +rustc_data_structures = { path = "../librustc_data_structures" } +rustc_index = { path = "../librustc_index" } +rustc_lexer = { path = "../librustc_lexer" } +rustc_target = { path = "../librustc_target" } +smallvec = { version = "0.6.7", features = ["union", "may_dangle"] } +syntax = { path = "../libsyntax" } diff --git a/src/libsyntax_expand/allocator.rs b/src/libsyntax_expand/allocator.rs new file mode 100644 index 00000000000..3526be17721 --- /dev/null +++ b/src/libsyntax_expand/allocator.rs @@ -0,0 +1,75 @@ +use syntax::{ast, attr, visit}; +use syntax::symbol::{sym, Symbol}; +use syntax_pos::Span; + +#[derive(Clone, Copy)] +pub enum AllocatorKind { + Global, + DefaultLib, + DefaultExe, +} + +impl AllocatorKind { + pub fn fn_name(&self, base: &str) -> String { + match *self { + AllocatorKind::Global => format!("__rg_{}", base), + AllocatorKind::DefaultLib => format!("__rdl_{}", base), + AllocatorKind::DefaultExe => format!("__rde_{}", base), + } + } +} + +pub enum AllocatorTy { + Layout, + Ptr, + ResultPtr, + Unit, + Usize, +} + +pub struct AllocatorMethod { + pub name: &'static str, + pub inputs: &'static [AllocatorTy], + pub output: AllocatorTy, +} + +pub static ALLOCATOR_METHODS: &[AllocatorMethod] = &[ + AllocatorMethod { + name: "alloc", + inputs: &[AllocatorTy::Layout], + output: AllocatorTy::ResultPtr, + }, + AllocatorMethod { + name: "dealloc", + inputs: &[AllocatorTy::Ptr, AllocatorTy::Layout], + output: AllocatorTy::Unit, + }, + AllocatorMethod { + name: "realloc", + inputs: &[AllocatorTy::Ptr, AllocatorTy::Layout, AllocatorTy::Usize], + output: AllocatorTy::ResultPtr, + }, + AllocatorMethod { + name: "alloc_zeroed", + inputs: &[AllocatorTy::Layout], + output: AllocatorTy::ResultPtr, + }, +]; + +pub fn global_allocator_spans(krate: &ast::Crate) -> Vec<Span> { + struct Finder { name: Symbol, spans: Vec<Span> } + impl<'ast> visit::Visitor<'ast> for Finder { + fn visit_item(&mut self, item: &'ast ast::Item) { + if item.ident.name == self.name && + attr::contains_name(&item.attrs, sym::rustc_std_internal_symbol) { + self.spans.push(item.span); + } + visit::walk_item(self, item) + } + } + + let name = Symbol::intern(&AllocatorKind::Global.fn_name("alloc")); + let mut f = Finder { name, spans: Vec::new() }; + visit::walk_crate(&mut f, krate); + f.spans +} diff --git a/src/libsyntax_expand/base.rs b/src/libsyntax_expand/base.rs new file mode 100644 index 00000000000..593e06f29b9 --- /dev/null +++ b/src/libsyntax_expand/base.rs @@ -0,0 +1,1189 @@ +use crate::expand::{self, AstFragment, Invocation}; +use crate::hygiene::ExpnId; + +use syntax::ast::{self, NodeId, Attribute, Name, PatKind}; +use syntax::attr::{self, HasAttrs, Stability, Deprecation}; +use syntax::source_map::SourceMap; +use syntax::edition::Edition; +use syntax::mut_visit::{self, MutVisitor}; +use syntax::parse::{self, parser, DirectoryOwnership}; +use syntax::parse::token; +use syntax::ptr::P; +use syntax::sess::ParseSess; +use syntax::symbol::{kw, sym, Ident, Symbol}; +use syntax::{ThinVec, MACRO_ARGUMENTS}; +use syntax::tokenstream::{self, TokenStream}; +use syntax::visit::Visitor; + +use errors::{DiagnosticBuilder, DiagnosticId}; +use smallvec::{smallvec, SmallVec}; +use syntax_pos::{FileName, Span, MultiSpan, DUMMY_SP}; +use syntax_pos::hygiene::{AstPass, ExpnData, ExpnKind}; + +use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::sync::{self, Lrc}; +use std::iter; +use std::path::PathBuf; +use std::rc::Rc; +use std::default::Default; + +pub use syntax_pos::hygiene::MacroKind; + +#[derive(Debug,Clone)] +pub enum Annotatable { + Item(P<ast::Item>), + TraitItem(P<ast::TraitItem>), + ImplItem(P<ast::ImplItem>), + ForeignItem(P<ast::ForeignItem>), + Stmt(P<ast::Stmt>), + Expr(P<ast::Expr>), + Arm(ast::Arm), + Field(ast::Field), + FieldPat(ast::FieldPat), + GenericParam(ast::GenericParam), + Param(ast::Param), + StructField(ast::StructField), + Variant(ast::Variant), +} + +impl HasAttrs for Annotatable { + fn attrs(&self) -> &[Attribute] { + match *self { + Annotatable::Item(ref item) => &item.attrs, + Annotatable::TraitItem(ref trait_item) => &trait_item.attrs, + Annotatable::ImplItem(ref impl_item) => &impl_item.attrs, + Annotatable::ForeignItem(ref foreign_item) => &foreign_item.attrs, + Annotatable::Stmt(ref stmt) => stmt.attrs(), + Annotatable::Expr(ref expr) => &expr.attrs, + Annotatable::Arm(ref arm) => &arm.attrs, + Annotatable::Field(ref field) => &field.attrs, + Annotatable::FieldPat(ref fp) => &fp.attrs, + Annotatable::GenericParam(ref gp) => &gp.attrs, + Annotatable::Param(ref p) => &p.attrs, + Annotatable::StructField(ref sf) => &sf.attrs, + Annotatable::Variant(ref v) => &v.attrs(), + } + } + + fn visit_attrs<F: FnOnce(&mut Vec<Attribute>)>(&mut self, f: F) { + match self { + Annotatable::Item(item) => item.visit_attrs(f), + Annotatable::TraitItem(trait_item) => trait_item.visit_attrs(f), + Annotatable::ImplItem(impl_item) => impl_item.visit_attrs(f), + Annotatable::ForeignItem(foreign_item) => foreign_item.visit_attrs(f), + Annotatable::Stmt(stmt) => stmt.visit_attrs(f), + Annotatable::Expr(expr) => expr.visit_attrs(f), + Annotatable::Arm(arm) => arm.visit_attrs(f), + Annotatable::Field(field) => field.visit_attrs(f), + Annotatable::FieldPat(fp) => fp.visit_attrs(f), + Annotatable::GenericParam(gp) => gp.visit_attrs(f), + Annotatable::Param(p) => p.visit_attrs(f), + Annotatable::StructField(sf) => sf.visit_attrs(f), + Annotatable::Variant(v) => v.visit_attrs(f), + } + } +} + +impl Annotatable { + pub fn span(&self) -> Span { + match *self { + Annotatable::Item(ref item) => item.span, + Annotatable::TraitItem(ref trait_item) => trait_item.span, + Annotatable::ImplItem(ref impl_item) => impl_item.span, + Annotatable::ForeignItem(ref foreign_item) => foreign_item.span, + Annotatable::Stmt(ref stmt) => stmt.span, + Annotatable::Expr(ref expr) => expr.span, + Annotatable::Arm(ref arm) => arm.span, + Annotatable::Field(ref field) => field.span, + Annotatable::FieldPat(ref fp) => fp.pat.span, + Annotatable::GenericParam(ref gp) => gp.ident.span, + Annotatable::Param(ref p) => p.span, + Annotatable::StructField(ref sf) => sf.span, + Annotatable::Variant(ref v) => v.span, + } + } + + pub fn visit_with<'a, V: Visitor<'a>>(&'a self, visitor: &mut V) { + match self { + Annotatable::Item(item) => visitor.visit_item(item), + Annotatable::TraitItem(trait_item) => visitor.visit_trait_item(trait_item), + Annotatable::ImplItem(impl_item) => visitor.visit_impl_item(impl_item), + Annotatable::ForeignItem(foreign_item) => visitor.visit_foreign_item(foreign_item), + Annotatable::Stmt(stmt) => visitor.visit_stmt(stmt), + Annotatable::Expr(expr) => visitor.visit_expr(expr), + Annotatable::Arm(arm) => visitor.visit_arm(arm), + Annotatable::Field(field) => visitor.visit_field(field), + Annotatable::FieldPat(fp) => visitor.visit_field_pattern(fp), + Annotatable::GenericParam(gp) => visitor.visit_generic_param(gp), + Annotatable::Param(p) => visitor.visit_param(p), + Annotatable::StructField(sf) =>visitor.visit_struct_field(sf), + Annotatable::Variant(v) => visitor.visit_variant(v), + } + } + + pub fn expect_item(self) -> P<ast::Item> { + match self { + Annotatable::Item(i) => i, + _ => panic!("expected Item") + } + } + + pub fn map_item_or<F, G>(self, mut f: F, mut or: G) -> Annotatable + where F: FnMut(P<ast::Item>) -> P<ast::Item>, + G: FnMut(Annotatable) -> Annotatable + { + match self { + Annotatable::Item(i) => Annotatable::Item(f(i)), + _ => or(self) + } + } + + pub fn expect_trait_item(self) -> ast::TraitItem { + match self { + Annotatable::TraitItem(i) => i.into_inner(), + _ => panic!("expected Item") + } + } + + pub fn expect_impl_item(self) -> ast::ImplItem { + match self { + Annotatable::ImplItem(i) => i.into_inner(), + _ => panic!("expected Item") + } + } + + pub fn expect_foreign_item(self) -> ast::ForeignItem { + match self { + Annotatable::ForeignItem(i) => i.into_inner(), + _ => panic!("expected foreign item") + } + } + + pub fn expect_stmt(self) -> ast::Stmt { + match self { + Annotatable::Stmt(stmt) => stmt.into_inner(), + _ => panic!("expected statement"), + } + } + + pub fn expect_expr(self) -> P<ast::Expr> { + match self { + Annotatable::Expr(expr) => expr, + _ => panic!("expected expression"), + } + } + + pub fn expect_arm(self) -> ast::Arm { + match self { + Annotatable::Arm(arm) => arm, + _ => panic!("expected match arm") + } + } + + pub fn expect_field(self) -> ast::Field { + match self { + Annotatable::Field(field) => field, + _ => panic!("expected field") + } + } + + pub fn expect_field_pattern(self) -> ast::FieldPat { + match self { + Annotatable::FieldPat(fp) => fp, + _ => panic!("expected field pattern") + } + } + + pub fn expect_generic_param(self) -> ast::GenericParam { + match self { + Annotatable::GenericParam(gp) => gp, + _ => panic!("expected generic parameter") + } + } + + pub fn expect_param(self) -> ast::Param { + match self { + Annotatable::Param(param) => param, + _ => panic!("expected parameter") + } + } + + pub fn expect_struct_field(self) -> ast::StructField { + match self { + Annotatable::StructField(sf) => sf, + _ => panic!("expected struct field") + } + } + + pub fn expect_variant(self) -> ast::Variant { + match self { + Annotatable::Variant(v) => v, + _ => panic!("expected variant") + } + } + + pub fn derive_allowed(&self) -> bool { + match *self { + Annotatable::Item(ref item) => match item.kind { + ast::ItemKind::Struct(..) | + ast::ItemKind::Enum(..) | + ast::ItemKind::Union(..) => true, + _ => false, + }, + _ => false, + } + } +} + +// `meta_item` is the annotation, and `item` is the item being modified. +// FIXME Decorators should follow the same pattern too. +pub trait MultiItemModifier { + fn expand(&self, + ecx: &mut ExtCtxt<'_>, + span: Span, + meta_item: &ast::MetaItem, + item: Annotatable) + -> Vec<Annotatable>; +} + +impl<F, T> MultiItemModifier for F + where F: Fn(&mut ExtCtxt<'_>, Span, &ast::MetaItem, Annotatable) -> T, + T: Into<Vec<Annotatable>>, +{ + fn expand(&self, + ecx: &mut ExtCtxt<'_>, + span: Span, + meta_item: &ast::MetaItem, + item: Annotatable) + -> Vec<Annotatable> { + (*self)(ecx, span, meta_item, item).into() + } +} + +impl Into<Vec<Annotatable>> for Annotatable { + fn into(self) -> Vec<Annotatable> { + vec![self] + } +} + +pub trait ProcMacro { + fn expand<'cx>(&self, + ecx: &'cx mut ExtCtxt<'_>, + span: Span, + ts: TokenStream) + -> TokenStream; +} + +impl<F> ProcMacro for F + where F: Fn(TokenStream) -> TokenStream +{ + fn expand<'cx>(&self, + _ecx: &'cx mut ExtCtxt<'_>, + _span: Span, + ts: TokenStream) + -> TokenStream { + // FIXME setup implicit context in TLS before calling self. + (*self)(ts) + } +} + +pub trait AttrProcMacro { + fn expand<'cx>(&self, + ecx: &'cx mut ExtCtxt<'_>, + span: Span, + annotation: TokenStream, + annotated: TokenStream) + -> TokenStream; +} + +impl<F> AttrProcMacro for F + where F: Fn(TokenStream, TokenStream) -> TokenStream +{ + fn expand<'cx>(&self, + _ecx: &'cx mut ExtCtxt<'_>, + _span: Span, + annotation: TokenStream, + annotated: TokenStream) + -> TokenStream { + // FIXME setup implicit context in TLS before calling self. + (*self)(annotation, annotated) + } +} + +/// Represents a thing that maps token trees to Macro Results +pub trait TTMacroExpander { + fn expand<'cx>( + &self, + ecx: &'cx mut ExtCtxt<'_>, + span: Span, + input: TokenStream, + ) -> Box<dyn MacResult+'cx>; +} + +pub type MacroExpanderFn = + for<'cx> fn(&'cx mut ExtCtxt<'_>, Span, TokenStream) + -> Box<dyn MacResult+'cx>; + +impl<F> TTMacroExpander for F + where F: for<'cx> Fn(&'cx mut ExtCtxt<'_>, Span, TokenStream) + -> Box<dyn MacResult+'cx> +{ + fn expand<'cx>( + &self, + ecx: &'cx mut ExtCtxt<'_>, + span: Span, + mut input: TokenStream, + ) -> Box<dyn MacResult+'cx> { + struct AvoidInterpolatedIdents; + + impl MutVisitor for AvoidInterpolatedIdents { + fn visit_tt(&mut self, tt: &mut tokenstream::TokenTree) { + if let tokenstream::TokenTree::Token(token) = tt { + if let token::Interpolated(nt) = &token.kind { + if let token::NtIdent(ident, is_raw) = **nt { + *tt = tokenstream::TokenTree::token( + token::Ident(ident.name, is_raw), ident.span + ); + } + } + } + mut_visit::noop_visit_tt(tt, self) + } + + fn visit_mac(&mut self, mac: &mut ast::Mac) { + mut_visit::noop_visit_mac(mac, self) + } + } + AvoidInterpolatedIdents.visit_tts(&mut input); + (*self)(ecx, span, input) + } +} + +// Use a macro because forwarding to a simple function has type system issues +macro_rules! make_stmts_default { + ($me:expr) => { + $me.make_expr().map(|e| smallvec![ast::Stmt { + id: ast::DUMMY_NODE_ID, + span: e.span, + kind: ast::StmtKind::Expr(e), + }]) + } +} + +/// The result of a macro expansion. The return values of the various +/// methods are spliced into the AST at the callsite of the macro. +pub trait MacResult { + /// Creates an expression. + fn make_expr(self: Box<Self>) -> Option<P<ast::Expr>> { + None + } + /// Creates zero or more items. + fn make_items(self: Box<Self>) -> Option<SmallVec<[P<ast::Item>; 1]>> { + None + } + + /// Creates zero or more impl items. + fn make_impl_items(self: Box<Self>) -> Option<SmallVec<[ast::ImplItem; 1]>> { + None + } + + /// Creates zero or more trait items. + fn make_trait_items(self: Box<Self>) -> Option<SmallVec<[ast::TraitItem; 1]>> { + None + } + + /// Creates zero or more items in an `extern {}` block + fn make_foreign_items(self: Box<Self>) -> Option<SmallVec<[ast::ForeignItem; 1]>> { None } + + /// Creates a pattern. + fn make_pat(self: Box<Self>) -> Option<P<ast::Pat>> { + None + } + + /// Creates zero or more statements. + /// + /// By default this attempts to create an expression statement, + /// returning None if that fails. + fn make_stmts(self: Box<Self>) -> Option<SmallVec<[ast::Stmt; 1]>> { + make_stmts_default!(self) + } + + fn make_ty(self: Box<Self>) -> Option<P<ast::Ty>> { + None + } + + fn make_arms(self: Box<Self>) -> Option<SmallVec<[ast::Arm; 1]>> { + None + } + + fn make_fields(self: Box<Self>) -> Option<SmallVec<[ast::Field; 1]>> { + None + } + + fn make_field_patterns(self: Box<Self>) -> Option<SmallVec<[ast::FieldPat; 1]>> { + None + } + + fn make_generic_params(self: Box<Self>) -> Option<SmallVec<[ast::GenericParam; 1]>> { + None + } + + fn make_params(self: Box<Self>) -> Option<SmallVec<[ast::Param; 1]>> { + None + } + + fn make_struct_fields(self: Box<Self>) -> Option<SmallVec<[ast::StructField; 1]>> { + None + } + + fn make_variants(self: Box<Self>) -> Option<SmallVec<[ast::Variant; 1]>> { + None + } +} + +macro_rules! make_MacEager { + ( $( $fld:ident: $t:ty, )* ) => { + /// `MacResult` implementation for the common case where you've already + /// built each form of AST that you might return. + #[derive(Default)] + pub struct MacEager { + $( + pub $fld: Option<$t>, + )* + } + + impl MacEager { + $( + pub fn $fld(v: $t) -> Box<dyn MacResult> { + Box::new(MacEager { + $fld: Some(v), + ..Default::default() + }) + } + )* + } + } +} + +make_MacEager! { + expr: P<ast::Expr>, + pat: P<ast::Pat>, + items: SmallVec<[P<ast::Item>; 1]>, + impl_items: SmallVec<[ast::ImplItem; 1]>, + trait_items: SmallVec<[ast::TraitItem; 1]>, + foreign_items: SmallVec<[ast::ForeignItem; 1]>, + stmts: SmallVec<[ast::Stmt; 1]>, + ty: P<ast::Ty>, +} + +impl MacResult for MacEager { + fn make_expr(self: Box<Self>) -> Option<P<ast::Expr>> { + self.expr + } + + fn make_items(self: Box<Self>) -> Option<SmallVec<[P<ast::Item>; 1]>> { + self.items + } + + fn make_impl_items(self: Box<Self>) -> Option<SmallVec<[ast::ImplItem; 1]>> { + self.impl_items + } + + fn make_trait_items(self: Box<Self>) -> Option<SmallVec<[ast::TraitItem; 1]>> { + self.trait_items + } + + fn make_foreign_items(self: Box<Self>) -> Option<SmallVec<[ast::ForeignItem; 1]>> { + self.foreign_items + } + + fn make_stmts(self: Box<Self>) -> Option<SmallVec<[ast::Stmt; 1]>> { + match self.stmts.as_ref().map_or(0, |s| s.len()) { + 0 => make_stmts_default!(self), + _ => self.stmts, + } + } + + fn make_pat(self: Box<Self>) -> Option<P<ast::Pat>> { + if let Some(p) = self.pat { + return Some(p); + } + if let Some(e) = self.expr { + if let ast::ExprKind::Lit(_) = e.kind { + return Some(P(ast::Pat { + id: ast::DUMMY_NODE_ID, + span: e.span, + kind: PatKind::Lit(e), + })); + } + } + None + } + + fn make_ty(self: Box<Self>) -> Option<P<ast::Ty>> { + self.ty + } +} + +/// Fill-in macro expansion result, to allow compilation to continue +/// after hitting errors. +#[derive(Copy, Clone)] +pub struct DummyResult { + is_error: bool, + span: Span, +} + +impl DummyResult { + /// Creates a default MacResult that can be anything. + /// + /// Use this as a return value after hitting any errors and + /// calling `span_err`. + pub fn any(span: Span) -> Box<dyn MacResult+'static> { + Box::new(DummyResult { is_error: true, span }) + } + + /// Same as `any`, but must be a valid fragment, not error. + pub fn any_valid(span: Span) -> Box<dyn MacResult+'static> { + Box::new(DummyResult { is_error: false, span }) + } + + /// A plain dummy expression. + pub fn raw_expr(sp: Span, is_error: bool) -> P<ast::Expr> { + P(ast::Expr { + id: ast::DUMMY_NODE_ID, + kind: if is_error { ast::ExprKind::Err } else { ast::ExprKind::Tup(Vec::new()) }, + span: sp, + attrs: ThinVec::new(), + }) + } + + /// A plain dummy pattern. + pub fn raw_pat(sp: Span) -> ast::Pat { + ast::Pat { + id: ast::DUMMY_NODE_ID, + kind: PatKind::Wild, + span: sp, + } + } + + /// A plain dummy type. + pub fn raw_ty(sp: Span, is_error: bool) -> P<ast::Ty> { + P(ast::Ty { + id: ast::DUMMY_NODE_ID, + kind: if is_error { ast::TyKind::Err } else { ast::TyKind::Tup(Vec::new()) }, + span: sp + }) + } +} + +impl MacResult for DummyResult { + fn make_expr(self: Box<DummyResult>) -> Option<P<ast::Expr>> { + Some(DummyResult::raw_expr(self.span, self.is_error)) + } + + fn make_pat(self: Box<DummyResult>) -> Option<P<ast::Pat>> { + Some(P(DummyResult::raw_pat(self.span))) + } + + fn make_items(self: Box<DummyResult>) -> Option<SmallVec<[P<ast::Item>; 1]>> { + Some(SmallVec::new()) + } + + fn make_impl_items(self: Box<DummyResult>) -> Option<SmallVec<[ast::ImplItem; 1]>> { + Some(SmallVec::new()) + } + + fn make_trait_items(self: Box<DummyResult>) -> Option<SmallVec<[ast::TraitItem; 1]>> { + Some(SmallVec::new()) + } + + fn make_foreign_items(self: Box<Self>) -> Option<SmallVec<[ast::ForeignItem; 1]>> { + Some(SmallVec::new()) + } + + fn make_stmts(self: Box<DummyResult>) -> Option<SmallVec<[ast::Stmt; 1]>> { + Some(smallvec![ast::Stmt { + id: ast::DUMMY_NODE_ID, + kind: ast::StmtKind::Expr(DummyResult::raw_expr(self.span, self.is_error)), + span: self.span, + }]) + } + + fn make_ty(self: Box<DummyResult>) -> Option<P<ast::Ty>> { + Some(DummyResult::raw_ty(self.span, self.is_error)) + } + + fn make_arms(self: Box<DummyResult>) -> Option<SmallVec<[ast::Arm; 1]>> { + Some(SmallVec::new()) + } + + fn make_fields(self: Box<DummyResult>) -> Option<SmallVec<[ast::Field; 1]>> { + Some(SmallVec::new()) + } + + fn make_field_patterns(self: Box<DummyResult>) -> Option<SmallVec<[ast::FieldPat; 1]>> { + Some(SmallVec::new()) + } + + fn make_generic_params(self: Box<DummyResult>) -> Option<SmallVec<[ast::GenericParam; 1]>> { + Some(SmallVec::new()) + } + + fn make_params(self: Box<DummyResult>) -> Option<SmallVec<[ast::Param; 1]>> { + Some(SmallVec::new()) + } + + fn make_struct_fields(self: Box<DummyResult>) -> Option<SmallVec<[ast::StructField; 1]>> { + Some(SmallVec::new()) + } + + fn make_variants(self: Box<DummyResult>) -> Option<SmallVec<[ast::Variant; 1]>> { + Some(SmallVec::new()) + } +} + +/// A syntax extension kind. +pub enum SyntaxExtensionKind { + /// A token-based function-like macro. + Bang( + /// An expander with signature TokenStream -> TokenStream. + Box<dyn ProcMacro + sync::Sync + sync::Send>, + ), + + /// An AST-based function-like macro. + LegacyBang( + /// An expander with signature TokenStream -> AST. + Box<dyn TTMacroExpander + sync::Sync + sync::Send>, + ), + + /// A token-based attribute macro. + Attr( + /// An expander with signature (TokenStream, TokenStream) -> TokenStream. + /// The first TokenSteam is the attribute itself, the second is the annotated item. + /// The produced TokenSteam replaces the input TokenSteam. + Box<dyn AttrProcMacro + sync::Sync + sync::Send>, + ), + + /// An AST-based attribute macro. + LegacyAttr( + /// An expander with signature (AST, AST) -> AST. + /// The first AST fragment is the attribute itself, the second is the annotated item. + /// The produced AST fragment replaces the input AST fragment. + Box<dyn MultiItemModifier + sync::Sync + sync::Send>, + ), + + /// A trivial attribute "macro" that does nothing, + /// only keeps the attribute and marks it as inert, + /// thus making it ineligible for further expansion. + NonMacroAttr { + /// Suppresses the `unused_attributes` lint for this attribute. + mark_used: bool, + }, + + /// A token-based derive macro. + Derive( + /// An expander with signature TokenStream -> TokenStream (not yet). + /// The produced TokenSteam is appended to the input TokenSteam. + Box<dyn MultiItemModifier + sync::Sync + sync::Send>, + ), + + /// An AST-based derive macro. + LegacyDerive( + /// An expander with signature AST -> AST. + /// The produced AST fragment is appended to the input AST fragment. + Box<dyn MultiItemModifier + sync::Sync + sync::Send>, + ), +} + +/// A struct representing a macro definition in "lowered" form ready for expansion. +pub struct SyntaxExtension { + /// A syntax extension kind. + pub kind: SyntaxExtensionKind, + /// Span of the macro definition. + pub span: Span, + /// Whitelist of unstable features that are treated as stable inside this macro. + pub allow_internal_unstable: Option<Lrc<[Symbol]>>, + /// Suppresses the `unsafe_code` lint for code produced by this macro. + pub allow_internal_unsafe: bool, + /// Enables the macro helper hack (`ident!(...)` -> `$crate::ident!(...)`) for this macro. + pub local_inner_macros: bool, + /// The macro's stability info. + pub stability: Option<Stability>, + /// The macro's deprecation info. + pub deprecation: Option<Deprecation>, + /// Names of helper attributes registered by this macro. + pub helper_attrs: Vec<Symbol>, + /// Edition of the crate in which this macro is defined. + pub edition: Edition, + /// Built-in macros have a couple of special properties like availability + /// in `#[no_implicit_prelude]` modules, so we have to keep this flag. + pub is_builtin: bool, + /// We have to identify macros providing a `Copy` impl early for compatibility reasons. + pub is_derive_copy: bool, +} + +impl SyntaxExtension { + /// Returns which kind of macro calls this syntax extension. + pub fn macro_kind(&self) -> MacroKind { + match self.kind { + SyntaxExtensionKind::Bang(..) | + SyntaxExtensionKind::LegacyBang(..) => MacroKind::Bang, + SyntaxExtensionKind::Attr(..) | + SyntaxExtensionKind::LegacyAttr(..) | + SyntaxExtensionKind::NonMacroAttr { .. } => MacroKind::Attr, + SyntaxExtensionKind::Derive(..) | + SyntaxExtensionKind::LegacyDerive(..) => MacroKind::Derive, + } + } + + /// Constructs a syntax extension with default properties. + pub fn default(kind: SyntaxExtensionKind, edition: Edition) -> SyntaxExtension { + SyntaxExtension { + span: DUMMY_SP, + allow_internal_unstable: None, + allow_internal_unsafe: false, + local_inner_macros: false, + stability: None, + deprecation: None, + helper_attrs: Vec::new(), + edition, + is_builtin: false, + is_derive_copy: false, + kind, + } + } + + /// Constructs a syntax extension with the given properties + /// and other properties converted from attributes. + pub fn new( + sess: &ParseSess, + kind: SyntaxExtensionKind, + span: Span, + helper_attrs: Vec<Symbol>, + edition: Edition, + name: Name, + attrs: &[ast::Attribute], + ) -> SyntaxExtension { + let allow_internal_unstable = attr::allow_internal_unstable( + &attrs, &sess.span_diagnostic, + ).map(|features| features.collect::<Vec<Symbol>>().into()); + + let mut local_inner_macros = false; + if let Some(macro_export) = attr::find_by_name(attrs, sym::macro_export) { + if let Some(l) = macro_export.meta_item_list() { + local_inner_macros = attr::list_contains_name(&l, sym::local_inner_macros); + } + } + + let is_builtin = attr::contains_name(attrs, sym::rustc_builtin_macro); + + SyntaxExtension { + kind, + span, + allow_internal_unstable, + allow_internal_unsafe: attr::contains_name(attrs, sym::allow_internal_unsafe), + local_inner_macros, + stability: attr::find_stability(&sess, attrs, span), + deprecation: attr::find_deprecation(&sess, attrs, span), + helper_attrs, + edition, + is_builtin, + is_derive_copy: is_builtin && name == sym::Copy, + } + } + + pub fn dummy_bang(edition: Edition) -> SyntaxExtension { + fn expander<'cx>(_: &'cx mut ExtCtxt<'_>, span: Span, _: TokenStream) + -> Box<dyn MacResult + 'cx> { + DummyResult::any(span) + } + SyntaxExtension::default(SyntaxExtensionKind::LegacyBang(Box::new(expander)), edition) + } + + pub fn dummy_derive(edition: Edition) -> SyntaxExtension { + fn expander(_: &mut ExtCtxt<'_>, _: Span, _: &ast::MetaItem, _: Annotatable) + -> Vec<Annotatable> { + Vec::new() + } + SyntaxExtension::default(SyntaxExtensionKind::Derive(Box::new(expander)), edition) + } + + pub fn non_macro_attr(mark_used: bool, edition: Edition) -> SyntaxExtension { + SyntaxExtension::default(SyntaxExtensionKind::NonMacroAttr { mark_used }, edition) + } + + pub fn expn_data(&self, parent: ExpnId, call_site: Span, descr: Symbol) -> ExpnData { + ExpnData { + kind: ExpnKind::Macro(self.macro_kind(), descr), + parent, + call_site, + def_site: self.span, + allow_internal_unstable: self.allow_internal_unstable.clone(), + allow_internal_unsafe: self.allow_internal_unsafe, + local_inner_macros: self.local_inner_macros, + edition: self.edition, + } + } +} + +pub type NamedSyntaxExtension = (Name, SyntaxExtension); + +/// Result of resolving a macro invocation. +pub enum InvocationRes { + Single(Lrc<SyntaxExtension>), + DeriveContainer(Vec<Lrc<SyntaxExtension>>), +} + +/// Error type that denotes indeterminacy. +pub struct Indeterminate; + +bitflags::bitflags! { + /// Built-in derives that need some extra tracking beyond the usual macro functionality. + #[derive(Default)] + pub struct SpecialDerives: u8 { + const PARTIAL_EQ = 1 << 0; + const EQ = 1 << 1; + const COPY = 1 << 2; + } +} + +pub trait Resolver { + fn next_node_id(&mut self) -> NodeId; + + fn resolve_dollar_crates(&mut self); + fn visit_ast_fragment_with_placeholders(&mut self, expn_id: ExpnId, fragment: &AstFragment, + extra_placeholders: &[NodeId]); + fn register_builtin_macro(&mut self, ident: ast::Ident, ext: SyntaxExtension); + + fn expansion_for_ast_pass( + &mut self, + call_site: Span, + pass: AstPass, + features: &[Symbol], + parent_module_id: Option<NodeId>, + ) -> ExpnId; + + fn resolve_imports(&mut self); + + fn resolve_macro_invocation( + &mut self, invoc: &Invocation, eager_expansion_root: ExpnId, force: bool + ) -> Result<InvocationRes, Indeterminate>; + + fn check_unused_macros(&self); + + fn has_derives(&self, expn_id: ExpnId, derives: SpecialDerives) -> bool; + fn add_derives(&mut self, expn_id: ExpnId, derives: SpecialDerives); +} + +#[derive(Clone)] +pub struct ModuleData { + pub mod_path: Vec<ast::Ident>, + pub directory: PathBuf, +} + +#[derive(Clone)] +pub struct ExpansionData { + pub id: ExpnId, + pub depth: usize, + pub module: Rc<ModuleData>, + pub directory_ownership: DirectoryOwnership, + pub prior_type_ascription: Option<(Span, bool)>, +} + +/// One of these is made during expansion and incrementally updated as we go; +/// when a macro expansion occurs, the resulting nodes have the `backtrace() +/// -> expn_data` of their expansion context stored into their span. +pub struct ExtCtxt<'a> { + pub parse_sess: &'a ParseSess, + pub ecfg: expand::ExpansionConfig<'a>, + pub root_path: PathBuf, + pub resolver: &'a mut dyn Resolver, + pub current_expansion: ExpansionData, + pub expansions: FxHashMap<Span, Vec<String>>, +} + +impl<'a> ExtCtxt<'a> { + pub fn new(parse_sess: &'a ParseSess, + ecfg: expand::ExpansionConfig<'a>, + resolver: &'a mut dyn Resolver) + -> ExtCtxt<'a> { + ExtCtxt { + parse_sess, + ecfg, + root_path: PathBuf::new(), + resolver, + current_expansion: ExpansionData { + id: ExpnId::root(), + depth: 0, + module: Rc::new(ModuleData { mod_path: Vec::new(), directory: PathBuf::new() }), + directory_ownership: DirectoryOwnership::Owned { relative: None }, + prior_type_ascription: None, + }, + expansions: FxHashMap::default(), + } + } + + /// Returns a `Folder` for deeply expanding all macros in an AST node. + pub fn expander<'b>(&'b mut self) -> expand::MacroExpander<'b, 'a> { + expand::MacroExpander::new(self, false) + } + + /// Returns a `Folder` that deeply expands all macros and assigns all `NodeId`s in an AST node. + /// Once `NodeId`s are assigned, the node may not be expanded, removed, or otherwise modified. + pub fn monotonic_expander<'b>(&'b mut self) -> expand::MacroExpander<'b, 'a> { + expand::MacroExpander::new(self, true) + } + pub fn new_parser_from_tts(&self, stream: TokenStream) -> parser::Parser<'a> { + parse::stream_to_parser(self.parse_sess, stream, MACRO_ARGUMENTS) + } + pub fn source_map(&self) -> &'a SourceMap { self.parse_sess.source_map() } + pub fn parse_sess(&self) -> &'a ParseSess { self.parse_sess } + pub fn cfg(&self) -> &ast::CrateConfig { &self.parse_sess.config } + pub fn call_site(&self) -> Span { + self.current_expansion.id.expn_data().call_site + } + + /// Equivalent of `Span::def_site` from the proc macro API, + /// except that the location is taken from the span passed as an argument. + pub fn with_def_site_ctxt(&self, span: Span) -> Span { + span.with_def_site_ctxt(self.current_expansion.id) + } + + /// Equivalent of `Span::call_site` from the proc macro API, + /// except that the location is taken from the span passed as an argument. + pub fn with_call_site_ctxt(&self, span: Span) -> Span { + span.with_call_site_ctxt(self.current_expansion.id) + } + + /// Equivalent of `Span::mixed_site` from the proc macro API, + /// except that the location is taken from the span passed as an argument. + pub fn with_mixed_site_ctxt(&self, span: Span) -> Span { + span.with_mixed_site_ctxt(self.current_expansion.id) + } + + /// Returns span for the macro which originally caused the current expansion to happen. + /// + /// Stops backtracing at include! boundary. + pub fn expansion_cause(&self) -> Option<Span> { + let mut expn_id = self.current_expansion.id; + let mut last_macro = None; + loop { + let expn_data = expn_id.expn_data(); + // Stop going up the backtrace once include! is encountered + if expn_data.is_root() || expn_data.kind.descr() == sym::include { + break; + } + expn_id = expn_data.call_site.ctxt().outer_expn(); + last_macro = Some(expn_data.call_site); + } + last_macro + } + + pub fn struct_span_warn<S: Into<MultiSpan>>(&self, + sp: S, + msg: &str) + -> DiagnosticBuilder<'a> { + self.parse_sess.span_diagnostic.struct_span_warn(sp, msg) + } + pub fn struct_span_err<S: Into<MultiSpan>>(&self, + sp: S, + msg: &str) + -> DiagnosticBuilder<'a> { + self.parse_sess.span_diagnostic.struct_span_err(sp, msg) + } + pub fn struct_span_fatal<S: Into<MultiSpan>>(&self, + sp: S, + msg: &str) + -> DiagnosticBuilder<'a> { + self.parse_sess.span_diagnostic.struct_span_fatal(sp, msg) + } + + /// Emit `msg` attached to `sp`, and stop compilation immediately. + /// + /// `span_err` should be strongly preferred where-ever possible: + /// this should *only* be used when: + /// + /// - continuing has a high risk of flow-on errors (e.g., errors in + /// declaring a macro would cause all uses of that macro to + /// complain about "undefined macro"), or + /// - there is literally nothing else that can be done (however, + /// in most cases one can construct a dummy expression/item to + /// substitute; we never hit resolve/type-checking so the dummy + /// value doesn't have to match anything) + pub fn span_fatal<S: Into<MultiSpan>>(&self, sp: S, msg: &str) -> ! { + self.parse_sess.span_diagnostic.span_fatal(sp, msg).raise(); + } + + /// Emit `msg` attached to `sp`, without immediately stopping + /// compilation. + /// + /// Compilation will be stopped in the near future (at the end of + /// the macro expansion phase). + pub fn span_err<S: Into<MultiSpan>>(&self, sp: S, msg: &str) { + self.parse_sess.span_diagnostic.span_err(sp, msg); + } + pub fn span_err_with_code<S: Into<MultiSpan>>(&self, sp: S, msg: &str, code: DiagnosticId) { + self.parse_sess.span_diagnostic.span_err_with_code(sp, msg, code); + } + pub fn span_warn<S: Into<MultiSpan>>(&self, sp: S, msg: &str) { + self.parse_sess.span_diagnostic.span_warn(sp, msg); + } + pub fn span_bug<S: Into<MultiSpan>>(&self, sp: S, msg: &str) -> ! { + self.parse_sess.span_diagnostic.span_bug(sp, msg); + } + pub fn trace_macros_diag(&mut self) { + for (sp, notes) in self.expansions.iter() { + let mut db = self.parse_sess.span_diagnostic.span_note_diag(*sp, "trace_macro"); + for note in notes { + db.note(note); + } + db.emit(); + } + // Fixme: does this result in errors? + self.expansions.clear(); + } + pub fn bug(&self, msg: &str) -> ! { + self.parse_sess.span_diagnostic.bug(msg); + } + pub fn trace_macros(&self) -> bool { + self.ecfg.trace_mac + } + pub fn set_trace_macros(&mut self, x: bool) { + self.ecfg.trace_mac = x + } + pub fn ident_of(&self, st: &str, sp: Span) -> ast::Ident { + ast::Ident::from_str_and_span(st, sp) + } + pub fn std_path(&self, components: &[Symbol]) -> Vec<ast::Ident> { + let def_site = self.with_def_site_ctxt(DUMMY_SP); + iter::once(Ident::new(kw::DollarCrate, def_site)) + .chain(components.iter().map(|&s| Ident::with_dummy_span(s))) + .collect() + } + pub fn name_of(&self, st: &str) -> ast::Name { + Symbol::intern(st) + } + + pub fn check_unused_macros(&self) { + self.resolver.check_unused_macros(); + } + + /// Resolves a path mentioned inside Rust code. + /// + /// This unifies the logic used for resolving `include_X!`, and `#[doc(include)]` file paths. + /// + /// Returns an absolute path to the file that `path` refers to. + pub fn resolve_path(&self, path: impl Into<PathBuf>, span: Span) -> PathBuf { + let path = path.into(); + + // Relative paths are resolved relative to the file in which they are found + // after macro expansion (that is, they are unhygienic). + if !path.is_absolute() { + let callsite = span.source_callsite(); + let mut result = match self.source_map().span_to_unmapped_path(callsite) { + FileName::Real(path) => path, + FileName::DocTest(path, _) => path, + other => panic!("cannot resolve relative path in non-file source `{}`", other), + }; + result.pop(); + result.push(path); + result + } else { + path + } + } +} + +/// Extracts a string literal from the macro expanded version of `expr`, +/// emitting `err_msg` if `expr` is not a string literal. This does not stop +/// compilation on error, merely emits a non-fatal error and returns `None`. +pub fn expr_to_spanned_string<'a>( + cx: &'a mut ExtCtxt<'_>, + expr: P<ast::Expr>, + err_msg: &str, +) -> Result<(Symbol, ast::StrStyle, Span), Option<DiagnosticBuilder<'a>>> { + // Perform eager expansion on the expression. + // We want to be able to handle e.g., `concat!("foo", "bar")`. + let expr = cx.expander().fully_expand_fragment(AstFragment::Expr(expr)).make_expr(); + + Err(match expr.kind { + ast::ExprKind::Lit(ref l) => match l.kind { + ast::LitKind::Str(s, style) => return Ok((s, style, expr.span)), + ast::LitKind::Err(_) => None, + _ => Some(cx.struct_span_err(l.span, err_msg)) + }, + ast::ExprKind::Err => None, + _ => Some(cx.struct_span_err(expr.span, err_msg)) + }) +} + +pub fn expr_to_string(cx: &mut ExtCtxt<'_>, expr: P<ast::Expr>, err_msg: &str) + -> Option<(Symbol, ast::StrStyle)> { + expr_to_spanned_string(cx, expr, err_msg) + .map_err(|err| err.map(|mut err| err.emit())) + .ok() + .map(|(symbol, style, _)| (symbol, style)) +} + +/// Non-fatally assert that `tts` is empty. Note that this function +/// returns even when `tts` is non-empty, macros that *need* to stop +/// compilation should call +/// `cx.parse_sess.span_diagnostic.abort_if_errors()` (this should be +/// done as rarely as possible). +pub fn check_zero_tts(cx: &ExtCtxt<'_>, + sp: Span, + tts: TokenStream, + name: &str) { + if !tts.is_empty() { + cx.span_err(sp, &format!("{} takes no arguments", name)); + } +} + +/// Interpreting `tts` as a comma-separated sequence of expressions, +/// expect exactly one string literal, or emit an error and return `None`. +pub fn get_single_str_from_tts(cx: &mut ExtCtxt<'_>, + sp: Span, + tts: TokenStream, + name: &str) + -> Option<String> { + let mut p = cx.new_parser_from_tts(tts); + if p.token == token::Eof { + cx.span_err(sp, &format!("{} takes 1 argument", name)); + return None + } + let ret = panictry!(p.parse_expr()); + let _ = p.eat(&token::Comma); + + if p.token != token::Eof { + cx.span_err(sp, &format!("{} takes 1 argument", name)); + } + expr_to_string(cx, ret, "argument must be a string literal").map(|(s, _)| { + s.to_string() + }) +} + +/// Extracts comma-separated expressions from `tts`. If there is a +/// parsing error, emit a non-fatal error and return `None`. +pub fn get_exprs_from_tts(cx: &mut ExtCtxt<'_>, + sp: Span, + tts: TokenStream) -> Option<Vec<P<ast::Expr>>> { + let mut p = cx.new_parser_from_tts(tts); + let mut es = Vec::new(); + while p.token != token::Eof { + let expr = panictry!(p.parse_expr()); + + // Perform eager expansion on the expression. + // We want to be able to handle e.g., `concat!("foo", "bar")`. + let expr = cx.expander().fully_expand_fragment(AstFragment::Expr(expr)).make_expr(); + + es.push(expr); + if p.eat(&token::Comma) { + continue; + } + if p.token != token::Eof { + cx.span_err(sp, "expected token: `,`"); + return None; + } + } + Some(es) +} diff --git a/src/libsyntax_expand/build.rs b/src/libsyntax_expand/build.rs new file mode 100644 index 00000000000..105ffe3ee8a --- /dev/null +++ b/src/libsyntax_expand/build.rs @@ -0,0 +1,640 @@ +use crate::base::ExtCtxt; + +use syntax::ast::{self, Ident, Expr, BlockCheckMode, UnOp, PatKind}; +use syntax::attr; +use syntax::source_map::{respan, Spanned}; +use syntax::ptr::P; +use syntax::symbol::{kw, sym, Symbol}; +use syntax::ThinVec; + +use syntax_pos::{Pos, Span}; + +impl<'a> ExtCtxt<'a> { + pub fn path(&self, span: Span, strs: Vec<ast::Ident> ) -> ast::Path { + self.path_all(span, false, strs, vec![]) + } + pub fn path_ident(&self, span: Span, id: ast::Ident) -> ast::Path { + self.path(span, vec![id]) + } + pub fn path_global(&self, span: Span, strs: Vec<ast::Ident> ) -> ast::Path { + self.path_all(span, true, strs, vec![]) + } + pub fn path_all(&self, + span: Span, + global: bool, + mut idents: Vec<ast::Ident> , + args: Vec<ast::GenericArg>) + -> ast::Path { + assert!(!idents.is_empty()); + let add_root = global && !idents[0].is_path_segment_keyword(); + let mut segments = Vec::with_capacity(idents.len() + add_root as usize); + if add_root { + segments.push(ast::PathSegment::path_root(span)); + } + let last_ident = idents.pop().unwrap(); + segments.extend(idents.into_iter().map(|ident| { + ast::PathSegment::from_ident(ident.with_span_pos(span)) + })); + let args = if !args.is_empty() { + ast::AngleBracketedArgs { args, constraints: Vec::new(), span }.into() + } else { + None + }; + segments.push(ast::PathSegment { + ident: last_ident.with_span_pos(span), + id: ast::DUMMY_NODE_ID, + args, + }); + ast::Path { span, segments } + } + + pub fn ty_mt(&self, ty: P<ast::Ty>, mutbl: ast::Mutability) -> ast::MutTy { + ast::MutTy { + ty, + mutbl, + } + } + + pub fn ty(&self, span: Span, kind: ast::TyKind) -> P<ast::Ty> { + P(ast::Ty { + id: ast::DUMMY_NODE_ID, + span, + kind, + }) + } + + pub fn ty_path(&self, path: ast::Path) -> P<ast::Ty> { + self.ty(path.span, ast::TyKind::Path(None, path)) + } + + // Might need to take bounds as an argument in the future, if you ever want + // to generate a bounded existential trait type. + pub fn ty_ident(&self, span: Span, ident: ast::Ident) + -> P<ast::Ty> { + self.ty_path(self.path_ident(span, ident)) + } + + pub fn anon_const(&self, span: Span, kind: ast::ExprKind) -> ast::AnonConst { + ast::AnonConst { + id: ast::DUMMY_NODE_ID, + value: P(ast::Expr { + id: ast::DUMMY_NODE_ID, + kind, + span, + attrs: ThinVec::new(), + }) + } + } + + pub fn const_ident(&self, span: Span, ident: ast::Ident) -> ast::AnonConst { + self.anon_const(span, ast::ExprKind::Path(None, self.path_ident(span, ident))) + } + + pub fn ty_rptr(&self, + span: Span, + ty: P<ast::Ty>, + lifetime: Option<ast::Lifetime>, + mutbl: ast::Mutability) + -> P<ast::Ty> { + self.ty(span, + ast::TyKind::Rptr(lifetime, self.ty_mt(ty, mutbl))) + } + + pub fn ty_ptr(&self, + span: Span, + ty: P<ast::Ty>, + mutbl: ast::Mutability) + -> P<ast::Ty> { + self.ty(span, + ast::TyKind::Ptr(self.ty_mt(ty, mutbl))) + } + + pub fn typaram(&self, + span: Span, + ident: ast::Ident, + attrs: Vec<ast::Attribute>, + bounds: ast::GenericBounds, + default: Option<P<ast::Ty>>) -> ast::GenericParam { + ast::GenericParam { + ident: ident.with_span_pos(span), + id: ast::DUMMY_NODE_ID, + attrs: attrs.into(), + bounds, + kind: ast::GenericParamKind::Type { + default, + }, + is_placeholder: false + } + } + + pub fn trait_ref(&self, path: ast::Path) -> ast::TraitRef { + ast::TraitRef { + path, + ref_id: ast::DUMMY_NODE_ID, + } + } + + pub fn poly_trait_ref(&self, span: Span, path: ast::Path) -> ast::PolyTraitRef { + ast::PolyTraitRef { + bound_generic_params: Vec::new(), + trait_ref: self.trait_ref(path), + span, + } + } + + pub fn trait_bound(&self, path: ast::Path) -> ast::GenericBound { + ast::GenericBound::Trait(self.poly_trait_ref(path.span, path), + ast::TraitBoundModifier::None) + } + + pub fn lifetime(&self, span: Span, ident: ast::Ident) -> ast::Lifetime { + ast::Lifetime { id: ast::DUMMY_NODE_ID, ident: ident.with_span_pos(span) } + } + + pub fn lifetime_def(&self, + span: Span, + ident: ast::Ident, + attrs: Vec<ast::Attribute>, + bounds: ast::GenericBounds) + -> ast::GenericParam { + let lifetime = self.lifetime(span, ident); + ast::GenericParam { + ident: lifetime.ident, + id: lifetime.id, + attrs: attrs.into(), + bounds, + kind: ast::GenericParamKind::Lifetime, + is_placeholder: false + } + } + + pub fn stmt_expr(&self, expr: P<ast::Expr>) -> ast::Stmt { + ast::Stmt { + id: ast::DUMMY_NODE_ID, + span: expr.span, + kind: ast::StmtKind::Expr(expr), + } + } + + pub fn stmt_let(&self, sp: Span, mutbl: bool, ident: ast::Ident, + ex: P<ast::Expr>) -> ast::Stmt { + let pat = if mutbl { + let binding_mode = ast::BindingMode::ByValue(ast::Mutability::Mutable); + self.pat_ident_binding_mode(sp, ident, binding_mode) + } else { + self.pat_ident(sp, ident) + }; + let local = P(ast::Local { + pat, + ty: None, + init: Some(ex), + id: ast::DUMMY_NODE_ID, + span: sp, + attrs: ThinVec::new(), + }); + ast::Stmt { + id: ast::DUMMY_NODE_ID, + kind: ast::StmtKind::Local(local), + span: sp, + } + } + + // Generates `let _: Type;`, which is usually used for type assertions. + pub fn stmt_let_type_only(&self, span: Span, ty: P<ast::Ty>) -> ast::Stmt { + let local = P(ast::Local { + pat: self.pat_wild(span), + ty: Some(ty), + init: None, + id: ast::DUMMY_NODE_ID, + span, + attrs: ThinVec::new(), + }); + ast::Stmt { + id: ast::DUMMY_NODE_ID, + kind: ast::StmtKind::Local(local), + span, + } + } + + pub fn stmt_item(&self, sp: Span, item: P<ast::Item>) -> ast::Stmt { + ast::Stmt { + id: ast::DUMMY_NODE_ID, + kind: ast::StmtKind::Item(item), + span: sp, + } + } + + pub fn block_expr(&self, expr: P<ast::Expr>) -> P<ast::Block> { + self.block(expr.span, vec![ast::Stmt { + id: ast::DUMMY_NODE_ID, + span: expr.span, + kind: ast::StmtKind::Expr(expr), + }]) + } + pub fn block(&self, span: Span, stmts: Vec<ast::Stmt>) -> P<ast::Block> { + P(ast::Block { + stmts, + id: ast::DUMMY_NODE_ID, + rules: BlockCheckMode::Default, + span, + }) + } + + pub fn expr(&self, span: Span, kind: ast::ExprKind) -> P<ast::Expr> { + P(ast::Expr { + id: ast::DUMMY_NODE_ID, + kind, + span, + attrs: ThinVec::new(), + }) + } + + pub fn expr_path(&self, path: ast::Path) -> P<ast::Expr> { + self.expr(path.span, ast::ExprKind::Path(None, path)) + } + + pub fn expr_ident(&self, span: Span, id: ast::Ident) -> P<ast::Expr> { + self.expr_path(self.path_ident(span, id)) + } + pub fn expr_self(&self, span: Span) -> P<ast::Expr> { + self.expr_ident(span, Ident::with_dummy_span(kw::SelfLower)) + } + + pub fn expr_binary(&self, sp: Span, op: ast::BinOpKind, + lhs: P<ast::Expr>, rhs: P<ast::Expr>) -> P<ast::Expr> { + self.expr(sp, ast::ExprKind::Binary(Spanned { node: op, span: sp }, lhs, rhs)) + } + + pub fn expr_deref(&self, sp: Span, e: P<ast::Expr>) -> P<ast::Expr> { + self.expr(sp, ast::ExprKind::Unary(UnOp::Deref, e)) + } + + pub fn expr_addr_of(&self, sp: Span, e: P<ast::Expr>) -> P<ast::Expr> { + self.expr(sp, ast::ExprKind::AddrOf(ast::Mutability::Immutable, e)) + } + + pub fn expr_call( + &self, span: Span, expr: P<ast::Expr>, args: Vec<P<ast::Expr>>, + ) -> P<ast::Expr> { + self.expr(span, ast::ExprKind::Call(expr, args)) + } + pub fn expr_call_ident(&self, span: Span, id: ast::Ident, + args: Vec<P<ast::Expr>>) -> P<ast::Expr> { + self.expr(span, ast::ExprKind::Call(self.expr_ident(span, id), args)) + } + pub fn expr_call_global(&self, sp: Span, fn_path: Vec<ast::Ident> , + args: Vec<P<ast::Expr>> ) -> P<ast::Expr> { + let pathexpr = self.expr_path(self.path_global(sp, fn_path)); + self.expr_call(sp, pathexpr, args) + } + pub fn expr_method_call(&self, span: Span, + expr: P<ast::Expr>, + ident: ast::Ident, + mut args: Vec<P<ast::Expr>> ) -> P<ast::Expr> { + args.insert(0, expr); + let segment = ast::PathSegment::from_ident(ident.with_span_pos(span)); + self.expr(span, ast::ExprKind::MethodCall(segment, args)) + } + pub fn expr_block(&self, b: P<ast::Block>) -> P<ast::Expr> { + self.expr(b.span, ast::ExprKind::Block(b, None)) + } + pub fn field_imm(&self, span: Span, ident: Ident, e: P<ast::Expr>) -> ast::Field { + ast::Field { + ident: ident.with_span_pos(span), + expr: e, + span, + is_shorthand: false, + attrs: ThinVec::new(), + id: ast::DUMMY_NODE_ID, + is_placeholder: false, + } + } + pub fn expr_struct( + &self, span: Span, path: ast::Path, fields: Vec<ast::Field> + ) -> P<ast::Expr> { + self.expr(span, ast::ExprKind::Struct(path, fields, None)) + } + pub fn expr_struct_ident(&self, span: Span, + id: ast::Ident, fields: Vec<ast::Field>) -> P<ast::Expr> { + self.expr_struct(span, self.path_ident(span, id), fields) + } + + pub fn expr_lit(&self, span: Span, lit_kind: ast::LitKind) -> P<ast::Expr> { + let lit = ast::Lit::from_lit_kind(lit_kind, span); + self.expr(span, ast::ExprKind::Lit(lit)) + } + pub fn expr_usize(&self, span: Span, i: usize) -> P<ast::Expr> { + self.expr_lit(span, ast::LitKind::Int(i as u128, + ast::LitIntType::Unsigned(ast::UintTy::Usize))) + } + pub fn expr_u32(&self, sp: Span, u: u32) -> P<ast::Expr> { + self.expr_lit(sp, ast::LitKind::Int(u as u128, + ast::LitIntType::Unsigned(ast::UintTy::U32))) + } + pub fn expr_bool(&self, sp: Span, value: bool) -> P<ast::Expr> { + self.expr_lit(sp, ast::LitKind::Bool(value)) + } + + pub fn expr_vec(&self, sp: Span, exprs: Vec<P<ast::Expr>>) -> P<ast::Expr> { + self.expr(sp, ast::ExprKind::Array(exprs)) + } + pub fn expr_vec_slice(&self, sp: Span, exprs: Vec<P<ast::Expr>>) -> P<ast::Expr> { + self.expr_addr_of(sp, self.expr_vec(sp, exprs)) + } + pub fn expr_str(&self, sp: Span, s: Symbol) -> P<ast::Expr> { + self.expr_lit(sp, ast::LitKind::Str(s, ast::StrStyle::Cooked)) + } + + pub fn expr_cast(&self, sp: Span, expr: P<ast::Expr>, ty: P<ast::Ty>) -> P<ast::Expr> { + self.expr(sp, ast::ExprKind::Cast(expr, ty)) + } + + pub fn expr_some(&self, sp: Span, expr: P<ast::Expr>) -> P<ast::Expr> { + let some = self.std_path(&[sym::option, sym::Option, sym::Some]); + self.expr_call_global(sp, some, vec![expr]) + } + + pub fn expr_tuple(&self, sp: Span, exprs: Vec<P<ast::Expr>>) -> P<ast::Expr> { + self.expr(sp, ast::ExprKind::Tup(exprs)) + } + + pub fn expr_fail(&self, span: Span, msg: Symbol) -> P<ast::Expr> { + let loc = self.source_map().lookup_char_pos(span.lo()); + let expr_file = self.expr_str(span, Symbol::intern(&loc.file.name.to_string())); + let expr_line = self.expr_u32(span, loc.line as u32); + let expr_col = self.expr_u32(span, loc.col.to_usize() as u32 + 1); + let expr_loc_tuple = self.expr_tuple(span, vec![expr_file, expr_line, expr_col]); + let expr_loc_ptr = self.expr_addr_of(span, expr_loc_tuple); + self.expr_call_global( + span, + [sym::std, sym::rt, sym::begin_panic].iter().map(|s| Ident::new(*s, span)).collect(), + vec![ + self.expr_str(span, msg), + expr_loc_ptr]) + } + + pub fn expr_unreachable(&self, span: Span) -> P<ast::Expr> { + self.expr_fail(span, Symbol::intern("internal error: entered unreachable code")) + } + + pub fn expr_ok(&self, sp: Span, expr: P<ast::Expr>) -> P<ast::Expr> { + let ok = self.std_path(&[sym::result, sym::Result, sym::Ok]); + self.expr_call_global(sp, ok, vec![expr]) + } + + pub fn expr_try(&self, sp: Span, head: P<ast::Expr>) -> P<ast::Expr> { + let ok = self.std_path(&[sym::result, sym::Result, sym::Ok]); + let ok_path = self.path_global(sp, ok); + let err = self.std_path(&[sym::result, sym::Result, sym::Err]); + let err_path = self.path_global(sp, err); + + let binding_variable = self.ident_of("__try_var", sp); + let binding_pat = self.pat_ident(sp, binding_variable); + let binding_expr = self.expr_ident(sp, binding_variable); + + // `Ok(__try_var)` pattern + let ok_pat = self.pat_tuple_struct(sp, ok_path, vec![binding_pat.clone()]); + + // `Err(__try_var)` (pattern and expression respectively) + let err_pat = self.pat_tuple_struct(sp, err_path.clone(), vec![binding_pat]); + let err_inner_expr = self.expr_call(sp, self.expr_path(err_path), + vec![binding_expr.clone()]); + // `return Err(__try_var)` + let err_expr = self.expr(sp, ast::ExprKind::Ret(Some(err_inner_expr))); + + // `Ok(__try_var) => __try_var` + let ok_arm = self.arm(sp, ok_pat, binding_expr); + // `Err(__try_var) => return Err(__try_var)` + let err_arm = self.arm(sp, err_pat, err_expr); + + // `match head { Ok() => ..., Err() => ... }` + self.expr_match(sp, head, vec![ok_arm, err_arm]) + } + + + pub fn pat(&self, span: Span, kind: PatKind) -> P<ast::Pat> { + P(ast::Pat { id: ast::DUMMY_NODE_ID, kind, span }) + } + pub fn pat_wild(&self, span: Span) -> P<ast::Pat> { + self.pat(span, PatKind::Wild) + } + pub fn pat_lit(&self, span: Span, expr: P<ast::Expr>) -> P<ast::Pat> { + self.pat(span, PatKind::Lit(expr)) + } + pub fn pat_ident(&self, span: Span, ident: ast::Ident) -> P<ast::Pat> { + let binding_mode = ast::BindingMode::ByValue(ast::Mutability::Immutable); + self.pat_ident_binding_mode(span, ident, binding_mode) + } + + pub fn pat_ident_binding_mode(&self, + span: Span, + ident: ast::Ident, + bm: ast::BindingMode) -> P<ast::Pat> { + let pat = PatKind::Ident(bm, ident.with_span_pos(span), None); + self.pat(span, pat) + } + pub fn pat_path(&self, span: Span, path: ast::Path) -> P<ast::Pat> { + self.pat(span, PatKind::Path(None, path)) + } + pub fn pat_tuple_struct(&self, span: Span, path: ast::Path, + subpats: Vec<P<ast::Pat>>) -> P<ast::Pat> { + self.pat(span, PatKind::TupleStruct(path, subpats)) + } + pub fn pat_struct(&self, span: Span, path: ast::Path, + field_pats: Vec<ast::FieldPat>) -> P<ast::Pat> { + self.pat(span, PatKind::Struct(path, field_pats, false)) + } + pub fn pat_tuple(&self, span: Span, pats: Vec<P<ast::Pat>>) -> P<ast::Pat> { + self.pat(span, PatKind::Tuple(pats)) + } + + pub fn pat_some(&self, span: Span, pat: P<ast::Pat>) -> P<ast::Pat> { + let some = self.std_path(&[sym::option, sym::Option, sym::Some]); + let path = self.path_global(span, some); + self.pat_tuple_struct(span, path, vec![pat]) + } + + pub fn pat_none(&self, span: Span) -> P<ast::Pat> { + let some = self.std_path(&[sym::option, sym::Option, sym::None]); + let path = self.path_global(span, some); + self.pat_path(span, path) + } + + pub fn pat_ok(&self, span: Span, pat: P<ast::Pat>) -> P<ast::Pat> { + let some = self.std_path(&[sym::result, sym::Result, sym::Ok]); + let path = self.path_global(span, some); + self.pat_tuple_struct(span, path, vec![pat]) + } + + pub fn pat_err(&self, span: Span, pat: P<ast::Pat>) -> P<ast::Pat> { + let some = self.std_path(&[sym::result, sym::Result, sym::Err]); + let path = self.path_global(span, some); + self.pat_tuple_struct(span, path, vec![pat]) + } + + pub fn arm(&self, span: Span, pat: P<ast::Pat>, expr: P<ast::Expr>) -> ast::Arm { + ast::Arm { + attrs: vec![], + pat, + guard: None, + body: expr, + span, + id: ast::DUMMY_NODE_ID, + is_placeholder: false, + } + } + + pub fn arm_unreachable(&self, span: Span) -> ast::Arm { + self.arm(span, self.pat_wild(span), self.expr_unreachable(span)) + } + + pub fn expr_match(&self, span: Span, arg: P<ast::Expr>, arms: Vec<ast::Arm>) -> P<Expr> { + self.expr(span, ast::ExprKind::Match(arg, arms)) + } + + pub fn expr_if(&self, span: Span, cond: P<ast::Expr>, + then: P<ast::Expr>, els: Option<P<ast::Expr>>) -> P<ast::Expr> { + let els = els.map(|x| self.expr_block(self.block_expr(x))); + self.expr(span, ast::ExprKind::If(cond, self.block_expr(then), els)) + } + + pub fn lambda_fn_decl(&self, + span: Span, + fn_decl: P<ast::FnDecl>, + body: P<ast::Expr>, + fn_decl_span: Span) // span of the `|...|` part + -> P<ast::Expr> { + self.expr(span, ast::ExprKind::Closure(ast::CaptureBy::Ref, + ast::IsAsync::NotAsync, + ast::Movability::Movable, + fn_decl, + body, + fn_decl_span)) + } + + pub fn lambda(&self, + span: Span, + ids: Vec<ast::Ident>, + body: P<ast::Expr>) + -> P<ast::Expr> { + let fn_decl = self.fn_decl( + ids.iter().map(|id| self.param(span, *id, self.ty(span, ast::TyKind::Infer))).collect(), + ast::FunctionRetTy::Default(span)); + + // FIXME -- We are using `span` as the span of the `|...|` + // part of the lambda, but it probably (maybe?) corresponds to + // the entire lambda body. Probably we should extend the API + // here, but that's not entirely clear. + self.expr(span, ast::ExprKind::Closure(ast::CaptureBy::Ref, + ast::IsAsync::NotAsync, + ast::Movability::Movable, + fn_decl, + body, + span)) + } + + pub fn lambda0(&self, span: Span, body: P<ast::Expr>) -> P<ast::Expr> { + self.lambda(span, Vec::new(), body) + } + + pub fn lambda1(&self, span: Span, body: P<ast::Expr>, ident: ast::Ident) -> P<ast::Expr> { + self.lambda(span, vec![ident], body) + } + + pub fn lambda_stmts_1(&self, span: Span, stmts: Vec<ast::Stmt>, + ident: ast::Ident) -> P<ast::Expr> { + self.lambda1(span, self.expr_block(self.block(span, stmts)), ident) + } + + pub fn param(&self, span: Span, ident: ast::Ident, ty: P<ast::Ty>) -> ast::Param { + let arg_pat = self.pat_ident(span, ident); + ast::Param { + attrs: ThinVec::default(), + id: ast::DUMMY_NODE_ID, + pat: arg_pat, + span, + ty, + is_placeholder: false, + } + } + + // FIXME: unused `self` + pub fn fn_decl(&self, inputs: Vec<ast::Param>, output: ast::FunctionRetTy) -> P<ast::FnDecl> { + P(ast::FnDecl { + inputs, + output, + }) + } + + pub fn item(&self, span: Span, name: Ident, + attrs: Vec<ast::Attribute>, kind: ast::ItemKind) -> P<ast::Item> { + // FIXME: Would be nice if our generated code didn't violate + // Rust coding conventions + P(ast::Item { + ident: name, + attrs, + id: ast::DUMMY_NODE_ID, + kind, + vis: respan(span.shrink_to_lo(), ast::VisibilityKind::Inherited), + span, + tokens: None, + }) + } + + pub fn variant(&self, span: Span, ident: Ident, tys: Vec<P<ast::Ty>> ) -> ast::Variant { + let fields: Vec<_> = tys.into_iter().map(|ty| { + ast::StructField { + span: ty.span, + ty, + ident: None, + vis: respan(span.shrink_to_lo(), ast::VisibilityKind::Inherited), + attrs: Vec::new(), + id: ast::DUMMY_NODE_ID, + is_placeholder: false, + } + }).collect(); + + let vdata = if fields.is_empty() { + ast::VariantData::Unit(ast::DUMMY_NODE_ID) + } else { + ast::VariantData::Tuple(fields, ast::DUMMY_NODE_ID) + }; + + ast::Variant { + attrs: Vec::new(), + data: vdata, + disr_expr: None, + id: ast::DUMMY_NODE_ID, + ident, + span, + is_placeholder: false, + } + } + + pub fn item_static(&self, + span: Span, + name: Ident, + ty: P<ast::Ty>, + mutbl: ast::Mutability, + expr: P<ast::Expr>) + -> P<ast::Item> { + self.item(span, name, Vec::new(), ast::ItemKind::Static(ty, mutbl, expr)) + } + + pub fn item_const(&self, + span: Span, + name: Ident, + ty: P<ast::Ty>, + expr: P<ast::Expr>) + -> P<ast::Item> { + self.item(span, name, Vec::new(), ast::ItemKind::Const(ty, expr)) + } + + pub fn attribute(&self, mi: ast::MetaItem) -> ast::Attribute { + attr::mk_attr_outer(mi) + } + + pub fn meta_word(&self, sp: Span, w: ast::Name) -> ast::MetaItem { + attr::mk_word_item(Ident::new(w, sp)) + } +} diff --git a/src/libsyntax_expand/expand.rs b/src/libsyntax_expand/expand.rs new file mode 100644 index 00000000000..47b4bca314a --- /dev/null +++ b/src/libsyntax_expand/expand.rs @@ -0,0 +1,1551 @@ +use crate::base::*; +use crate::proc_macro::{collect_derives, MarkAttrs}; +use crate::hygiene::{ExpnId, SyntaxContext, ExpnData, ExpnKind}; +use crate::mbe::macro_rules::annotate_err_with_kind; +use crate::placeholders::{placeholder, PlaceholderExpander}; + +use syntax::ast::{self, AttrItem, Block, Ident, LitKind, NodeId, PatKind, Path}; +use syntax::ast::{MacStmtStyle, StmtKind, ItemKind}; +use syntax::attr::{self, HasAttrs}; +use syntax::source_map::respan; +use syntax::configure; +use syntax::config::StripUnconfigured; +use syntax::feature_gate::{self, Features, GateIssue, is_builtin_attr, emit_feature_err}; +use syntax::mut_visit::*; +use syntax::parse::{DirectoryOwnership, PResult}; +use syntax::parse::token; +use syntax::parse::parser::Parser; +use syntax::print::pprust; +use syntax::ptr::P; +use syntax::symbol::{sym, Symbol}; +use syntax::tokenstream::{TokenStream, TokenTree}; +use syntax::visit::Visitor; +use syntax::util::map_in_place::MapInPlace; + +use errors::{Applicability, FatalError}; +use smallvec::{smallvec, SmallVec}; +use syntax_pos::{Span, DUMMY_SP, FileName}; + +use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::sync::Lrc; +use std::io::ErrorKind; +use std::{iter, mem, slice}; +use std::ops::DerefMut; +use std::rc::Rc; +use std::path::PathBuf; + +macro_rules! ast_fragments { + ( + $($Kind:ident($AstTy:ty) { + $kind_name:expr; + $(one fn $mut_visit_ast:ident; fn $visit_ast:ident;)? + $(many fn $flat_map_ast_elt:ident; fn $visit_ast_elt:ident;)? + fn $make_ast:ident; + })* + ) => { + /// A fragment of AST that can be produced by a single macro expansion. + /// Can also serve as an input and intermediate result for macro expansion operations. + pub enum AstFragment { + OptExpr(Option<P<ast::Expr>>), + $($Kind($AstTy),)* + } + + /// "Discriminant" of an AST fragment. + #[derive(Copy, Clone, PartialEq, Eq)] + pub enum AstFragmentKind { + OptExpr, + $($Kind,)* + } + + impl AstFragmentKind { + pub fn name(self) -> &'static str { + match self { + AstFragmentKind::OptExpr => "expression", + $(AstFragmentKind::$Kind => $kind_name,)* + } + } + + fn make_from<'a>(self, result: Box<dyn MacResult + 'a>) -> Option<AstFragment> { + match self { + AstFragmentKind::OptExpr => + result.make_expr().map(Some).map(AstFragment::OptExpr), + $(AstFragmentKind::$Kind => result.$make_ast().map(AstFragment::$Kind),)* + } + } + } + + impl AstFragment { + pub fn make_opt_expr(self) -> Option<P<ast::Expr>> { + match self { + AstFragment::OptExpr(expr) => expr, + _ => panic!("AstFragment::make_* called on the wrong kind of fragment"), + } + } + + $(pub fn $make_ast(self) -> $AstTy { + match self { + AstFragment::$Kind(ast) => ast, + _ => panic!("AstFragment::make_* called on the wrong kind of fragment"), + } + })* + + pub fn mut_visit_with<F: MutVisitor>(&mut self, vis: &mut F) { + match self { + AstFragment::OptExpr(opt_expr) => { + visit_clobber(opt_expr, |opt_expr| { + if let Some(expr) = opt_expr { + vis.filter_map_expr(expr) + } else { + None + } + }); + } + $($(AstFragment::$Kind(ast) => vis.$mut_visit_ast(ast),)?)* + $($(AstFragment::$Kind(ast) => + ast.flat_map_in_place(|ast| vis.$flat_map_ast_elt(ast)),)?)* + } + } + + pub fn visit_with<'a, V: Visitor<'a>>(&'a self, visitor: &mut V) { + match *self { + AstFragment::OptExpr(Some(ref expr)) => visitor.visit_expr(expr), + AstFragment::OptExpr(None) => {} + $($(AstFragment::$Kind(ref ast) => visitor.$visit_ast(ast),)?)* + $($(AstFragment::$Kind(ref ast) => for ast_elt in &ast[..] { + visitor.$visit_ast_elt(ast_elt); + })?)* + } + } + } + + impl<'a> MacResult for crate::mbe::macro_rules::ParserAnyMacro<'a> { + $(fn $make_ast(self: Box<crate::mbe::macro_rules::ParserAnyMacro<'a>>) + -> Option<$AstTy> { + Some(self.make(AstFragmentKind::$Kind).$make_ast()) + })* + } + } +} + +ast_fragments! { + Expr(P<ast::Expr>) { "expression"; one fn visit_expr; fn visit_expr; fn make_expr; } + Pat(P<ast::Pat>) { "pattern"; one fn visit_pat; fn visit_pat; fn make_pat; } + Ty(P<ast::Ty>) { "type"; one fn visit_ty; fn visit_ty; fn make_ty; } + Stmts(SmallVec<[ast::Stmt; 1]>) { + "statement"; many fn flat_map_stmt; fn visit_stmt; fn make_stmts; + } + Items(SmallVec<[P<ast::Item>; 1]>) { + "item"; many fn flat_map_item; fn visit_item; fn make_items; + } + TraitItems(SmallVec<[ast::TraitItem; 1]>) { + "trait item"; many fn flat_map_trait_item; fn visit_trait_item; fn make_trait_items; + } + ImplItems(SmallVec<[ast::ImplItem; 1]>) { + "impl item"; many fn flat_map_impl_item; fn visit_impl_item; fn make_impl_items; + } + ForeignItems(SmallVec<[ast::ForeignItem; 1]>) { + "foreign item"; + many fn flat_map_foreign_item; + fn visit_foreign_item; + fn make_foreign_items; + } + Arms(SmallVec<[ast::Arm; 1]>) { + "match arm"; many fn flat_map_arm; fn visit_arm; fn make_arms; + } + Fields(SmallVec<[ast::Field; 1]>) { + "field expression"; many fn flat_map_field; fn visit_field; fn make_fields; + } + FieldPats(SmallVec<[ast::FieldPat; 1]>) { + "field pattern"; + many fn flat_map_field_pattern; + fn visit_field_pattern; + fn make_field_patterns; + } + GenericParams(SmallVec<[ast::GenericParam; 1]>) { + "generic parameter"; + many fn flat_map_generic_param; + fn visit_generic_param; + fn make_generic_params; + } + Params(SmallVec<[ast::Param; 1]>) { + "function parameter"; many fn flat_map_param; fn visit_param; fn make_params; + } + StructFields(SmallVec<[ast::StructField; 1]>) { + "field"; + many fn flat_map_struct_field; + fn visit_struct_field; + fn make_struct_fields; + } + Variants(SmallVec<[ast::Variant; 1]>) { + "variant"; many fn flat_map_variant; fn visit_variant; fn make_variants; + } +} + +impl AstFragmentKind { + fn dummy(self, span: Span) -> AstFragment { + self.make_from(DummyResult::any(span)).expect("couldn't create a dummy AST fragment") + } + + fn expect_from_annotatables<I: IntoIterator<Item = Annotatable>>(self, items: I) + -> AstFragment { + let mut items = items.into_iter(); + match self { + AstFragmentKind::Arms => + AstFragment::Arms(items.map(Annotatable::expect_arm).collect()), + AstFragmentKind::Fields => + AstFragment::Fields(items.map(Annotatable::expect_field).collect()), + AstFragmentKind::FieldPats => + AstFragment::FieldPats(items.map(Annotatable::expect_field_pattern).collect()), + AstFragmentKind::GenericParams => + AstFragment::GenericParams(items.map(Annotatable::expect_generic_param).collect()), + AstFragmentKind::Params => + AstFragment::Params(items.map(Annotatable::expect_param).collect()), + AstFragmentKind::StructFields => AstFragment::StructFields( + items.map(Annotatable::expect_struct_field).collect() + ), + AstFragmentKind::Variants => + AstFragment::Variants(items.map(Annotatable::expect_variant).collect()), + AstFragmentKind::Items => + AstFragment::Items(items.map(Annotatable::expect_item).collect()), + AstFragmentKind::ImplItems => + AstFragment::ImplItems(items.map(Annotatable::expect_impl_item).collect()), + AstFragmentKind::TraitItems => + AstFragment::TraitItems(items.map(Annotatable::expect_trait_item).collect()), + AstFragmentKind::ForeignItems => + AstFragment::ForeignItems(items.map(Annotatable::expect_foreign_item).collect()), + AstFragmentKind::Stmts => + AstFragment::Stmts(items.map(Annotatable::expect_stmt).collect()), + AstFragmentKind::Expr => AstFragment::Expr( + items.next().expect("expected exactly one expression").expect_expr() + ), + AstFragmentKind::OptExpr => + AstFragment::OptExpr(items.next().map(Annotatable::expect_expr)), + AstFragmentKind::Pat | AstFragmentKind::Ty => + panic!("patterns and types aren't annotatable"), + } + } +} + +pub struct Invocation { + pub kind: InvocationKind, + pub fragment_kind: AstFragmentKind, + pub expansion_data: ExpansionData, +} + +pub enum InvocationKind { + Bang { + mac: ast::Mac, + span: Span, + }, + Attr { + attr: ast::Attribute, + item: Annotatable, + // Required for resolving derive helper attributes. + derives: Vec<Path>, + // We temporarily report errors for attribute macros placed after derives + after_derive: bool, + }, + Derive { + path: Path, + item: Annotatable, + }, + /// "Invocation" that contains all derives from an item, + /// broken into multiple `Derive` invocations when expanded. + /// FIXME: Find a way to remove it. + DeriveContainer { + derives: Vec<Path>, + item: Annotatable, + }, +} + +impl Invocation { + pub fn span(&self) -> Span { + match &self.kind { + InvocationKind::Bang { span, .. } => *span, + InvocationKind::Attr { attr, .. } => attr.span, + InvocationKind::Derive { path, .. } => path.span, + InvocationKind::DeriveContainer { item, .. } => item.span(), + } + } +} + +pub struct MacroExpander<'a, 'b> { + pub cx: &'a mut ExtCtxt<'b>, + monotonic: bool, // cf. `cx.monotonic_expander()` +} + +impl<'a, 'b> MacroExpander<'a, 'b> { + pub fn new(cx: &'a mut ExtCtxt<'b>, monotonic: bool) -> Self { + MacroExpander { cx, monotonic } + } + + pub fn expand_crate(&mut self, mut krate: ast::Crate) -> ast::Crate { + let mut module = ModuleData { + mod_path: vec![Ident::from_str(&self.cx.ecfg.crate_name)], + directory: match self.cx.source_map().span_to_unmapped_path(krate.span) { + FileName::Real(path) => path, + other => PathBuf::from(other.to_string()), + }, + }; + module.directory.pop(); + self.cx.root_path = module.directory.clone(); + self.cx.current_expansion.module = Rc::new(module); + + let orig_mod_span = krate.module.inner; + + let krate_item = AstFragment::Items(smallvec![P(ast::Item { + attrs: krate.attrs, + span: krate.span, + kind: ast::ItemKind::Mod(krate.module), + ident: Ident::invalid(), + id: ast::DUMMY_NODE_ID, + vis: respan(krate.span.shrink_to_lo(), ast::VisibilityKind::Public), + tokens: None, + })]); + + match self.fully_expand_fragment(krate_item).make_items().pop().map(P::into_inner) { + Some(ast::Item { attrs, kind: ast::ItemKind::Mod(module), .. }) => { + krate.attrs = attrs; + krate.module = module; + }, + None => { + // Resolution failed so we return an empty expansion + krate.attrs = vec![]; + krate.module = ast::Mod { + inner: orig_mod_span, + items: vec![], + inline: true, + }; + }, + _ => unreachable!(), + }; + self.cx.trace_macros_diag(); + krate + } + + // Recursively expand all macro invocations in this AST fragment. + pub fn fully_expand_fragment(&mut self, input_fragment: AstFragment) -> AstFragment { + let orig_expansion_data = self.cx.current_expansion.clone(); + self.cx.current_expansion.depth = 0; + + // Collect all macro invocations and replace them with placeholders. + let (mut fragment_with_placeholders, mut invocations) + = self.collect_invocations(input_fragment, &[]); + + // Optimization: if we resolve all imports now, + // we'll be able to immediately resolve most of imported macros. + self.resolve_imports(); + + // Resolve paths in all invocations and produce output expanded fragments for them, but + // do not insert them into our input AST fragment yet, only store in `expanded_fragments`. + // The output fragments also go through expansion recursively until no invocations are left. + // Unresolved macros produce dummy outputs as a recovery measure. + invocations.reverse(); + let mut expanded_fragments = Vec::new(); + let mut all_derive_placeholders: FxHashMap<ExpnId, Vec<_>> = FxHashMap::default(); + let mut undetermined_invocations = Vec::new(); + let (mut progress, mut force) = (false, !self.monotonic); + loop { + let invoc = if let Some(invoc) = invocations.pop() { + invoc + } else { + self.resolve_imports(); + if undetermined_invocations.is_empty() { break } + invocations = mem::take(&mut undetermined_invocations); + force = !mem::replace(&mut progress, false); + continue + }; + + let eager_expansion_root = + if self.monotonic { invoc.expansion_data.id } else { orig_expansion_data.id }; + let res = match self.cx.resolver.resolve_macro_invocation( + &invoc, eager_expansion_root, force + ) { + Ok(res) => res, + Err(Indeterminate) => { + undetermined_invocations.push(invoc); + continue + } + }; + + progress = true; + let ExpansionData { depth, id: expn_id, .. } = invoc.expansion_data; + self.cx.current_expansion = invoc.expansion_data.clone(); + + // FIXME(jseyfried): Refactor out the following logic + let (expanded_fragment, new_invocations) = match res { + InvocationRes::Single(ext) => { + let fragment = self.expand_invoc(invoc, &ext.kind); + self.collect_invocations(fragment, &[]) + } + InvocationRes::DeriveContainer(exts) => { + let (derives, item) = match invoc.kind { + InvocationKind::DeriveContainer { derives, item } => (derives, item), + _ => unreachable!(), + }; + if !item.derive_allowed() { + let attr = attr::find_by_name(item.attrs(), sym::derive) + .expect("`derive` attribute should exist"); + let span = attr.span; + let mut err = self.cx.struct_span_err(span, + "`derive` may only be applied to structs, enums and unions"); + if let ast::AttrStyle::Inner = attr.style { + let trait_list = derives.iter() + .map(|t| pprust::path_to_string(t)) + .collect::<Vec<_>>(); + let suggestion = format!("#[derive({})]", trait_list.join(", ")); + err.span_suggestion( + span, "try an outer attribute", suggestion, + // We don't 𝑘𝑛𝑜𝑤 that the following item is an ADT + Applicability::MaybeIncorrect + ); + } + err.emit(); + } + + let mut item = self.fully_configure(item); + item.visit_attrs(|attrs| attrs.retain(|a| a.path != sym::derive)); + let mut helper_attrs = Vec::new(); + let mut has_copy = false; + for ext in exts { + helper_attrs.extend(&ext.helper_attrs); + has_copy |= ext.is_derive_copy; + } + // Mark derive helpers inside this item as known and used. + // FIXME: This is a hack, derive helpers should be integrated with regular name + // resolution instead. For example, helpers introduced by a derive container + // can be in scope for all code produced by that container's expansion. + item.visit_with(&mut MarkAttrs(&helper_attrs)); + if has_copy { + self.cx.resolver.add_derives(invoc.expansion_data.id, SpecialDerives::COPY); + } + + let derive_placeholders = + all_derive_placeholders.entry(invoc.expansion_data.id).or_default(); + derive_placeholders.reserve(derives.len()); + invocations.reserve(derives.len()); + for path in derives { + let expn_id = ExpnId::fresh(None); + derive_placeholders.push(NodeId::placeholder_from_expn_id(expn_id)); + invocations.push(Invocation { + kind: InvocationKind::Derive { path, item: item.clone() }, + fragment_kind: invoc.fragment_kind, + expansion_data: ExpansionData { + id: expn_id, + ..invoc.expansion_data.clone() + }, + }); + } + let fragment = invoc.fragment_kind + .expect_from_annotatables(::std::iter::once(item)); + self.collect_invocations(fragment, derive_placeholders) + } + }; + + if expanded_fragments.len() < depth { + expanded_fragments.push(Vec::new()); + } + expanded_fragments[depth - 1].push((expn_id, expanded_fragment)); + if !self.cx.ecfg.single_step { + invocations.extend(new_invocations.into_iter().rev()); + } + } + + self.cx.current_expansion = orig_expansion_data; + + // Finally incorporate all the expanded macros into the input AST fragment. + let mut placeholder_expander = PlaceholderExpander::new(self.cx, self.monotonic); + while let Some(expanded_fragments) = expanded_fragments.pop() { + for (expn_id, expanded_fragment) in expanded_fragments.into_iter().rev() { + let derive_placeholders = + all_derive_placeholders.remove(&expn_id).unwrap_or_else(Vec::new); + placeholder_expander.add(NodeId::placeholder_from_expn_id(expn_id), + expanded_fragment, derive_placeholders); + } + } + fragment_with_placeholders.mut_visit_with(&mut placeholder_expander); + fragment_with_placeholders + } + + fn resolve_imports(&mut self) { + if self.monotonic { + self.cx.resolver.resolve_imports(); + } + } + + /// Collects all macro invocations reachable at this time in this AST fragment, and replace + /// them with "placeholders" - dummy macro invocations with specially crafted `NodeId`s. + /// Then call into resolver that builds a skeleton ("reduced graph") of the fragment and + /// prepares data for resolving paths of macro invocations. + fn collect_invocations(&mut self, mut fragment: AstFragment, extra_placeholders: &[NodeId]) + -> (AstFragment, Vec<Invocation>) { + // Resolve `$crate`s in the fragment for pretty-printing. + self.cx.resolver.resolve_dollar_crates(); + + let invocations = { + let mut collector = InvocationCollector { + cfg: StripUnconfigured { + sess: self.cx.parse_sess, + features: self.cx.ecfg.features, + }, + cx: self.cx, + invocations: Vec::new(), + monotonic: self.monotonic, + }; + fragment.mut_visit_with(&mut collector); + collector.invocations + }; + + // FIXME: Merge `extra_placeholders` into the `fragment` as regular placeholders. + if self.monotonic { + self.cx.resolver.visit_ast_fragment_with_placeholders( + self.cx.current_expansion.id, &fragment, extra_placeholders); + } + + (fragment, invocations) + } + + fn fully_configure(&mut self, item: Annotatable) -> Annotatable { + let mut cfg = StripUnconfigured { + sess: self.cx.parse_sess, + features: self.cx.ecfg.features, + }; + // Since the item itself has already been configured by the InvocationCollector, + // we know that fold result vector will contain exactly one element + match item { + Annotatable::Item(item) => { + Annotatable::Item(cfg.flat_map_item(item).pop().unwrap()) + } + Annotatable::TraitItem(item) => { + Annotatable::TraitItem( + item.map(|item| cfg.flat_map_trait_item(item).pop().unwrap())) + } + Annotatable::ImplItem(item) => { + Annotatable::ImplItem(item.map(|item| cfg.flat_map_impl_item(item).pop().unwrap())) + } + Annotatable::ForeignItem(item) => { + Annotatable::ForeignItem( + item.map(|item| cfg.flat_map_foreign_item(item).pop().unwrap()) + ) + } + Annotatable::Stmt(stmt) => { + Annotatable::Stmt(stmt.map(|stmt| cfg.flat_map_stmt(stmt).pop().unwrap())) + } + Annotatable::Expr(mut expr) => { + Annotatable::Expr({ cfg.visit_expr(&mut expr); expr }) + } + Annotatable::Arm(arm) => { + Annotatable::Arm(cfg.flat_map_arm(arm).pop().unwrap()) + } + Annotatable::Field(field) => { + Annotatable::Field(cfg.flat_map_field(field).pop().unwrap()) + } + Annotatable::FieldPat(fp) => { + Annotatable::FieldPat(cfg.flat_map_field_pattern(fp).pop().unwrap()) + } + Annotatable::GenericParam(param) => { + Annotatable::GenericParam(cfg.flat_map_generic_param(param).pop().unwrap()) + } + Annotatable::Param(param) => { + Annotatable::Param(cfg.flat_map_param(param).pop().unwrap()) + } + Annotatable::StructField(sf) => { + Annotatable::StructField(cfg.flat_map_struct_field(sf).pop().unwrap()) + } + Annotatable::Variant(v) => { + Annotatable::Variant(cfg.flat_map_variant(v).pop().unwrap()) + } + } + } + + fn expand_invoc(&mut self, invoc: Invocation, ext: &SyntaxExtensionKind) -> AstFragment { + if self.cx.current_expansion.depth > self.cx.ecfg.recursion_limit { + let expn_data = self.cx.current_expansion.id.expn_data(); + let suggested_limit = self.cx.ecfg.recursion_limit * 2; + let mut err = self.cx.struct_span_err(expn_data.call_site, + &format!("recursion limit reached while expanding the macro `{}`", + expn_data.kind.descr())); + err.help(&format!( + "consider adding a `#![recursion_limit=\"{}\"]` attribute to your crate", + suggested_limit)); + err.emit(); + self.cx.trace_macros_diag(); + FatalError.raise(); + } + + let (fragment_kind, span) = (invoc.fragment_kind, invoc.span()); + match invoc.kind { + InvocationKind::Bang { mac, .. } => match ext { + SyntaxExtensionKind::Bang(expander) => { + self.gate_proc_macro_expansion_kind(span, fragment_kind); + let tok_result = expander.expand(self.cx, span, mac.stream()); + self.parse_ast_fragment(tok_result, fragment_kind, &mac.path, span) + } + SyntaxExtensionKind::LegacyBang(expander) => { + let prev = self.cx.current_expansion.prior_type_ascription; + self.cx.current_expansion.prior_type_ascription = mac.prior_type_ascription; + let tok_result = expander.expand(self.cx, span, mac.stream()); + let result = if let Some(result) = fragment_kind.make_from(tok_result) { + result + } else { + let msg = format!( + "non-{kind} macro in {kind} position: {path}", + kind = fragment_kind.name(), + path = pprust::path_to_string(&mac.path), + ); + self.cx.span_err(span, &msg); + self.cx.trace_macros_diag(); + fragment_kind.dummy(span) + }; + self.cx.current_expansion.prior_type_ascription = prev; + result + } + _ => unreachable!() + } + InvocationKind::Attr { attr, mut item, .. } => match ext { + SyntaxExtensionKind::Attr(expander) => { + self.gate_proc_macro_attr_item(span, &item); + let item_tok = TokenTree::token(token::Interpolated(Lrc::new(match item { + Annotatable::Item(item) => token::NtItem(item), + Annotatable::TraitItem(item) => token::NtTraitItem(item.into_inner()), + Annotatable::ImplItem(item) => token::NtImplItem(item.into_inner()), + Annotatable::ForeignItem(item) => token::NtForeignItem(item.into_inner()), + Annotatable::Stmt(stmt) => token::NtStmt(stmt.into_inner()), + Annotatable::Expr(expr) => token::NtExpr(expr), + Annotatable::Arm(..) + | Annotatable::Field(..) + | Annotatable::FieldPat(..) + | Annotatable::GenericParam(..) + | Annotatable::Param(..) + | Annotatable::StructField(..) + | Annotatable::Variant(..) + => panic!("unexpected annotatable"), + })), DUMMY_SP).into(); + let input = self.extract_proc_macro_attr_input(attr.item.tokens, span); + let tok_result = expander.expand(self.cx, span, input, item_tok); + self.parse_ast_fragment(tok_result, fragment_kind, &attr.item.path, span) + } + SyntaxExtensionKind::LegacyAttr(expander) => { + match attr.parse_meta(self.cx.parse_sess) { + Ok(meta) => { + let item = expander.expand(self.cx, span, &meta, item); + fragment_kind.expect_from_annotatables(item) + } + Err(mut err) => { + err.emit(); + fragment_kind.dummy(span) + } + } + } + SyntaxExtensionKind::NonMacroAttr { mark_used } => { + attr::mark_known(&attr); + if *mark_used { + attr::mark_used(&attr); + } + item.visit_attrs(|attrs| attrs.push(attr)); + fragment_kind.expect_from_annotatables(iter::once(item)) + } + _ => unreachable!() + } + InvocationKind::Derive { path, item } => match ext { + SyntaxExtensionKind::Derive(expander) | + SyntaxExtensionKind::LegacyDerive(expander) => { + if !item.derive_allowed() { + return fragment_kind.dummy(span); + } + let meta = ast::MetaItem { kind: ast::MetaItemKind::Word, span, path }; + let items = expander.expand(self.cx, span, &meta, item); + fragment_kind.expect_from_annotatables(items) + } + _ => unreachable!() + } + InvocationKind::DeriveContainer { .. } => unreachable!() + } + } + + fn extract_proc_macro_attr_input(&self, tokens: TokenStream, span: Span) -> TokenStream { + let mut trees = tokens.trees(); + match trees.next() { + Some(TokenTree::Delimited(_, _, tts)) => { + if trees.next().is_none() { + return tts.into() + } + } + Some(TokenTree::Token(..)) => {} + None => return TokenStream::default(), + } + self.cx.span_err(span, "custom attribute invocations must be \ + of the form `#[foo]` or `#[foo(..)]`, the macro name must only be \ + followed by a delimiter token"); + TokenStream::default() + } + + fn gate_proc_macro_attr_item(&self, span: Span, item: &Annotatable) { + let (kind, gate) = match *item { + Annotatable::Item(ref item) => { + match item.kind { + ItemKind::Mod(_) if self.cx.ecfg.proc_macro_hygiene() => return, + ItemKind::Mod(_) => ("modules", sym::proc_macro_hygiene), + _ => return, + } + } + Annotatable::TraitItem(_) => return, + Annotatable::ImplItem(_) => return, + Annotatable::ForeignItem(_) => return, + Annotatable::Stmt(_) | + Annotatable::Expr(_) if self.cx.ecfg.proc_macro_hygiene() => return, + Annotatable::Stmt(_) => ("statements", sym::proc_macro_hygiene), + Annotatable::Expr(_) => ("expressions", sym::proc_macro_hygiene), + Annotatable::Arm(..) + | Annotatable::Field(..) + | Annotatable::FieldPat(..) + | Annotatable::GenericParam(..) + | Annotatable::Param(..) + | Annotatable::StructField(..) + | Annotatable::Variant(..) + => panic!("unexpected annotatable"), + }; + emit_feature_err( + self.cx.parse_sess, + gate, + span, + GateIssue::Language, + &format!("custom attributes cannot be applied to {}", kind), + ); + } + + fn gate_proc_macro_expansion_kind(&self, span: Span, kind: AstFragmentKind) { + let kind = match kind { + AstFragmentKind::Expr | + AstFragmentKind::OptExpr => "expressions", + AstFragmentKind::Pat => "patterns", + AstFragmentKind::Stmts => "statements", + AstFragmentKind::Ty | + AstFragmentKind::Items | + AstFragmentKind::TraitItems | + AstFragmentKind::ImplItems | + AstFragmentKind::ForeignItems => return, + AstFragmentKind::Arms + | AstFragmentKind::Fields + | AstFragmentKind::FieldPats + | AstFragmentKind::GenericParams + | AstFragmentKind::Params + | AstFragmentKind::StructFields + | AstFragmentKind::Variants + => panic!("unexpected AST fragment kind"), + }; + if self.cx.ecfg.proc_macro_hygiene() { + return + } + emit_feature_err( + self.cx.parse_sess, + sym::proc_macro_hygiene, + span, + GateIssue::Language, + &format!("procedural macros cannot be expanded to {}", kind), + ); + } + + fn parse_ast_fragment( + &mut self, + toks: TokenStream, + kind: AstFragmentKind, + path: &Path, + span: Span, + ) -> AstFragment { + let mut parser = self.cx.new_parser_from_tts(toks); + match parse_ast_fragment(&mut parser, kind, false) { + Ok(fragment) => { + ensure_complete_parse(&mut parser, path, kind.name(), span); + fragment + } + Err(mut err) => { + err.set_span(span); + annotate_err_with_kind(&mut err, kind, span); + err.emit(); + self.cx.trace_macros_diag(); + kind.dummy(span) + } + } + } +} + +pub fn parse_ast_fragment<'a>( + this: &mut Parser<'a>, + kind: AstFragmentKind, + macro_legacy_warnings: bool, +) -> PResult<'a, AstFragment> { + Ok(match kind { + AstFragmentKind::Items => { + let mut items = SmallVec::new(); + while let Some(item) = this.parse_item()? { + items.push(item); + } + AstFragment::Items(items) + } + AstFragmentKind::TraitItems => { + let mut items = SmallVec::new(); + while this.token != token::Eof { + items.push(this.parse_trait_item(&mut false)?); + } + AstFragment::TraitItems(items) + } + AstFragmentKind::ImplItems => { + let mut items = SmallVec::new(); + while this.token != token::Eof { + items.push(this.parse_impl_item(&mut false)?); + } + AstFragment::ImplItems(items) + } + AstFragmentKind::ForeignItems => { + let mut items = SmallVec::new(); + while this.token != token::Eof { + items.push(this.parse_foreign_item(DUMMY_SP)?); + } + AstFragment::ForeignItems(items) + } + AstFragmentKind::Stmts => { + let mut stmts = SmallVec::new(); + while this.token != token::Eof && + // won't make progress on a `}` + this.token != token::CloseDelim(token::Brace) { + if let Some(stmt) = this.parse_full_stmt(macro_legacy_warnings)? { + stmts.push(stmt); + } + } + AstFragment::Stmts(stmts) + } + AstFragmentKind::Expr => AstFragment::Expr(this.parse_expr()?), + AstFragmentKind::OptExpr => { + if this.token != token::Eof { + AstFragment::OptExpr(Some(this.parse_expr()?)) + } else { + AstFragment::OptExpr(None) + } + }, + AstFragmentKind::Ty => AstFragment::Ty(this.parse_ty()?), + AstFragmentKind::Pat => AstFragment::Pat(this.parse_pat(None)?), + AstFragmentKind::Arms + | AstFragmentKind::Fields + | AstFragmentKind::FieldPats + | AstFragmentKind::GenericParams + | AstFragmentKind::Params + | AstFragmentKind::StructFields + | AstFragmentKind::Variants + => panic!("unexpected AST fragment kind"), + }) +} + +pub fn ensure_complete_parse<'a>( + this: &mut Parser<'a>, + macro_path: &Path, + kind_name: &str, + span: Span, +) { + if this.token != token::Eof { + let msg = format!("macro expansion ignores token `{}` and any following", + this.this_token_to_string()); + // Avoid emitting backtrace info twice. + let def_site_span = this.token.span.with_ctxt(SyntaxContext::root()); + let mut err = this.struct_span_err(def_site_span, &msg); + err.span_label(span, "caused by the macro expansion here"); + let msg = format!( + "the usage of `{}!` is likely invalid in {} context", + pprust::path_to_string(macro_path), + kind_name, + ); + err.note(&msg); + let semi_span = this.sess.source_map().next_point(span); + + let semi_full_span = semi_span.to(this.sess.source_map().next_point(semi_span)); + match this.sess.source_map().span_to_snippet(semi_full_span) { + Ok(ref snippet) if &snippet[..] != ";" && kind_name == "expression" => { + err.span_suggestion( + semi_span, + "you might be missing a semicolon here", + ";".to_owned(), + Applicability::MaybeIncorrect, + ); + } + _ => {} + } + err.emit(); + } +} + +struct InvocationCollector<'a, 'b> { + cx: &'a mut ExtCtxt<'b>, + cfg: StripUnconfigured<'a>, + invocations: Vec<Invocation>, + monotonic: bool, +} + +impl<'a, 'b> InvocationCollector<'a, 'b> { + fn collect(&mut self, fragment_kind: AstFragmentKind, kind: InvocationKind) -> AstFragment { + // Expansion data for all the collected invocations is set upon their resolution, + // with exception of the derive container case which is not resolved and can get + // its expansion data immediately. + let expn_data = match &kind { + InvocationKind::DeriveContainer { item, .. } => Some(ExpnData { + parent: self.cx.current_expansion.id, + ..ExpnData::default( + ExpnKind::Macro(MacroKind::Attr, sym::derive), + item.span(), self.cx.parse_sess.edition, + ) + }), + _ => None, + }; + let expn_id = ExpnId::fresh(expn_data); + self.invocations.push(Invocation { + kind, + fragment_kind, + expansion_data: ExpansionData { + id: expn_id, + depth: self.cx.current_expansion.depth + 1, + ..self.cx.current_expansion.clone() + }, + }); + placeholder(fragment_kind, NodeId::placeholder_from_expn_id(expn_id)) + } + + fn collect_bang(&mut self, mac: ast::Mac, span: Span, kind: AstFragmentKind) -> AstFragment { + self.collect(kind, InvocationKind::Bang { mac, span }) + } + + fn collect_attr(&mut self, + attr: Option<ast::Attribute>, + derives: Vec<Path>, + item: Annotatable, + kind: AstFragmentKind, + after_derive: bool) + -> AstFragment { + self.collect(kind, match attr { + Some(attr) => InvocationKind::Attr { attr, item, derives, after_derive }, + None => InvocationKind::DeriveContainer { derives, item }, + }) + } + + fn find_attr_invoc(&self, attrs: &mut Vec<ast::Attribute>, after_derive: &mut bool) + -> Option<ast::Attribute> { + let attr = attrs.iter() + .position(|a| { + if a.path == sym::derive { + *after_derive = true; + } + !attr::is_known(a) && !is_builtin_attr(a) + }) + .map(|i| attrs.remove(i)); + if let Some(attr) = &attr { + if !self.cx.ecfg.custom_inner_attributes() && + attr.style == ast::AttrStyle::Inner && attr.path != sym::test { + emit_feature_err(&self.cx.parse_sess, sym::custom_inner_attributes, + attr.span, GateIssue::Language, + "non-builtin inner attributes are unstable"); + } + } + attr + } + + /// If `item` is an attr invocation, remove and return the macro attribute and derive traits. + fn classify_item<T>(&mut self, item: &mut T) + -> (Option<ast::Attribute>, Vec<Path>, /* after_derive */ bool) + where T: HasAttrs, + { + let (mut attr, mut traits, mut after_derive) = (None, Vec::new(), false); + + item.visit_attrs(|mut attrs| { + attr = self.find_attr_invoc(&mut attrs, &mut after_derive); + traits = collect_derives(&mut self.cx, &mut attrs); + }); + + (attr, traits, after_derive) + } + + /// Alternative to `classify_item()` that ignores `#[derive]` so invocations fallthrough + /// to the unused-attributes lint (making it an error on statements and expressions + /// is a breaking change) + fn classify_nonitem<T: HasAttrs>(&mut self, nonitem: &mut T) + -> (Option<ast::Attribute>, /* after_derive */ bool) { + let (mut attr, mut after_derive) = (None, false); + + nonitem.visit_attrs(|mut attrs| { + attr = self.find_attr_invoc(&mut attrs, &mut after_derive); + }); + + (attr, after_derive) + } + + fn configure<T: HasAttrs>(&mut self, node: T) -> Option<T> { + self.cfg.configure(node) + } + + // Detect use of feature-gated or invalid attributes on macro invocations + // since they will not be detected after macro expansion. + fn check_attributes(&mut self, attrs: &[ast::Attribute]) { + let features = self.cx.ecfg.features.unwrap(); + for attr in attrs.iter() { + feature_gate::check_attribute(attr, self.cx.parse_sess, features); + + // macros are expanded before any lint passes so this warning has to be hardcoded + if attr.path == sym::derive { + self.cx.struct_span_warn(attr.span, "`#[derive]` does nothing on macro invocations") + .note("this may become a hard error in a future release") + .emit(); + } + } + } +} + +impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> { + fn visit_expr(&mut self, expr: &mut P<ast::Expr>) { + self.cfg.configure_expr(expr); + visit_clobber(expr.deref_mut(), |mut expr| { + self.cfg.configure_expr_kind(&mut expr.kind); + + // ignore derives so they remain unused + let (attr, after_derive) = self.classify_nonitem(&mut expr); + + if attr.is_some() { + // Collect the invoc regardless of whether or not attributes are permitted here + // expansion will eat the attribute so it won't error later. + attr.as_ref().map(|a| self.cfg.maybe_emit_expr_attr_err(a)); + + // AstFragmentKind::Expr requires the macro to emit an expression. + return self.collect_attr(attr, vec![], Annotatable::Expr(P(expr)), + AstFragmentKind::Expr, after_derive) + .make_expr() + .into_inner() + } + + if let ast::ExprKind::Mac(mac) = expr.kind { + self.check_attributes(&expr.attrs); + self.collect_bang(mac, expr.span, AstFragmentKind::Expr) + .make_expr() + .into_inner() + } else { + noop_visit_expr(&mut expr, self); + expr + } + }); + } + + fn flat_map_arm(&mut self, arm: ast::Arm) -> SmallVec<[ast::Arm; 1]> { + let mut arm = configure!(self, arm); + + let (attr, traits, after_derive) = self.classify_item(&mut arm); + if attr.is_some() || !traits.is_empty() { + return self.collect_attr(attr, traits, Annotatable::Arm(arm), + AstFragmentKind::Arms, after_derive) + .make_arms(); + } + + noop_flat_map_arm(arm, self) + } + + fn flat_map_field(&mut self, field: ast::Field) -> SmallVec<[ast::Field; 1]> { + let mut field = configure!(self, field); + + let (attr, traits, after_derive) = self.classify_item(&mut field); + if attr.is_some() || !traits.is_empty() { + return self.collect_attr(attr, traits, Annotatable::Field(field), + AstFragmentKind::Fields, after_derive) + .make_fields(); + } + + noop_flat_map_field(field, self) + } + + fn flat_map_field_pattern(&mut self, fp: ast::FieldPat) -> SmallVec<[ast::FieldPat; 1]> { + let mut fp = configure!(self, fp); + + let (attr, traits, after_derive) = self.classify_item(&mut fp); + if attr.is_some() || !traits.is_empty() { + return self.collect_attr(attr, traits, Annotatable::FieldPat(fp), + AstFragmentKind::FieldPats, after_derive) + .make_field_patterns(); + } + + noop_flat_map_field_pattern(fp, self) + } + + fn flat_map_param(&mut self, p: ast::Param) -> SmallVec<[ast::Param; 1]> { + let mut p = configure!(self, p); + + let (attr, traits, after_derive) = self.classify_item(&mut p); + if attr.is_some() || !traits.is_empty() { + return self.collect_attr(attr, traits, Annotatable::Param(p), + AstFragmentKind::Params, after_derive) + .make_params(); + } + + noop_flat_map_param(p, self) + } + + fn flat_map_struct_field(&mut self, sf: ast::StructField) -> SmallVec<[ast::StructField; 1]> { + let mut sf = configure!(self, sf); + + let (attr, traits, after_derive) = self.classify_item(&mut sf); + if attr.is_some() || !traits.is_empty() { + return self.collect_attr(attr, traits, Annotatable::StructField(sf), + AstFragmentKind::StructFields, after_derive) + .make_struct_fields(); + } + + noop_flat_map_struct_field(sf, self) + } + + fn flat_map_variant(&mut self, variant: ast::Variant) -> SmallVec<[ast::Variant; 1]> { + let mut variant = configure!(self, variant); + + let (attr, traits, after_derive) = self.classify_item(&mut variant); + if attr.is_some() || !traits.is_empty() { + return self.collect_attr(attr, traits, Annotatable::Variant(variant), + AstFragmentKind::Variants, after_derive) + .make_variants(); + } + + noop_flat_map_variant(variant, self) + } + + fn filter_map_expr(&mut self, expr: P<ast::Expr>) -> Option<P<ast::Expr>> { + let expr = configure!(self, expr); + expr.filter_map(|mut expr| { + self.cfg.configure_expr_kind(&mut expr.kind); + + // Ignore derives so they remain unused. + let (attr, after_derive) = self.classify_nonitem(&mut expr); + + if attr.is_some() { + attr.as_ref().map(|a| self.cfg.maybe_emit_expr_attr_err(a)); + + return self.collect_attr(attr, vec![], Annotatable::Expr(P(expr)), + AstFragmentKind::OptExpr, after_derive) + .make_opt_expr() + .map(|expr| expr.into_inner()) + } + + if let ast::ExprKind::Mac(mac) = expr.kind { + self.check_attributes(&expr.attrs); + self.collect_bang(mac, expr.span, AstFragmentKind::OptExpr) + .make_opt_expr() + .map(|expr| expr.into_inner()) + } else { + Some({ noop_visit_expr(&mut expr, self); expr }) + } + }) + } + + fn visit_pat(&mut self, pat: &mut P<ast::Pat>) { + self.cfg.configure_pat(pat); + match pat.kind { + PatKind::Mac(_) => {} + _ => return noop_visit_pat(pat, self), + } + + visit_clobber(pat, |mut pat| { + match mem::replace(&mut pat.kind, PatKind::Wild) { + PatKind::Mac(mac) => + self.collect_bang(mac, pat.span, AstFragmentKind::Pat).make_pat(), + _ => unreachable!(), + } + }); + } + + fn flat_map_stmt(&mut self, stmt: ast::Stmt) -> SmallVec<[ast::Stmt; 1]> { + let mut stmt = configure!(self, stmt); + + // we'll expand attributes on expressions separately + if !stmt.is_expr() { + let (attr, derives, after_derive) = if stmt.is_item() { + self.classify_item(&mut stmt) + } else { + // ignore derives on non-item statements so it falls through + // to the unused-attributes lint + let (attr, after_derive) = self.classify_nonitem(&mut stmt); + (attr, vec![], after_derive) + }; + + if attr.is_some() || !derives.is_empty() { + return self.collect_attr(attr, derives, Annotatable::Stmt(P(stmt)), + AstFragmentKind::Stmts, after_derive).make_stmts(); + } + } + + if let StmtKind::Mac(mac) = stmt.kind { + let (mac, style, attrs) = mac.into_inner(); + self.check_attributes(&attrs); + let mut placeholder = self.collect_bang(mac, stmt.span, AstFragmentKind::Stmts) + .make_stmts(); + + // If this is a macro invocation with a semicolon, then apply that + // semicolon to the final statement produced by expansion. + if style == MacStmtStyle::Semicolon { + if let Some(stmt) = placeholder.pop() { + placeholder.push(stmt.add_trailing_semicolon()); + } + } + + return placeholder; + } + + // The placeholder expander gives ids to statements, so we avoid folding the id here. + let ast::Stmt { id, kind, span } = stmt; + noop_flat_map_stmt_kind(kind, self).into_iter().map(|kind| { + ast::Stmt { id, kind, span } + }).collect() + + } + + fn visit_block(&mut self, block: &mut P<Block>) { + let old_directory_ownership = self.cx.current_expansion.directory_ownership; + self.cx.current_expansion.directory_ownership = DirectoryOwnership::UnownedViaBlock; + noop_visit_block(block, self); + self.cx.current_expansion.directory_ownership = old_directory_ownership; + } + + fn flat_map_item(&mut self, item: P<ast::Item>) -> SmallVec<[P<ast::Item>; 1]> { + let mut item = configure!(self, item); + + let (attr, traits, after_derive) = self.classify_item(&mut item); + if attr.is_some() || !traits.is_empty() { + return self.collect_attr(attr, traits, Annotatable::Item(item), + AstFragmentKind::Items, after_derive).make_items(); + } + + match item.kind { + ast::ItemKind::Mac(..) => { + self.check_attributes(&item.attrs); + item.and_then(|item| match item.kind { + ItemKind::Mac(mac) => self.collect( + AstFragmentKind::Items, InvocationKind::Bang { mac, span: item.span } + ).make_items(), + _ => unreachable!(), + }) + } + ast::ItemKind::Mod(ast::Mod { inner, .. }) => { + if item.ident == Ident::invalid() { + return noop_flat_map_item(item, self); + } + + let orig_directory_ownership = self.cx.current_expansion.directory_ownership; + let mut module = (*self.cx.current_expansion.module).clone(); + module.mod_path.push(item.ident); + + // Detect if this is an inline module (`mod m { ... }` as opposed to `mod m;`). + // In the non-inline case, `inner` is never the dummy span (cf. `parse_item_mod`). + // Thus, if `inner` is the dummy span, we know the module is inline. + let inline_module = item.span.contains(inner) || inner.is_dummy(); + + if inline_module { + if let Some(path) = attr::first_attr_value_str_by_name(&item.attrs, sym::path) { + self.cx.current_expansion.directory_ownership = + DirectoryOwnership::Owned { relative: None }; + module.directory.push(&*path.as_str()); + } else { + module.directory.push(&*item.ident.as_str()); + } + } else { + let path = self.cx.parse_sess.source_map().span_to_unmapped_path(inner); + let mut path = match path { + FileName::Real(path) => path, + other => PathBuf::from(other.to_string()), + }; + let directory_ownership = match path.file_name().unwrap().to_str() { + Some("mod.rs") => DirectoryOwnership::Owned { relative: None }, + Some(_) => DirectoryOwnership::Owned { + relative: Some(item.ident), + }, + None => DirectoryOwnership::UnownedViaMod(false), + }; + path.pop(); + module.directory = path; + self.cx.current_expansion.directory_ownership = directory_ownership; + } + + let orig_module = + mem::replace(&mut self.cx.current_expansion.module, Rc::new(module)); + let result = noop_flat_map_item(item, self); + self.cx.current_expansion.module = orig_module; + self.cx.current_expansion.directory_ownership = orig_directory_ownership; + result + } + + _ => noop_flat_map_item(item, self), + } + } + + fn flat_map_trait_item(&mut self, item: ast::TraitItem) -> SmallVec<[ast::TraitItem; 1]> { + let mut item = configure!(self, item); + + let (attr, traits, after_derive) = self.classify_item(&mut item); + if attr.is_some() || !traits.is_empty() { + return self.collect_attr(attr, traits, Annotatable::TraitItem(P(item)), + AstFragmentKind::TraitItems, after_derive).make_trait_items() + } + + match item.kind { + ast::TraitItemKind::Macro(mac) => { + let ast::TraitItem { attrs, span, .. } = item; + self.check_attributes(&attrs); + self.collect_bang(mac, span, AstFragmentKind::TraitItems).make_trait_items() + } + _ => noop_flat_map_trait_item(item, self), + } + } + + fn flat_map_impl_item(&mut self, item: ast::ImplItem) -> SmallVec<[ast::ImplItem; 1]> { + let mut item = configure!(self, item); + + let (attr, traits, after_derive) = self.classify_item(&mut item); + if attr.is_some() || !traits.is_empty() { + return self.collect_attr(attr, traits, Annotatable::ImplItem(P(item)), + AstFragmentKind::ImplItems, after_derive).make_impl_items(); + } + + match item.kind { + ast::ImplItemKind::Macro(mac) => { + let ast::ImplItem { attrs, span, .. } = item; + self.check_attributes(&attrs); + self.collect_bang(mac, span, AstFragmentKind::ImplItems).make_impl_items() + } + _ => noop_flat_map_impl_item(item, self), + } + } + + fn visit_ty(&mut self, ty: &mut P<ast::Ty>) { + match ty.kind { + ast::TyKind::Mac(_) => {} + _ => return noop_visit_ty(ty, self), + }; + + visit_clobber(ty, |mut ty| { + match mem::replace(&mut ty.kind, ast::TyKind::Err) { + ast::TyKind::Mac(mac) => + self.collect_bang(mac, ty.span, AstFragmentKind::Ty).make_ty(), + _ => unreachable!(), + } + }); + } + + fn visit_foreign_mod(&mut self, foreign_mod: &mut ast::ForeignMod) { + self.cfg.configure_foreign_mod(foreign_mod); + noop_visit_foreign_mod(foreign_mod, self); + } + + fn flat_map_foreign_item(&mut self, mut foreign_item: ast::ForeignItem) + -> SmallVec<[ast::ForeignItem; 1]> + { + let (attr, traits, after_derive) = self.classify_item(&mut foreign_item); + + if attr.is_some() || !traits.is_empty() { + return self.collect_attr(attr, traits, Annotatable::ForeignItem(P(foreign_item)), + AstFragmentKind::ForeignItems, after_derive) + .make_foreign_items(); + } + + if let ast::ForeignItemKind::Macro(mac) = foreign_item.kind { + self.check_attributes(&foreign_item.attrs); + return self.collect_bang(mac, foreign_item.span, AstFragmentKind::ForeignItems) + .make_foreign_items(); + } + + noop_flat_map_foreign_item(foreign_item, self) + } + + fn visit_item_kind(&mut self, item: &mut ast::ItemKind) { + match item { + ast::ItemKind::MacroDef(..) => {} + _ => { + self.cfg.configure_item_kind(item); + noop_visit_item_kind(item, self); + } + } + } + + fn flat_map_generic_param( + &mut self, + param: ast::GenericParam + ) -> SmallVec<[ast::GenericParam; 1]> + { + let mut param = configure!(self, param); + + let (attr, traits, after_derive) = self.classify_item(&mut param); + if attr.is_some() || !traits.is_empty() { + return self.collect_attr(attr, traits, Annotatable::GenericParam(param), + AstFragmentKind::GenericParams, after_derive) + .make_generic_params(); + } + + noop_flat_map_generic_param(param, self) + } + + fn visit_attribute(&mut self, at: &mut ast::Attribute) { + // turn `#[doc(include="filename")]` attributes into `#[doc(include(file="filename", + // contents="file contents")]` attributes + if !at.check_name(sym::doc) { + return noop_visit_attribute(at, self); + } + + if let Some(list) = at.meta_item_list() { + if !list.iter().any(|it| it.check_name(sym::include)) { + return noop_visit_attribute(at, self); + } + + let mut items = vec![]; + + for mut it in list { + if !it.check_name(sym::include) { + items.push({ noop_visit_meta_list_item(&mut it, self); it }); + continue; + } + + if let Some(file) = it.value_str() { + let err_count = self.cx.parse_sess.span_diagnostic.err_count(); + self.check_attributes(slice::from_ref(at)); + if self.cx.parse_sess.span_diagnostic.err_count() > err_count { + // avoid loading the file if they haven't enabled the feature + return noop_visit_attribute(at, self); + } + + let filename = self.cx.resolve_path(&*file.as_str(), it.span()); + match self.cx.source_map().load_file(&filename) { + Ok(source_file) => { + let src = source_file.src.as_ref() + .expect("freshly loaded file should have a source"); + let src_interned = Symbol::intern(src.as_str()); + + let include_info = vec![ + ast::NestedMetaItem::MetaItem( + attr::mk_name_value_item_str( + Ident::with_dummy_span(sym::file), + file, + DUMMY_SP, + ), + ), + ast::NestedMetaItem::MetaItem( + attr::mk_name_value_item_str( + Ident::with_dummy_span(sym::contents), + src_interned, + DUMMY_SP, + ), + ), + ]; + + let include_ident = Ident::with_dummy_span(sym::include); + let item = attr::mk_list_item(include_ident, include_info); + items.push(ast::NestedMetaItem::MetaItem(item)); + } + Err(e) => { + let lit = it + .meta_item() + .and_then(|item| item.name_value_literal()) + .unwrap(); + + if e.kind() == ErrorKind::InvalidData { + self.cx + .struct_span_err( + lit.span, + &format!("{} wasn't a utf-8 file", filename.display()), + ) + .span_label(lit.span, "contains invalid utf-8") + .emit(); + } else { + let mut err = self.cx.struct_span_err( + lit.span, + &format!("couldn't read {}: {}", filename.display(), e), + ); + err.span_label(lit.span, "couldn't read file"); + + err.emit(); + } + } + } + } else { + let mut err = self.cx.struct_span_err( + it.span(), + &format!("expected path to external documentation"), + ); + + // Check if the user erroneously used `doc(include(...))` syntax. + let literal = it.meta_item_list().and_then(|list| { + if list.len() == 1 { + list[0].literal().map(|literal| &literal.kind) + } else { + None + } + }); + + let (path, applicability) = match &literal { + Some(LitKind::Str(path, ..)) => { + (path.to_string(), Applicability::MachineApplicable) + } + _ => (String::from("<path>"), Applicability::HasPlaceholders), + }; + + err.span_suggestion( + it.span(), + "provide a file path with `=`", + format!("include = \"{}\"", path), + applicability, + ); + + err.emit(); + } + } + + let meta = attr::mk_list_item(Ident::with_dummy_span(sym::doc), items); + *at = attr::Attribute { + item: AttrItem { path: meta.path, tokens: meta.kind.tokens(meta.span) }, + span: at.span, + id: at.id, + style: at.style, + is_sugared_doc: false, + }; + } else { + noop_visit_attribute(at, self) + } + } + + fn visit_id(&mut self, id: &mut ast::NodeId) { + if self.monotonic { + debug_assert_eq!(*id, ast::DUMMY_NODE_ID); + *id = self.cx.resolver.next_node_id() + } + } + + fn visit_fn_decl(&mut self, mut fn_decl: &mut P<ast::FnDecl>) { + self.cfg.configure_fn_decl(&mut fn_decl); + noop_visit_fn_decl(fn_decl, self); + } +} + +pub struct ExpansionConfig<'feat> { + pub crate_name: String, + pub features: Option<&'feat Features>, + pub recursion_limit: usize, + pub trace_mac: bool, + pub should_test: bool, // If false, strip `#[test]` nodes + pub single_step: bool, + pub keep_macs: bool, +} + +impl<'feat> ExpansionConfig<'feat> { + pub fn default(crate_name: String) -> ExpansionConfig<'static> { + ExpansionConfig { + crate_name, + features: None, + recursion_limit: 1024, + trace_mac: false, + should_test: false, + single_step: false, + keep_macs: false, + } + } + + fn proc_macro_hygiene(&self) -> bool { + self.features.map_or(false, |features| features.proc_macro_hygiene) + } + fn custom_inner_attributes(&self) -> bool { + self.features.map_or(false, |features| features.custom_inner_attributes) + } +} diff --git a/src/libsyntax_expand/lib.rs b/src/libsyntax_expand/lib.rs new file mode 100644 index 00000000000..88e69d79397 --- /dev/null +++ b/src/libsyntax_expand/lib.rs @@ -0,0 +1,38 @@ +#![feature(crate_visibility_modifier)] +#![feature(proc_macro_diagnostic)] +#![feature(proc_macro_internals)] +#![feature(proc_macro_span)] + +extern crate proc_macro as pm; + +// A variant of 'try!' that panics on an Err. This is used as a crutch on the +// way towards a non-panic!-prone parser. It should be used for fatal parsing +// errors; eventually we plan to convert all code using panictry to just use +// normal try. +#[macro_export] +macro_rules! panictry { + ($e:expr) => ({ + use std::result::Result::{Ok, Err}; + use errors::FatalError; + match $e { + Ok(e) => e, + Err(mut e) => { + e.emit(); + FatalError.raise() + } + } + }) +} + +mod placeholders; +mod proc_macro_server; + +pub use syntax_pos::hygiene; +pub use mbe::macro_rules::compile_declarative_macro; +pub mod allocator; +pub mod base; +pub mod build; +pub mod expand; +pub mod proc_macro; + +crate mod mbe; diff --git a/src/libsyntax_expand/mbe.rs b/src/libsyntax_expand/mbe.rs new file mode 100644 index 00000000000..453fe94f1de --- /dev/null +++ b/src/libsyntax_expand/mbe.rs @@ -0,0 +1,166 @@ +//! This module implements declarative macros: old `macro_rules` and the newer +//! `macro`. Declarative macros are also known as "macro by example", and that's +//! why we call this module `mbe`. For external documentation, prefer the +//! official terminology: "declarative macros". + +crate mod transcribe; +crate mod macro_check; +crate mod macro_parser; +crate mod macro_rules; +crate mod quoted; + +use syntax::ast; +use syntax::parse::token::{self, Token, TokenKind}; +use syntax::tokenstream::{DelimSpan}; + +use syntax_pos::{BytePos, Span}; + +use rustc_data_structures::sync::Lrc; + +/// Contains the sub-token-trees of a "delimited" token tree, such as the contents of `(`. Note +/// that the delimiter itself might be `NoDelim`. +#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug)] +struct Delimited { + delim: token::DelimToken, + tts: Vec<TokenTree>, +} + +impl Delimited { + /// Returns a `self::TokenTree` with a `Span` corresponding to the opening delimiter. + fn open_tt(&self, span: Span) -> TokenTree { + let open_span = if span.is_dummy() { + span + } else { + span.with_hi(span.lo() + BytePos(self.delim.len() as u32)) + }; + TokenTree::token(token::OpenDelim(self.delim), open_span) + } + + /// Returns a `self::TokenTree` with a `Span` corresponding to the closing delimiter. + fn close_tt(&self, span: Span) -> TokenTree { + let close_span = if span.is_dummy() { + span + } else { + span.with_lo(span.hi() - BytePos(self.delim.len() as u32)) + }; + TokenTree::token(token::CloseDelim(self.delim), close_span) + } +} + +#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug)] +struct SequenceRepetition { + /// The sequence of token trees + tts: Vec<TokenTree>, + /// The optional separator + separator: Option<Token>, + /// Whether the sequence can be repeated zero (*), or one or more times (+) + kleene: KleeneToken, + /// The number of `Match`s that appear in the sequence (and subsequences) + num_captures: usize, +} + +#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug, Copy)] +struct KleeneToken { + span: Span, + op: KleeneOp, +} + +impl KleeneToken { + fn new(op: KleeneOp, span: Span) -> KleeneToken { + KleeneToken { span, op } + } +} + +/// A Kleene-style [repetition operator](http://en.wikipedia.org/wiki/Kleene_star) +/// for token sequences. +#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] +enum KleeneOp { + /// Kleene star (`*`) for zero or more repetitions + ZeroOrMore, + /// Kleene plus (`+`) for one or more repetitions + OneOrMore, + /// Kleene optional (`?`) for zero or one reptitions + ZeroOrOne, +} + +/// Similar to `tokenstream::TokenTree`, except that `$i`, `$i:ident`, and `$(...)` +/// are "first-class" token trees. Useful for parsing macros. +#[derive(Debug, Clone, PartialEq, RustcEncodable, RustcDecodable)] +enum TokenTree { + Token(Token), + Delimited(DelimSpan, Lrc<Delimited>), + /// A kleene-style repetition sequence + Sequence(DelimSpan, Lrc<SequenceRepetition>), + /// e.g., `$var` + MetaVar(Span, ast::Ident), + /// e.g., `$var:expr`. This is only used in the left hand side of MBE macros. + MetaVarDecl( + Span, + ast::Ident, /* name to bind */ + ast::Ident, /* kind of nonterminal */ + ), +} + +impl TokenTree { + /// Return the number of tokens in the tree. + fn len(&self) -> usize { + match *self { + TokenTree::Delimited(_, ref delimed) => match delimed.delim { + token::NoDelim => delimed.tts.len(), + _ => delimed.tts.len() + 2, + }, + TokenTree::Sequence(_, ref seq) => seq.tts.len(), + _ => 0, + } + } + + /// Returns `true` if the given token tree is delimited. + fn is_delimited(&self) -> bool { + match *self { + TokenTree::Delimited(..) => true, + _ => false, + } + } + + /// Returns `true` if the given token tree is a token of the given kind. + fn is_token(&self, expected_kind: &TokenKind) -> bool { + match self { + TokenTree::Token(Token { kind: actual_kind, .. }) => actual_kind == expected_kind, + _ => false, + } + } + + /// Gets the `index`-th sub-token-tree. This only makes sense for delimited trees and sequences. + fn get_tt(&self, index: usize) -> TokenTree { + match (self, index) { + (&TokenTree::Delimited(_, ref delimed), _) if delimed.delim == token::NoDelim => { + delimed.tts[index].clone() + } + (&TokenTree::Delimited(span, ref delimed), _) => { + if index == 0 { + return delimed.open_tt(span.open); + } + if index == delimed.tts.len() + 1 { + return delimed.close_tt(span.close); + } + delimed.tts[index - 1].clone() + } + (&TokenTree::Sequence(_, ref seq), _) => seq.tts[index].clone(), + _ => panic!("Cannot expand a token tree"), + } + } + + /// Retrieves the `TokenTree`'s span. + fn span(&self) -> Span { + match *self { + TokenTree::Token(Token { span, .. }) + | TokenTree::MetaVar(span, _) + | TokenTree::MetaVarDecl(span, _, _) => span, + TokenTree::Delimited(span, _) | TokenTree::Sequence(span, _) => span.entire(), + } + } + + fn token(kind: TokenKind, span: Span) -> TokenTree { + TokenTree::Token(Token::new(kind, span)) + } +} diff --git a/src/libsyntax_expand/mbe/macro_check.rs b/src/libsyntax_expand/mbe/macro_check.rs new file mode 100644 index 00000000000..50abda8d45e --- /dev/null +++ b/src/libsyntax_expand/mbe/macro_check.rs @@ -0,0 +1,626 @@ +//! Checks that meta-variables in macro definition are correctly declared and used. +//! +//! # What is checked +//! +//! ## Meta-variables must not be bound twice +//! +//! ``` +//! macro_rules! foo { ($x:tt $x:tt) => { $x }; } +//! ``` +//! +//! This check is sound (no false-negative) and complete (no false-positive). +//! +//! ## Meta-variables must not be free +//! +//! ``` +//! macro_rules! foo { () => { $x }; } +//! ``` +//! +//! This check is also done at macro instantiation but only if the branch is taken. +//! +//! ## Meta-variables must repeat at least as many times as their binder +//! +//! ``` +//! macro_rules! foo { ($($x:tt)*) => { $x }; } +//! ``` +//! +//! This check is also done at macro instantiation but only if the branch is taken. +//! +//! ## Meta-variables must repeat with the same Kleene operators as their binder +//! +//! ``` +//! macro_rules! foo { ($($x:tt)+) => { $($x)* }; } +//! ``` +//! +//! This check is not done at macro instantiation. +//! +//! # Disclaimer +//! +//! In the presence of nested macros (a macro defined in a macro), those checks may have false +//! positives and false negatives. We try to detect those cases by recognizing potential macro +//! definitions in RHSes, but nested macros may be hidden through the use of particular values of +//! meta-variables. +//! +//! ## Examples of false positive +//! +//! False positives can come from cases where we don't recognize a nested macro, because it depends +//! on particular values of meta-variables. In the following example, we think both instances of +//! `$x` are free, which is a correct statement if `$name` is anything but `macro_rules`. But when +//! `$name` is `macro_rules`, like in the instantiation below, then `$x:tt` is actually a binder of +//! the nested macro and `$x` is bound to it. +//! +//! ``` +//! macro_rules! foo { ($name:ident) => { $name! bar { ($x:tt) => { $x }; } }; } +//! foo!(macro_rules); +//! ``` +//! +//! False positives can also come from cases where we think there is a nested macro while there +//! isn't. In the following example, we think `$x` is free, which is incorrect because `bar` is not +//! a nested macro since it is not evaluated as code by `stringify!`. +//! +//! ``` +//! macro_rules! foo { () => { stringify!(macro_rules! bar { () => { $x }; }) }; } +//! ``` +//! +//! ## Examples of false negative +//! +//! False negatives can come from cases where we don't recognize a meta-variable, because it depends +//! on particular values of meta-variables. In the following examples, we don't see that if `$d` is +//! instantiated with `$` then `$d z` becomes `$z` in the nested macro definition and is thus a free +//! meta-variable. Note however, that if `foo` is instantiated, then we would check the definition +//! of `bar` and would see the issue. +//! +//! ``` +//! macro_rules! foo { ($d:tt) => { macro_rules! bar { ($y:tt) => { $d z }; } }; } +//! ``` +//! +//! # How it is checked +//! +//! There are 3 main functions: `check_binders`, `check_occurrences`, and `check_nested_macro`. They +//! all need some kind of environment. +//! +//! ## Environments +//! +//! Environments are used to pass information. +//! +//! ### From LHS to RHS +//! +//! When checking a LHS with `check_binders`, we produce (and use) an environment for binders, +//! namely `Binders`. This is a mapping from binder name to information about that binder: the span +//! of the binder for error messages and the stack of Kleene operators under which it was bound in +//! the LHS. +//! +//! This environment is used by both the LHS and RHS. The LHS uses it to detect duplicate binders. +//! The RHS uses it to detect the other errors. +//! +//! ### From outer macro to inner macro +//! +//! When checking the RHS of an outer macro and we detect a nested macro definition, we push the +//! current state, namely `MacroState`, to an environment of nested macro definitions. Each state +//! stores the LHS binders when entering the macro definition as well as the stack of Kleene +//! operators under which the inner macro is defined in the RHS. +//! +//! This environment is a stack representing the nesting of macro definitions. As such, the stack of +//! Kleene operators under which a meta-variable is repeating is the concatenation of the stacks +//! stored when entering a macro definition starting from the state in which the meta-variable is +//! bound. +use crate::mbe::{KleeneToken, TokenTree}; + +use syntax::ast::NodeId; +use syntax::early_buffered_lints::BufferedEarlyLintId; +use syntax::parse::token::{DelimToken, Token, TokenKind}; +use syntax::sess::ParseSess; +use syntax::symbol::{kw, sym}; + +use rustc_data_structures::fx::FxHashMap; +use smallvec::SmallVec; +use syntax_pos::{symbol::Ident, MultiSpan, Span}; + +/// Stack represented as linked list. +/// +/// Those are used for environments because they grow incrementally and are not mutable. +enum Stack<'a, T> { + /// Empty stack. + Empty, + /// A non-empty stack. + Push { + /// The top element. + top: T, + /// The previous elements. + prev: &'a Stack<'a, T>, + }, +} + +impl<'a, T> Stack<'a, T> { + /// Returns whether a stack is empty. + fn is_empty(&self) -> bool { + match *self { + Stack::Empty => true, + _ => false, + } + } + + /// Returns a new stack with an element of top. + fn push(&'a self, top: T) -> Stack<'a, T> { + Stack::Push { top, prev: self } + } +} + +impl<'a, T> Iterator for &'a Stack<'a, T> { + type Item = &'a T; + + // Iterates from top to bottom of the stack. + fn next(&mut self) -> Option<&'a T> { + match *self { + Stack::Empty => None, + Stack::Push { ref top, ref prev } => { + *self = prev; + Some(top) + } + } + } +} + +impl From<&Stack<'_, KleeneToken>> for SmallVec<[KleeneToken; 1]> { + fn from(ops: &Stack<'_, KleeneToken>) -> SmallVec<[KleeneToken; 1]> { + let mut ops: SmallVec<[KleeneToken; 1]> = ops.cloned().collect(); + // The stack is innermost on top. We want outermost first. + ops.reverse(); + ops + } +} + +/// Information attached to a meta-variable binder in LHS. +struct BinderInfo { + /// The span of the meta-variable in LHS. + span: Span, + /// The stack of Kleene operators (outermost first). + ops: SmallVec<[KleeneToken; 1]>, +} + +/// An environment of meta-variables to their binder information. +type Binders = FxHashMap<Ident, BinderInfo>; + +/// The state at which we entered a macro definition in the RHS of another macro definition. +struct MacroState<'a> { + /// The binders of the branch where we entered the macro definition. + binders: &'a Binders, + /// The stack of Kleene operators (outermost first) where we entered the macro definition. + ops: SmallVec<[KleeneToken; 1]>, +} + +/// Checks that meta-variables are used correctly in a macro definition. +/// +/// Arguments: +/// - `sess` is used to emit diagnostics and lints +/// - `node_id` is used to emit lints +/// - `span` is used when no spans are available +/// - `lhses` and `rhses` should have the same length and represent the macro definition +pub(super) fn check_meta_variables( + sess: &ParseSess, + node_id: NodeId, + span: Span, + lhses: &[TokenTree], + rhses: &[TokenTree], +) -> bool { + if lhses.len() != rhses.len() { + sess.span_diagnostic.span_bug(span, "length mismatch between LHSes and RHSes") + } + let mut valid = true; + for (lhs, rhs) in lhses.iter().zip(rhses.iter()) { + let mut binders = Binders::default(); + check_binders(sess, node_id, lhs, &Stack::Empty, &mut binders, &Stack::Empty, &mut valid); + check_occurrences(sess, node_id, rhs, &Stack::Empty, &binders, &Stack::Empty, &mut valid); + } + valid +} + +/// Checks `lhs` as part of the LHS of a macro definition, extends `binders` with new binders, and +/// sets `valid` to false in case of errors. +/// +/// Arguments: +/// - `sess` is used to emit diagnostics and lints +/// - `node_id` is used to emit lints +/// - `lhs` is checked as part of a LHS +/// - `macros` is the stack of possible outer macros +/// - `binders` contains the binders of the LHS +/// - `ops` is the stack of Kleene operators from the LHS +/// - `valid` is set in case of errors +fn check_binders( + sess: &ParseSess, + node_id: NodeId, + lhs: &TokenTree, + macros: &Stack<'_, MacroState<'_>>, + binders: &mut Binders, + ops: &Stack<'_, KleeneToken>, + valid: &mut bool, +) { + match *lhs { + TokenTree::Token(..) => {} + // This can only happen when checking a nested macro because this LHS is then in the RHS of + // the outer macro. See ui/macros/macro-of-higher-order.rs where $y:$fragment in the + // LHS of the nested macro (and RHS of the outer macro) is parsed as MetaVar(y) Colon + // MetaVar(fragment) and not as MetaVarDecl(y, fragment). + TokenTree::MetaVar(span, name) => { + if macros.is_empty() { + sess.span_diagnostic.span_bug(span, "unexpected MetaVar in lhs"); + } + // There are 3 possibilities: + if let Some(prev_info) = binders.get(&name) { + // 1. The meta-variable is already bound in the current LHS: This is an error. + let mut span = MultiSpan::from_span(span); + span.push_span_label(prev_info.span, "previous declaration".into()); + buffer_lint(sess, span, node_id, "duplicate matcher binding"); + } else if get_binder_info(macros, binders, name).is_none() { + // 2. The meta-variable is free: This is a binder. + binders.insert(name, BinderInfo { span, ops: ops.into() }); + } else { + // 3. The meta-variable is bound: This is an occurrence. + check_occurrences(sess, node_id, lhs, macros, binders, ops, valid); + } + } + // Similarly, this can only happen when checking a toplevel macro. + TokenTree::MetaVarDecl(span, name, _kind) => { + if !macros.is_empty() { + sess.span_diagnostic.span_bug(span, "unexpected MetaVarDecl in nested lhs"); + } + if let Some(prev_info) = get_binder_info(macros, binders, name) { + // Duplicate binders at the top-level macro definition are errors. The lint is only + // for nested macro definitions. + sess.span_diagnostic + .struct_span_err(span, "duplicate matcher binding") + .span_note(prev_info.span, "previous declaration was here") + .emit(); + *valid = false; + } else { + binders.insert(name, BinderInfo { span, ops: ops.into() }); + } + } + TokenTree::Delimited(_, ref del) => { + for tt in &del.tts { + check_binders(sess, node_id, tt, macros, binders, ops, valid); + } + } + TokenTree::Sequence(_, ref seq) => { + let ops = ops.push(seq.kleene); + for tt in &seq.tts { + check_binders(sess, node_id, tt, macros, binders, &ops, valid); + } + } + } +} + +/// Returns the binder information of a meta-variable. +/// +/// Arguments: +/// - `macros` is the stack of possible outer macros +/// - `binders` contains the current binders +/// - `name` is the name of the meta-variable we are looking for +fn get_binder_info<'a>( + mut macros: &'a Stack<'a, MacroState<'a>>, + binders: &'a Binders, + name: Ident, +) -> Option<&'a BinderInfo> { + binders.get(&name).or_else(|| macros.find_map(|state| state.binders.get(&name))) +} + +/// Checks `rhs` as part of the RHS of a macro definition and sets `valid` to false in case of +/// errors. +/// +/// Arguments: +/// - `sess` is used to emit diagnostics and lints +/// - `node_id` is used to emit lints +/// - `rhs` is checked as part of a RHS +/// - `macros` is the stack of possible outer macros +/// - `binders` contains the binders of the associated LHS +/// - `ops` is the stack of Kleene operators from the RHS +/// - `valid` is set in case of errors +fn check_occurrences( + sess: &ParseSess, + node_id: NodeId, + rhs: &TokenTree, + macros: &Stack<'_, MacroState<'_>>, + binders: &Binders, + ops: &Stack<'_, KleeneToken>, + valid: &mut bool, +) { + match *rhs { + TokenTree::Token(..) => {} + TokenTree::MetaVarDecl(span, _name, _kind) => { + sess.span_diagnostic.span_bug(span, "unexpected MetaVarDecl in rhs") + } + TokenTree::MetaVar(span, name) => { + check_ops_is_prefix(sess, node_id, macros, binders, ops, span, name); + } + TokenTree::Delimited(_, ref del) => { + check_nested_occurrences(sess, node_id, &del.tts, macros, binders, ops, valid); + } + TokenTree::Sequence(_, ref seq) => { + let ops = ops.push(seq.kleene); + check_nested_occurrences(sess, node_id, &seq.tts, macros, binders, &ops, valid); + } + } +} + +/// Represents the processed prefix of a nested macro. +#[derive(Clone, Copy, PartialEq, Eq)] +enum NestedMacroState { + /// Nothing that matches a nested macro definition was processed yet. + Empty, + /// The token `macro_rules` was processed. + MacroRules, + /// The tokens `macro_rules!` were processed. + MacroRulesNot, + /// The tokens `macro_rules!` followed by a name were processed. The name may be either directly + /// an identifier or a meta-variable (that hopefully would be instantiated by an identifier). + MacroRulesNotName, + /// The keyword `macro` was processed. + Macro, + /// The keyword `macro` followed by a name was processed. + MacroName, + /// The keyword `macro` followed by a name and a token delimited by parentheses was processed. + MacroNameParen, +} + +/// Checks `tts` as part of the RHS of a macro definition, tries to recognize nested macro +/// definitions, and sets `valid` to false in case of errors. +/// +/// Arguments: +/// - `sess` is used to emit diagnostics and lints +/// - `node_id` is used to emit lints +/// - `tts` is checked as part of a RHS and may contain macro definitions +/// - `macros` is the stack of possible outer macros +/// - `binders` contains the binders of the associated LHS +/// - `ops` is the stack of Kleene operators from the RHS +/// - `valid` is set in case of errors +fn check_nested_occurrences( + sess: &ParseSess, + node_id: NodeId, + tts: &[TokenTree], + macros: &Stack<'_, MacroState<'_>>, + binders: &Binders, + ops: &Stack<'_, KleeneToken>, + valid: &mut bool, +) { + let mut state = NestedMacroState::Empty; + let nested_macros = macros.push(MacroState { binders, ops: ops.into() }); + let mut nested_binders = Binders::default(); + for tt in tts { + match (state, tt) { + ( + NestedMacroState::Empty, + &TokenTree::Token(Token { kind: TokenKind::Ident(name, false), .. }), + ) => { + if name == sym::macro_rules { + state = NestedMacroState::MacroRules; + } else if name == kw::Macro { + state = NestedMacroState::Macro; + } + } + ( + NestedMacroState::MacroRules, + &TokenTree::Token(Token { kind: TokenKind::Not, .. }), + ) => { + state = NestedMacroState::MacroRulesNot; + } + ( + NestedMacroState::MacroRulesNot, + &TokenTree::Token(Token { kind: TokenKind::Ident(..), .. }), + ) => { + state = NestedMacroState::MacroRulesNotName; + } + (NestedMacroState::MacroRulesNot, &TokenTree::MetaVar(..)) => { + state = NestedMacroState::MacroRulesNotName; + // We check that the meta-variable is correctly used. + check_occurrences(sess, node_id, tt, macros, binders, ops, valid); + } + (NestedMacroState::MacroRulesNotName, &TokenTree::Delimited(_, ref del)) + | (NestedMacroState::MacroName, &TokenTree::Delimited(_, ref del)) + if del.delim == DelimToken::Brace => + { + let legacy = state == NestedMacroState::MacroRulesNotName; + state = NestedMacroState::Empty; + let rest = + check_nested_macro(sess, node_id, legacy, &del.tts, &nested_macros, valid); + // If we did not check the whole macro definition, then check the rest as if outside + // the macro definition. + check_nested_occurrences( + sess, + node_id, + &del.tts[rest..], + macros, + binders, + ops, + valid, + ); + } + ( + NestedMacroState::Macro, + &TokenTree::Token(Token { kind: TokenKind::Ident(..), .. }), + ) => { + state = NestedMacroState::MacroName; + } + (NestedMacroState::Macro, &TokenTree::MetaVar(..)) => { + state = NestedMacroState::MacroName; + // We check that the meta-variable is correctly used. + check_occurrences(sess, node_id, tt, macros, binders, ops, valid); + } + (NestedMacroState::MacroName, &TokenTree::Delimited(_, ref del)) + if del.delim == DelimToken::Paren => + { + state = NestedMacroState::MacroNameParen; + nested_binders = Binders::default(); + check_binders( + sess, + node_id, + tt, + &nested_macros, + &mut nested_binders, + &Stack::Empty, + valid, + ); + } + (NestedMacroState::MacroNameParen, &TokenTree::Delimited(_, ref del)) + if del.delim == DelimToken::Brace => + { + state = NestedMacroState::Empty; + check_occurrences( + sess, + node_id, + tt, + &nested_macros, + &nested_binders, + &Stack::Empty, + valid, + ); + } + (_, ref tt) => { + state = NestedMacroState::Empty; + check_occurrences(sess, node_id, tt, macros, binders, ops, valid); + } + } + } +} + +/// Checks the body of nested macro, returns where the check stopped, and sets `valid` to false in +/// case of errors. +/// +/// The token trees are checked as long as they look like a list of (LHS) => {RHS} token trees. This +/// check is a best-effort to detect a macro definition. It returns the position in `tts` where we +/// stopped checking because we detected we were not in a macro definition anymore. +/// +/// Arguments: +/// - `sess` is used to emit diagnostics and lints +/// - `node_id` is used to emit lints +/// - `legacy` specifies whether the macro is legacy +/// - `tts` is checked as a list of (LHS) => {RHS} +/// - `macros` is the stack of outer macros +/// - `valid` is set in case of errors +fn check_nested_macro( + sess: &ParseSess, + node_id: NodeId, + legacy: bool, + tts: &[TokenTree], + macros: &Stack<'_, MacroState<'_>>, + valid: &mut bool, +) -> usize { + let n = tts.len(); + let mut i = 0; + let separator = if legacy { TokenKind::Semi } else { TokenKind::Comma }; + loop { + // We expect 3 token trees: `(LHS) => {RHS}`. The separator is checked after. + if i + 2 >= n + || !tts[i].is_delimited() + || !tts[i + 1].is_token(&TokenKind::FatArrow) + || !tts[i + 2].is_delimited() + { + break; + } + let lhs = &tts[i]; + let rhs = &tts[i + 2]; + let mut binders = Binders::default(); + check_binders(sess, node_id, lhs, macros, &mut binders, &Stack::Empty, valid); + check_occurrences(sess, node_id, rhs, macros, &binders, &Stack::Empty, valid); + // Since the last semicolon is optional for legacy macros and decl_macro are not terminated, + // we increment our checked position by how many token trees we already checked (the 3 + // above) before checking for the separator. + i += 3; + if i == n || !tts[i].is_token(&separator) { + break; + } + // We increment our checked position for the semicolon. + i += 1; + } + i +} + +/// Checks that a meta-variable occurrence is valid. +/// +/// Arguments: +/// - `sess` is used to emit diagnostics and lints +/// - `node_id` is used to emit lints +/// - `macros` is the stack of possible outer macros +/// - `binders` contains the binders of the associated LHS +/// - `ops` is the stack of Kleene operators from the RHS +/// - `span` is the span of the meta-variable to check +/// - `name` is the name of the meta-variable to check +fn check_ops_is_prefix( + sess: &ParseSess, + node_id: NodeId, + macros: &Stack<'_, MacroState<'_>>, + binders: &Binders, + ops: &Stack<'_, KleeneToken>, + span: Span, + name: Ident, +) { + let macros = macros.push(MacroState { binders, ops: ops.into() }); + // Accumulates the stacks the operators of each state until (and including when) the + // meta-variable is found. The innermost stack is first. + let mut acc: SmallVec<[&SmallVec<[KleeneToken; 1]>; 1]> = SmallVec::new(); + for state in ¯os { + acc.push(&state.ops); + if let Some(binder) = state.binders.get(&name) { + // This variable concatenates the stack of operators from the RHS of the LHS where the + // meta-variable was defined to where it is used (in possibly nested macros). The + // outermost operator is first. + let mut occurrence_ops: SmallVec<[KleeneToken; 2]> = SmallVec::new(); + // We need to iterate from the end to start with outermost stack. + for ops in acc.iter().rev() { + occurrence_ops.extend_from_slice(ops); + } + ops_is_prefix(sess, node_id, span, name, &binder.ops, &occurrence_ops); + return; + } + } + buffer_lint(sess, span.into(), node_id, &format!("unknown macro variable `{}`", name)); +} + +/// Returns whether `binder_ops` is a prefix of `occurrence_ops`. +/// +/// The stack of Kleene operators of a meta-variable occurrence just needs to have the stack of +/// Kleene operators of its binder as a prefix. +/// +/// Consider $i in the following example: +/// +/// ( $( $i:ident = $($j:ident),+ );* ) => { $($( $i += $j; )+)* } +/// +/// It occurs under the Kleene stack ["*", "+"] and is bound under ["*"] only. +/// +/// Arguments: +/// - `sess` is used to emit diagnostics and lints +/// - `node_id` is used to emit lints +/// - `span` is the span of the meta-variable being check +/// - `name` is the name of the meta-variable being check +/// - `binder_ops` is the stack of Kleene operators for the binder +/// - `occurrence_ops` is the stack of Kleene operators for the occurrence +fn ops_is_prefix( + sess: &ParseSess, + node_id: NodeId, + span: Span, + name: Ident, + binder_ops: &[KleeneToken], + occurrence_ops: &[KleeneToken], +) { + for (i, binder) in binder_ops.iter().enumerate() { + if i >= occurrence_ops.len() { + let mut span = MultiSpan::from_span(span); + span.push_span_label(binder.span, "expected repetition".into()); + let message = &format!("variable '{}' is still repeating at this depth", name); + buffer_lint(sess, span, node_id, message); + return; + } + let occurrence = &occurrence_ops[i]; + if occurrence.op != binder.op { + let mut span = MultiSpan::from_span(span); + span.push_span_label(binder.span, "expected repetition".into()); + span.push_span_label(occurrence.span, "conflicting repetition".into()); + let message = "meta-variable repeats with different Kleene operator"; + buffer_lint(sess, span, node_id, message); + return; + } + } +} + +fn buffer_lint(sess: &ParseSess, span: MultiSpan, node_id: NodeId, message: &str) { + sess.buffer_lint(BufferedEarlyLintId::MetaVariableMisuse, span, node_id, message); +} diff --git a/src/libsyntax_expand/mbe/macro_parser.rs b/src/libsyntax_expand/mbe/macro_parser.rs new file mode 100644 index 00000000000..2edb4925512 --- /dev/null +++ b/src/libsyntax_expand/mbe/macro_parser.rs @@ -0,0 +1,944 @@ +//! This is an NFA-based parser, which calls out to the main rust parser for named non-terminals +//! (which it commits to fully when it hits one in a grammar). There's a set of current NFA threads +//! and a set of next ones. Instead of NTs, we have a special case for Kleene star. The big-O, in +//! pathological cases, is worse than traditional use of NFA or Earley parsing, but it's an easier +//! fit for Macro-by-Example-style rules. +//! +//! (In order to prevent the pathological case, we'd need to lazily construct the resulting +//! `NamedMatch`es at the very end. It'd be a pain, and require more memory to keep around old +//! items, but it would also save overhead) +//! +//! We don't say this parser uses the Earley algorithm, because it's unnecessarily inaccurate. +//! The macro parser restricts itself to the features of finite state automata. Earley parsers +//! can be described as an extension of NFAs with completion rules, prediction rules, and recursion. +//! +//! Quick intro to how the parser works: +//! +//! A 'position' is a dot in the middle of a matcher, usually represented as a +//! dot. For example `· a $( a )* a b` is a position, as is `a $( · a )* a b`. +//! +//! The parser walks through the input a character at a time, maintaining a list +//! of threads consistent with the current position in the input string: `cur_items`. +//! +//! As it processes them, it fills up `eof_items` with threads that would be valid if +//! the macro invocation is now over, `bb_items` with threads that are waiting on +//! a Rust non-terminal like `$e:expr`, and `next_items` with threads that are waiting +//! on a particular token. Most of the logic concerns moving the · through the +//! repetitions indicated by Kleene stars. The rules for moving the · without +//! consuming any input are called epsilon transitions. It only advances or calls +//! out to the real Rust parser when no `cur_items` threads remain. +//! +//! Example: +//! +//! ```text, ignore +//! Start parsing a a a a b against [· a $( a )* a b]. +//! +//! Remaining input: a a a a b +//! next: [· a $( a )* a b] +//! +//! - - - Advance over an a. - - - +//! +//! Remaining input: a a a b +//! cur: [a · $( a )* a b] +//! Descend/Skip (first item). +//! next: [a $( · a )* a b] [a $( a )* · a b]. +//! +//! - - - Advance over an a. - - - +//! +//! Remaining input: a a b +//! cur: [a $( a · )* a b] [a $( a )* a · b] +//! Follow epsilon transition: Finish/Repeat (first item) +//! next: [a $( a )* · a b] [a $( · a )* a b] [a $( a )* a · b] +//! +//! - - - Advance over an a. - - - (this looks exactly like the last step) +//! +//! Remaining input: a b +//! cur: [a $( a · )* a b] [a $( a )* a · b] +//! Follow epsilon transition: Finish/Repeat (first item) +//! next: [a $( a )* · a b] [a $( · a )* a b] [a $( a )* a · b] +//! +//! - - - Advance over an a. - - - (this looks exactly like the last step) +//! +//! Remaining input: b +//! cur: [a $( a · )* a b] [a $( a )* a · b] +//! Follow epsilon transition: Finish/Repeat (first item) +//! next: [a $( a )* · a b] [a $( · a )* a b] [a $( a )* a · b] +//! +//! - - - Advance over a b. - - - +//! +//! Remaining input: '' +//! eof: [a $( a )* a b ·] +//! ``` + +crate use NamedMatch::*; +crate use ParseResult::*; +use TokenTreeOrTokenTreeSlice::*; + +use crate::mbe::{self, TokenTree}; + +use syntax::ast::{Ident, Name}; +use syntax::parse::{Directory, PResult}; +use syntax::parse::parser::{Parser, PathStyle}; +use syntax::parse::token::{self, DocComment, Nonterminal, Token}; +use syntax::print::pprust; +use syntax::sess::ParseSess; +use syntax::symbol::{kw, sym, Symbol}; +use syntax::tokenstream::{DelimSpan, TokenStream}; + +use errors::FatalError; +use smallvec::{smallvec, SmallVec}; +use syntax_pos::Span; + +use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::sync::Lrc; +use std::collections::hash_map::Entry::{Occupied, Vacant}; +use std::mem; +use std::ops::{Deref, DerefMut}; + +// To avoid costly uniqueness checks, we require that `MatchSeq` always has a nonempty body. + +/// Either a sequence of token trees or a single one. This is used as the representation of the +/// sequence of tokens that make up a matcher. +#[derive(Clone)] +enum TokenTreeOrTokenTreeSlice<'tt> { + Tt(TokenTree), + TtSeq(&'tt [TokenTree]), +} + +impl<'tt> TokenTreeOrTokenTreeSlice<'tt> { + /// Returns the number of constituent top-level token trees of `self` (top-level in that it + /// will not recursively descend into subtrees). + fn len(&self) -> usize { + match *self { + TtSeq(ref v) => v.len(), + Tt(ref tt) => tt.len(), + } + } + + /// The `index`-th token tree of `self`. + fn get_tt(&self, index: usize) -> TokenTree { + match *self { + TtSeq(ref v) => v[index].clone(), + Tt(ref tt) => tt.get_tt(index), + } + } +} + +/// An unzipping of `TokenTree`s... see the `stack` field of `MatcherPos`. +/// +/// This is used by `inner_parse_loop` to keep track of delimited submatchers that we have +/// descended into. +#[derive(Clone)] +struct MatcherTtFrame<'tt> { + /// The "parent" matcher that we are descending into. + elts: TokenTreeOrTokenTreeSlice<'tt>, + /// The position of the "dot" in `elts` at the time we descended. + idx: usize, +} + +type NamedMatchVec = SmallVec<[NamedMatch; 4]>; + +/// Represents a single "position" (aka "matcher position", aka "item"), as +/// described in the module documentation. +/// +/// Here: +/// +/// - `'root` represents the lifetime of the stack slot that holds the root +/// `MatcherPos`. As described in `MatcherPosHandle`, the root `MatcherPos` +/// structure is stored on the stack, but subsequent instances are put into +/// the heap. +/// - `'tt` represents the lifetime of the token trees that this matcher +/// position refers to. +/// +/// It is important to distinguish these two lifetimes because we have a +/// `SmallVec<TokenTreeOrTokenTreeSlice<'tt>>` below, and the destructor of +/// that is considered to possibly access the data from its elements (it lacks +/// a `#[may_dangle]` attribute). As a result, the compiler needs to know that +/// all the elements in that `SmallVec` strictly outlive the root stack slot +/// lifetime. By separating `'tt` from `'root`, we can show that. +#[derive(Clone)] +struct MatcherPos<'root, 'tt> { + /// The token or sequence of tokens that make up the matcher + top_elts: TokenTreeOrTokenTreeSlice<'tt>, + + /// The position of the "dot" in this matcher + idx: usize, + + /// The first span of source that the beginning of this matcher corresponds to. In other + /// words, the token in the source whose span is `sp_open` is matched against the first token of + /// the matcher. + sp_open: Span, + + /// For each named metavar in the matcher, we keep track of token trees matched against the + /// metavar by the black box parser. In particular, there may be more than one match per + /// metavar if we are in a repetition (each repetition matches each of the variables). + /// Moreover, matchers and repetitions can be nested; the `matches` field is shared (hence the + /// `Rc`) among all "nested" matchers. `match_lo`, `match_cur`, and `match_hi` keep track of + /// the current position of the `self` matcher position in the shared `matches` list. + /// + /// Also, note that while we are descending into a sequence, matchers are given their own + /// `matches` vector. Only once we reach the end of a full repetition of the sequence do we add + /// all bound matches from the submatcher into the shared top-level `matches` vector. If `sep` + /// and `up` are `Some`, then `matches` is _not_ the shared top-level list. Instead, if one + /// wants the shared `matches`, one should use `up.matches`. + matches: Box<[Lrc<NamedMatchVec>]>, + /// The position in `matches` corresponding to the first metavar in this matcher's sequence of + /// token trees. In other words, the first metavar in the first token of `top_elts` corresponds + /// to `matches[match_lo]`. + match_lo: usize, + /// The position in `matches` corresponding to the metavar we are currently trying to match + /// against the source token stream. `match_lo <= match_cur <= match_hi`. + match_cur: usize, + /// Similar to `match_lo` except `match_hi` is the position in `matches` of the _last_ metavar + /// in this matcher. + match_hi: usize, + + // The following fields are used if we are matching a repetition. If we aren't, they should be + // `None`. + + /// The KleeneOp of this sequence if we are in a repetition. + seq_op: Option<mbe::KleeneOp>, + + /// The separator if we are in a repetition. + sep: Option<Token>, + + /// The "parent" matcher position if we are in a repetition. That is, the matcher position just + /// before we enter the sequence. + up: Option<MatcherPosHandle<'root, 'tt>>, + + /// Specifically used to "unzip" token trees. By "unzip", we mean to unwrap the delimiters from + /// a delimited token tree (e.g., something wrapped in `(` `)`) or to get the contents of a doc + /// comment... + /// + /// When matching against matchers with nested delimited submatchers (e.g., `pat ( pat ( .. ) + /// pat ) pat`), we need to keep track of the matchers we are descending into. This stack does + /// that where the bottom of the stack is the outermost matcher. + /// Also, throughout the comments, this "descent" is often referred to as "unzipping"... + stack: SmallVec<[MatcherTtFrame<'tt>; 1]>, +} + +impl<'root, 'tt> MatcherPos<'root, 'tt> { + /// Adds `m` as a named match for the `idx`-th metavar. + fn push_match(&mut self, idx: usize, m: NamedMatch) { + let matches = Lrc::make_mut(&mut self.matches[idx]); + matches.push(m); + } +} + +// Lots of MatcherPos instances are created at runtime. Allocating them on the +// heap is slow. Furthermore, using SmallVec<MatcherPos> to allocate them all +// on the stack is also slow, because MatcherPos is quite a large type and +// instances get moved around a lot between vectors, which requires lots of +// slow memcpy calls. +// +// Therefore, the initial MatcherPos is always allocated on the stack, +// subsequent ones (of which there aren't that many) are allocated on the heap, +// and this type is used to encapsulate both cases. +enum MatcherPosHandle<'root, 'tt> { + Ref(&'root mut MatcherPos<'root, 'tt>), + Box(Box<MatcherPos<'root, 'tt>>), +} + +impl<'root, 'tt> Clone for MatcherPosHandle<'root, 'tt> { + // This always produces a new Box. + fn clone(&self) -> Self { + MatcherPosHandle::Box(match *self { + MatcherPosHandle::Ref(ref r) => Box::new((**r).clone()), + MatcherPosHandle::Box(ref b) => b.clone(), + }) + } +} + +impl<'root, 'tt> Deref for MatcherPosHandle<'root, 'tt> { + type Target = MatcherPos<'root, 'tt>; + fn deref(&self) -> &Self::Target { + match *self { + MatcherPosHandle::Ref(ref r) => r, + MatcherPosHandle::Box(ref b) => b, + } + } +} + +impl<'root, 'tt> DerefMut for MatcherPosHandle<'root, 'tt> { + fn deref_mut(&mut self) -> &mut MatcherPos<'root, 'tt> { + match *self { + MatcherPosHandle::Ref(ref mut r) => r, + MatcherPosHandle::Box(ref mut b) => b, + } + } +} + +/// Represents the possible results of an attempted parse. +crate enum ParseResult<T> { + /// Parsed successfully. + Success(T), + /// Arm failed to match. If the second parameter is `token::Eof`, it indicates an unexpected + /// end of macro invocation. Otherwise, it indicates that no rules expected the given token. + Failure(Token, &'static str), + /// Fatal error (malformed macro?). Abort compilation. + Error(syntax_pos::Span, String), +} + +/// A `ParseResult` where the `Success` variant contains a mapping of `Ident`s to `NamedMatch`es. +/// This represents the mapping of metavars to the token trees they bind to. +crate type NamedParseResult = ParseResult<FxHashMap<Ident, NamedMatch>>; + +/// Count how many metavars are named in the given matcher `ms`. +pub(super) fn count_names(ms: &[TokenTree]) -> usize { + ms.iter().fold(0, |count, elt| { + count + match *elt { + TokenTree::Sequence(_, ref seq) => seq.num_captures, + TokenTree::Delimited(_, ref delim) => count_names(&delim.tts), + TokenTree::MetaVar(..) => 0, + TokenTree::MetaVarDecl(..) => 1, + TokenTree::Token(..) => 0, + } + }) +} + +/// `len` `Vec`s (initially shared and empty) that will store matches of metavars. +fn create_matches(len: usize) -> Box<[Lrc<NamedMatchVec>]> { + if len == 0 { + vec![] + } else { + let empty_matches = Lrc::new(SmallVec::new()); + vec![empty_matches; len] + }.into_boxed_slice() +} + +/// Generates the top-level matcher position in which the "dot" is before the first token of the +/// matcher `ms` and we are going to start matching at the span `open` in the source. +fn initial_matcher_pos<'root, 'tt>(ms: &'tt [TokenTree], open: Span) -> MatcherPos<'root, 'tt> { + let match_idx_hi = count_names(ms); + let matches = create_matches(match_idx_hi); + MatcherPos { + // Start with the top level matcher given to us + top_elts: TtSeq(ms), // "elts" is an abbr. for "elements" + // The "dot" is before the first token of the matcher + idx: 0, + // We start matching at the span `open` in the source code + sp_open: open, + + // Initialize `matches` to a bunch of empty `Vec`s -- one for each metavar in `top_elts`. + // `match_lo` for `top_elts` is 0 and `match_hi` is `matches.len()`. `match_cur` is 0 since + // we haven't actually matched anything yet. + matches, + match_lo: 0, + match_cur: 0, + match_hi: match_idx_hi, + + // Haven't descended into any delimiters, so empty stack + stack: smallvec![], + + // Haven't descended into any sequences, so both of these are `None`. + seq_op: None, + sep: None, + up: None, + } +} + +/// `NamedMatch` is a pattern-match result for a single `token::MATCH_NONTERMINAL`: +/// so it is associated with a single ident in a parse, and all +/// `MatchedNonterminal`s in the `NamedMatch` have the same non-terminal type +/// (expr, item, etc). Each leaf in a single `NamedMatch` corresponds to a +/// single `token::MATCH_NONTERMINAL` in the `TokenTree` that produced it. +/// +/// The in-memory structure of a particular `NamedMatch` represents the match +/// that occurred when a particular subset of a matcher was applied to a +/// particular token tree. +/// +/// The width of each `MatchedSeq` in the `NamedMatch`, and the identity of +/// the `MatchedNonterminal`s, will depend on the token tree it was applied +/// to: each `MatchedSeq` corresponds to a single `TTSeq` in the originating +/// token tree. The depth of the `NamedMatch` structure will therefore depend +/// only on the nesting depth of `ast::TTSeq`s in the originating +/// token tree it was derived from. +#[derive(Debug, Clone)] +crate enum NamedMatch { + MatchedSeq(Lrc<NamedMatchVec>, DelimSpan), + MatchedNonterminal(Lrc<Nonterminal>), +} + +/// Takes a sequence of token trees `ms` representing a matcher which successfully matched input +/// and an iterator of items that matched input and produces a `NamedParseResult`. +fn nameize<I: Iterator<Item = NamedMatch>>( + sess: &ParseSess, + ms: &[TokenTree], + mut res: I, +) -> NamedParseResult { + // Recursively descend into each type of matcher (e.g., sequences, delimited, metavars) and make + // sure that each metavar has _exactly one_ binding. If a metavar does not have exactly one + // binding, then there is an error. If it does, then we insert the binding into the + // `NamedParseResult`. + fn n_rec<I: Iterator<Item = NamedMatch>>( + sess: &ParseSess, + m: &TokenTree, + res: &mut I, + ret_val: &mut FxHashMap<Ident, NamedMatch>, + ) -> Result<(), (syntax_pos::Span, String)> { + match *m { + TokenTree::Sequence(_, ref seq) => for next_m in &seq.tts { + n_rec(sess, next_m, res.by_ref(), ret_val)? + }, + TokenTree::Delimited(_, ref delim) => for next_m in &delim.tts { + n_rec(sess, next_m, res.by_ref(), ret_val)?; + }, + TokenTree::MetaVarDecl(span, _, id) if id.name == kw::Invalid => { + if sess.missing_fragment_specifiers.borrow_mut().remove(&span) { + return Err((span, "missing fragment specifier".to_string())); + } + } + TokenTree::MetaVarDecl(sp, bind_name, _) => { + match ret_val.entry(bind_name) { + Vacant(spot) => { + spot.insert(res.next().unwrap()); + } + Occupied(..) => { + return Err((sp, format!("duplicated bind name: {}", bind_name))) + } + } + } + TokenTree::MetaVar(..) | TokenTree::Token(..) => (), + } + + Ok(()) + } + + let mut ret_val = FxHashMap::default(); + for m in ms { + match n_rec(sess, m, res.by_ref(), &mut ret_val) { + Ok(_) => {} + Err((sp, msg)) => return Error(sp, msg), + } + } + + Success(ret_val) +} + +/// Performs a token equality check, ignoring syntax context (that is, an unhygienic comparison) +fn token_name_eq(t1: &Token, t2: &Token) -> bool { + if let (Some((ident1, is_raw1)), Some((ident2, is_raw2))) = (t1.ident(), t2.ident()) { + ident1.name == ident2.name && is_raw1 == is_raw2 + } else if let (Some(ident1), Some(ident2)) = (t1.lifetime(), t2.lifetime()) { + ident1.name == ident2.name + } else { + t1.kind == t2.kind + } +} + +/// Process the matcher positions of `cur_items` until it is empty. In the process, this will +/// produce more items in `next_items`, `eof_items`, and `bb_items`. +/// +/// For more info about the how this happens, see the module-level doc comments and the inline +/// comments of this function. +/// +/// # Parameters +/// +/// - `sess`: the parsing session into which errors are emitted. +/// - `cur_items`: the set of current items to be processed. This should be empty by the end of a +/// successful execution of this function. +/// - `next_items`: the set of newly generated items. These are used to replenish `cur_items` in +/// the function `parse`. +/// - `eof_items`: the set of items that would be valid if this was the EOF. +/// - `bb_items`: the set of items that are waiting for the black-box parser. +/// - `token`: the current token of the parser. +/// - `span`: the `Span` in the source code corresponding to the token trees we are trying to match +/// against the matcher positions in `cur_items`. +/// +/// # Returns +/// +/// A `ParseResult`. Note that matches are kept track of through the items generated. +fn inner_parse_loop<'root, 'tt>( + sess: &ParseSess, + cur_items: &mut SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>, + next_items: &mut Vec<MatcherPosHandle<'root, 'tt>>, + eof_items: &mut SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>, + bb_items: &mut SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>, + token: &Token, +) -> ParseResult<()> { + // Pop items from `cur_items` until it is empty. + while let Some(mut item) = cur_items.pop() { + // When unzipped trees end, remove them. This corresponds to backtracking out of a + // delimited submatcher into which we already descended. In backtracking out again, we need + // to advance the "dot" past the delimiters in the outer matcher. + while item.idx >= item.top_elts.len() { + match item.stack.pop() { + Some(MatcherTtFrame { elts, idx }) => { + item.top_elts = elts; + item.idx = idx + 1; + } + None => break, + } + } + + // Get the current position of the "dot" (`idx`) in `item` and the number of token trees in + // the matcher (`len`). + let idx = item.idx; + let len = item.top_elts.len(); + + // If `idx >= len`, then we are at or past the end of the matcher of `item`. + if idx >= len { + // We are repeating iff there is a parent. If the matcher is inside of a repetition, + // then we could be at the end of a sequence or at the beginning of the next + // repetition. + if item.up.is_some() { + // At this point, regardless of whether there is a separator, we should add all + // matches from the complete repetition of the sequence to the shared, top-level + // `matches` list (actually, `up.matches`, which could itself not be the top-level, + // but anyway...). Moreover, we add another item to `cur_items` in which the "dot" + // is at the end of the `up` matcher. This ensures that the "dot" in the `up` + // matcher is also advanced sufficiently. + // + // NOTE: removing the condition `idx == len` allows trailing separators. + if idx == len { + // Get the `up` matcher + let mut new_pos = item.up.clone().unwrap(); + + // Add matches from this repetition to the `matches` of `up` + for idx in item.match_lo..item.match_hi { + let sub = item.matches[idx].clone(); + let span = DelimSpan::from_pair(item.sp_open, token.span); + new_pos.push_match(idx, MatchedSeq(sub, span)); + } + + // Move the "dot" past the repetition in `up` + new_pos.match_cur = item.match_hi; + new_pos.idx += 1; + cur_items.push(new_pos); + } + + // Check if we need a separator. + if idx == len && item.sep.is_some() { + // We have a separator, and it is the current token. We can advance past the + // separator token. + if item.sep + .as_ref() + .map(|sep| token_name_eq(token, sep)) + .unwrap_or(false) + { + item.idx += 1; + next_items.push(item); + } + } + // We don't need a separator. Move the "dot" back to the beginning of the matcher + // and try to match again UNLESS we are only allowed to have _one_ repetition. + else if item.seq_op != Some(mbe::KleeneOp::ZeroOrOne) { + item.match_cur = item.match_lo; + item.idx = 0; + cur_items.push(item); + } + } + // If we are not in a repetition, then being at the end of a matcher means that we have + // reached the potential end of the input. + else { + eof_items.push(item); + } + } + // We are in the middle of a matcher. + else { + // Look at what token in the matcher we are trying to match the current token (`token`) + // against. Depending on that, we may generate new items. + match item.top_elts.get_tt(idx) { + // Need to descend into a sequence + TokenTree::Sequence(sp, seq) => { + // Examine the case where there are 0 matches of this sequence. We are + // implicitly disallowing OneOrMore from having 0 matches here. Thus, that will + // result in a "no rules expected token" error by virtue of this matcher not + // working. + if seq.kleene.op == mbe::KleeneOp::ZeroOrMore + || seq.kleene.op == mbe::KleeneOp::ZeroOrOne + { + let mut new_item = item.clone(); + new_item.match_cur += seq.num_captures; + new_item.idx += 1; + for idx in item.match_cur..item.match_cur + seq.num_captures { + new_item.push_match(idx, MatchedSeq(Lrc::new(smallvec![]), sp)); + } + cur_items.push(new_item); + } + + let matches = create_matches(item.matches.len()); + cur_items.push(MatcherPosHandle::Box(Box::new(MatcherPos { + stack: smallvec![], + sep: seq.separator.clone(), + seq_op: Some(seq.kleene.op), + idx: 0, + matches, + match_lo: item.match_cur, + match_cur: item.match_cur, + match_hi: item.match_cur + seq.num_captures, + up: Some(item), + sp_open: sp.open, + top_elts: Tt(TokenTree::Sequence(sp, seq)), + }))); + } + + // We need to match a metavar (but the identifier is invalid)... this is an error + TokenTree::MetaVarDecl(span, _, id) if id.name == kw::Invalid => { + if sess.missing_fragment_specifiers.borrow_mut().remove(&span) { + return Error(span, "missing fragment specifier".to_string()); + } + } + + // We need to match a metavar with a valid ident... call out to the black-box + // parser by adding an item to `bb_items`. + TokenTree::MetaVarDecl(_, _, id) => { + // Built-in nonterminals never start with these tokens, + // so we can eliminate them from consideration. + if may_begin_with(token, id.name) { + bb_items.push(item); + } + } + + // We need to descend into a delimited submatcher or a doc comment. To do this, we + // push the current matcher onto a stack and push a new item containing the + // submatcher onto `cur_items`. + // + // At the beginning of the loop, if we reach the end of the delimited submatcher, + // we pop the stack to backtrack out of the descent. + seq @ TokenTree::Delimited(..) | + seq @ TokenTree::Token(Token { kind: DocComment(..), .. }) => { + let lower_elts = mem::replace(&mut item.top_elts, Tt(seq)); + let idx = item.idx; + item.stack.push(MatcherTtFrame { + elts: lower_elts, + idx, + }); + item.idx = 0; + cur_items.push(item); + } + + // We just matched a normal token. We can just advance the parser. + TokenTree::Token(t) if token_name_eq(&t, token) => { + item.idx += 1; + next_items.push(item); + } + + // There was another token that was not `token`... This means we can't add any + // rules. NOTE that this is not necessarily an error unless _all_ items in + // `cur_items` end up doing this. There may still be some other matchers that do + // end up working out. + TokenTree::Token(..) | TokenTree::MetaVar(..) => {} + } + } + } + + // Yay a successful parse (so far)! + Success(()) +} + +/// Use the given sequence of token trees (`ms`) as a matcher. Match the given token stream `tts` +/// against it and return the match. +/// +/// # Parameters +/// +/// - `sess`: The session into which errors are emitted +/// - `tts`: The tokenstream we are matching against the pattern `ms` +/// - `ms`: A sequence of token trees representing a pattern against which we are matching +/// - `directory`: Information about the file locations (needed for the black-box parser) +/// - `recurse_into_modules`: Whether or not to recurse into modules (needed for the black-box +/// parser) +pub(super) fn parse( + sess: &ParseSess, + tts: TokenStream, + ms: &[TokenTree], + directory: Option<Directory<'_>>, + recurse_into_modules: bool, +) -> NamedParseResult { + // Create a parser that can be used for the "black box" parts. + let mut parser = Parser::new( + sess, + tts, + directory, + recurse_into_modules, + true, + syntax::MACRO_ARGUMENTS, + ); + + // A queue of possible matcher positions. We initialize it with the matcher position in which + // the "dot" is before the first token of the first token tree in `ms`. `inner_parse_loop` then + // processes all of these possible matcher positions and produces possible next positions into + // `next_items`. After some post-processing, the contents of `next_items` replenish `cur_items` + // and we start over again. + // + // This MatcherPos instance is allocated on the stack. All others -- and + // there are frequently *no* others! -- are allocated on the heap. + let mut initial = initial_matcher_pos(ms, parser.token.span); + let mut cur_items = smallvec![MatcherPosHandle::Ref(&mut initial)]; + let mut next_items = Vec::new(); + + loop { + // Matcher positions black-box parsed by parser.rs (`parser`) + let mut bb_items = SmallVec::new(); + + // Matcher positions that would be valid if the macro invocation was over now + let mut eof_items = SmallVec::new(); + assert!(next_items.is_empty()); + + // Process `cur_items` until either we have finished the input or we need to get some + // parsing from the black-box parser done. The result is that `next_items` will contain a + // bunch of possible next matcher positions in `next_items`. + match inner_parse_loop( + sess, + &mut cur_items, + &mut next_items, + &mut eof_items, + &mut bb_items, + &parser.token, + ) { + Success(_) => {} + Failure(token, msg) => return Failure(token, msg), + Error(sp, msg) => return Error(sp, msg), + } + + // inner parse loop handled all cur_items, so it's empty + assert!(cur_items.is_empty()); + + // We need to do some post processing after the `inner_parser_loop`. + // + // Error messages here could be improved with links to original rules. + + // If we reached the EOF, check that there is EXACTLY ONE possible matcher. Otherwise, + // either the parse is ambiguous (which should never happen) or there is a syntax error. + if parser.token == token::Eof { + if eof_items.len() == 1 { + let matches = eof_items[0] + .matches + .iter_mut() + .map(|dv| Lrc::make_mut(dv).pop().unwrap()); + return nameize(sess, ms, matches); + } else if eof_items.len() > 1 { + return Error( + parser.token.span, + "ambiguity: multiple successful parses".to_string(), + ); + } else { + return Failure( + Token::new(token::Eof, if parser.token.span.is_dummy() { + parser.token.span + } else { + sess.source_map().next_point(parser.token.span) + }), + "missing tokens in macro arguments", + ); + } + } + // Performance hack: eof_items may share matchers via Rc with other things that we want + // to modify. Dropping eof_items now may drop these refcounts to 1, preventing an + // unnecessary implicit clone later in Rc::make_mut. + drop(eof_items); + + // Another possibility is that we need to call out to parse some rust nonterminal + // (black-box) parser. However, if there is not EXACTLY ONE of these, something is wrong. + if (!bb_items.is_empty() && !next_items.is_empty()) || bb_items.len() > 1 { + let nts = bb_items + .iter() + .map(|item| match item.top_elts.get_tt(item.idx) { + TokenTree::MetaVarDecl(_, bind, name) => format!("{} ('{}')", name, bind), + _ => panic!(), + }) + .collect::<Vec<String>>() + .join(" or "); + + return Error( + parser.token.span, + format!( + "local ambiguity: multiple parsing options: {}", + match next_items.len() { + 0 => format!("built-in NTs {}.", nts), + 1 => format!("built-in NTs {} or 1 other option.", nts), + n => format!("built-in NTs {} or {} other options.", nts, n), + } + ), + ); + } + // If there are no possible next positions AND we aren't waiting for the black-box parser, + // then there is a syntax error. + else if bb_items.is_empty() && next_items.is_empty() { + return Failure( + parser.token.take(), + "no rules expected this token in macro call", + ); + } + // Dump all possible `next_items` into `cur_items` for the next iteration. + else if !next_items.is_empty() { + // Now process the next token + cur_items.extend(next_items.drain(..)); + parser.bump(); + } + // Finally, we have the case where we need to call the black-box parser to get some + // nonterminal. + else { + assert_eq!(bb_items.len(), 1); + + let mut item = bb_items.pop().unwrap(); + if let TokenTree::MetaVarDecl(span, _, ident) = item.top_elts.get_tt(item.idx) { + let match_cur = item.match_cur; + item.push_match( + match_cur, + MatchedNonterminal(Lrc::new(parse_nt(&mut parser, span, ident.name))), + ); + item.idx += 1; + item.match_cur += 1; + } else { + unreachable!() + } + cur_items.push(item); + } + + assert!(!cur_items.is_empty()); + } +} + +/// The token is an identifier, but not `_`. +/// We prohibit passing `_` to macros expecting `ident` for now. +fn get_macro_name(token: &Token) -> Option<(Name, bool)> { + match token.kind { + token::Ident(name, is_raw) if name != kw::Underscore => Some((name, is_raw)), + _ => None, + } +} + +/// Checks whether a non-terminal may begin with a particular token. +/// +/// Returning `false` is a *stability guarantee* that such a matcher will *never* begin with that +/// token. Be conservative (return true) if not sure. +fn may_begin_with(token: &Token, name: Name) -> bool { + /// Checks whether the non-terminal may contain a single (non-keyword) identifier. + fn may_be_ident(nt: &token::Nonterminal) -> bool { + match *nt { + token::NtItem(_) | token::NtBlock(_) | token::NtVis(_) => false, + _ => true, + } + } + + match name { + sym::expr => token.can_begin_expr() + // This exception is here for backwards compatibility. + && !token.is_keyword(kw::Let), + sym::ty => token.can_begin_type(), + sym::ident => get_macro_name(token).is_some(), + sym::literal => token.can_begin_literal_or_bool(), + sym::vis => match token.kind { + // The follow-set of :vis + "priv" keyword + interpolated + token::Comma | token::Ident(..) | token::Interpolated(_) => true, + _ => token.can_begin_type(), + }, + sym::block => match token.kind { + token::OpenDelim(token::Brace) => true, + token::Interpolated(ref nt) => match **nt { + token::NtItem(_) + | token::NtPat(_) + | token::NtTy(_) + | token::NtIdent(..) + | token::NtMeta(_) + | token::NtPath(_) + | token::NtVis(_) => false, // none of these may start with '{'. + _ => true, + }, + _ => false, + }, + sym::path | sym::meta => match token.kind { + token::ModSep | token::Ident(..) => true, + token::Interpolated(ref nt) => match **nt { + token::NtPath(_) | token::NtMeta(_) => true, + _ => may_be_ident(&nt), + }, + _ => false, + }, + sym::pat => match token.kind { + token::Ident(..) | // box, ref, mut, and other identifiers (can stricten) + token::OpenDelim(token::Paren) | // tuple pattern + token::OpenDelim(token::Bracket) | // slice pattern + token::BinOp(token::And) | // reference + token::BinOp(token::Minus) | // negative literal + token::AndAnd | // double reference + token::Literal(..) | // literal + token::DotDot | // range pattern (future compat) + token::DotDotDot | // range pattern (future compat) + token::ModSep | // path + token::Lt | // path (UFCS constant) + token::BinOp(token::Shl) => true, // path (double UFCS) + token::Interpolated(ref nt) => may_be_ident(nt), + _ => false, + }, + sym::lifetime => match token.kind { + token::Lifetime(_) => true, + token::Interpolated(ref nt) => match **nt { + token::NtLifetime(_) | token::NtTT(_) => true, + _ => false, + }, + _ => false, + }, + _ => match token.kind { + token::CloseDelim(_) => false, + _ => true, + }, + } +} + +/// A call to the "black-box" parser to parse some Rust non-terminal. +/// +/// # Parameters +/// +/// - `p`: the "black-box" parser to use +/// - `sp`: the `Span` we want to parse +/// - `name`: the name of the metavar _matcher_ we want to match (e.g., `tt`, `ident`, `block`, +/// etc...) +/// +/// # Returns +/// +/// The parsed non-terminal. +fn parse_nt(p: &mut Parser<'_>, sp: Span, name: Symbol) -> Nonterminal { + if name == sym::tt { + return token::NtTT(p.parse_token_tree()); + } + // check at the beginning and the parser checks after each bump + p.process_potential_macro_variable(); + match parse_nt_inner(p, sp, name) { + Ok(nt) => nt, + Err(mut err) => { + err.emit(); + FatalError.raise(); + } + } +} + +fn parse_nt_inner<'a>(p: &mut Parser<'a>, sp: Span, name: Symbol) -> PResult<'a, Nonterminal> { + Ok(match name { + sym::item => match p.parse_item()? { + Some(i) => token::NtItem(i), + None => return Err(p.fatal("expected an item keyword")), + }, + sym::block => token::NtBlock(p.parse_block()?), + sym::stmt => match p.parse_stmt()? { + Some(s) => token::NtStmt(s), + None => return Err(p.fatal("expected a statement")), + }, + sym::pat => token::NtPat(p.parse_pat(None)?), + sym::expr => token::NtExpr(p.parse_expr()?), + sym::literal => token::NtLiteral(p.parse_literal_maybe_minus()?), + sym::ty => token::NtTy(p.parse_ty()?), + // this could be handled like a token, since it is one + sym::ident => if let Some((name, is_raw)) = get_macro_name(&p.token) { + let span = p.token.span; + p.bump(); + token::NtIdent(Ident::new(name, span), is_raw) + } else { + let token_str = pprust::token_to_string(&p.token); + return Err(p.fatal(&format!("expected ident, found {}", &token_str))); + } + sym::path => token::NtPath(p.parse_path(PathStyle::Type)?), + sym::meta => token::NtMeta(p.parse_attr_item()?), + sym::vis => token::NtVis(p.parse_visibility(true)?), + sym::lifetime => if p.check_lifetime() { + token::NtLifetime(p.expect_lifetime().ident) + } else { + let token_str = pprust::token_to_string(&p.token); + return Err(p.fatal(&format!("expected a lifetime, found `{}`", &token_str))); + } + // this is not supposed to happen, since it has been checked + // when compiling the macro. + _ => p.span_bug(sp, "invalid fragment specifier"), + }) +} diff --git a/src/libsyntax_expand/mbe/macro_rules.rs b/src/libsyntax_expand/mbe/macro_rules.rs new file mode 100644 index 00000000000..9a4130b2d8d --- /dev/null +++ b/src/libsyntax_expand/mbe/macro_rules.rs @@ -0,0 +1,1192 @@ +use crate::base::{DummyResult, ExtCtxt, MacResult, TTMacroExpander}; +use crate::base::{SyntaxExtension, SyntaxExtensionKind}; +use crate::expand::{AstFragment, AstFragmentKind, ensure_complete_parse, parse_ast_fragment}; +use crate::mbe; +use crate::mbe::macro_check; +use crate::mbe::macro_parser::parse; +use crate::mbe::macro_parser::{Error, Failure, Success}; +use crate::mbe::macro_parser::{MatchedNonterminal, MatchedSeq, NamedParseResult}; +use crate::mbe::transcribe::transcribe; + +use syntax::ast; +use syntax::attr::{self, TransparencyError}; +use syntax::edition::Edition; +use syntax::feature_gate::Features; +use syntax::parse::parser::Parser; +use syntax::parse::token::TokenKind::*; +use syntax::parse::token::{self, NtTT, Token}; +use syntax::parse::Directory; +use syntax::print::pprust; +use syntax::sess::ParseSess; +use syntax::symbol::{kw, sym, Symbol}; +use syntax::tokenstream::{DelimSpan, TokenStream}; + +use errors::{DiagnosticBuilder, FatalError}; +use log::debug; +use syntax_pos::hygiene::Transparency; +use syntax_pos::Span; + +use rustc_data_structures::fx::FxHashMap; +use std::borrow::Cow; +use std::collections::hash_map::Entry; +use std::slice; + +use errors::Applicability; +use rustc_data_structures::sync::Lrc; + +const VALID_FRAGMENT_NAMES_MSG: &str = "valid fragment specifiers are \ + `ident`, `block`, `stmt`, `expr`, `pat`, `ty`, `lifetime`, \ + `literal`, `path`, `meta`, `tt`, `item` and `vis`"; + +crate struct ParserAnyMacro<'a> { + parser: Parser<'a>, + + /// Span of the expansion site of the macro this parser is for + site_span: Span, + /// The ident of the macro we're parsing + macro_ident: ast::Ident, + arm_span: Span, +} + +crate fn annotate_err_with_kind( + err: &mut DiagnosticBuilder<'_>, + kind: AstFragmentKind, + span: Span, +) { + match kind { + AstFragmentKind::Ty => { + err.span_label(span, "this macro call doesn't expand to a type"); + } + AstFragmentKind::Pat => { + err.span_label(span, "this macro call doesn't expand to a pattern"); + } + _ => {} + }; +} + +impl<'a> ParserAnyMacro<'a> { + crate fn make(mut self: Box<ParserAnyMacro<'a>>, kind: AstFragmentKind) -> AstFragment { + let ParserAnyMacro { site_span, macro_ident, ref mut parser, arm_span } = *self; + let fragment = panictry!(parse_ast_fragment(parser, kind, true).map_err(|mut e| { + if parser.token == token::Eof && e.message().ends_with(", found `<eof>`") { + if !e.span.is_dummy() { + // early end of macro arm (#52866) + e.replace_span_with(parser.sess.source_map().next_point(parser.token.span)); + } + let msg = &e.message[0]; + e.message[0] = ( + format!( + "macro expansion ends with an incomplete expression: {}", + msg.0.replace(", found `<eof>`", ""), + ), + msg.1, + ); + } + if e.span.is_dummy() { + // Get around lack of span in error (#30128) + e.replace_span_with(site_span); + if parser.sess.source_map().span_to_filename(arm_span).is_real() { + e.span_label(arm_span, "in this macro arm"); + } + } else if !parser.sess.source_map().span_to_filename(parser.token.span).is_real() { + e.span_label(site_span, "in this macro invocation"); + } + match kind { + AstFragmentKind::Pat if macro_ident.name == sym::vec => { + let mut suggestion = None; + if let Ok(code) = parser.sess.source_map().span_to_snippet(site_span) { + if let Some(bang) = code.find('!') { + suggestion = Some(code[bang + 1..].to_string()); + } + } + if let Some(suggestion) = suggestion { + e.span_suggestion( + site_span, + "use a slice pattern here instead", + suggestion, + Applicability::MachineApplicable, + ); + } else { + e.span_label( + site_span, + "use a slice pattern here instead", + ); + } + e.help("for more information, see https://doc.rust-lang.org/edition-guide/\ + rust-2018/slice-patterns.html"); + } + _ => annotate_err_with_kind(&mut e, kind, site_span), + }; + e + })); + + // We allow semicolons at the end of expressions -- e.g., the semicolon in + // `macro_rules! m { () => { panic!(); } }` isn't parsed by `.parse_expr()`, + // but `m!()` is allowed in expression positions (cf. issue #34706). + if kind == AstFragmentKind::Expr && parser.token == token::Semi { + parser.bump(); + } + + // Make sure we don't have any tokens left to parse so we don't silently drop anything. + let path = ast::Path::from_ident(macro_ident.with_span_pos(site_span)); + ensure_complete_parse(parser, &path, kind.name(), site_span); + fragment + } +} + +struct MacroRulesMacroExpander { + name: ast::Ident, + span: Span, + transparency: Transparency, + lhses: Vec<mbe::TokenTree>, + rhses: Vec<mbe::TokenTree>, + valid: bool, +} + +impl TTMacroExpander for MacroRulesMacroExpander { + fn expand<'cx>( + &self, + cx: &'cx mut ExtCtxt<'_>, + sp: Span, + input: TokenStream, + ) -> Box<dyn MacResult + 'cx> { + if !self.valid { + return DummyResult::any(sp); + } + generic_extension( + cx, sp, self.span, self.name, self.transparency, input, &self.lhses, &self.rhses + ) + } +} + +fn trace_macros_note(cx: &mut ExtCtxt<'_>, sp: Span, message: String) { + let sp = sp.macro_backtrace().last().map(|trace| trace.call_site).unwrap_or(sp); + cx.expansions.entry(sp).or_default().push(message); +} + +/// Given `lhses` and `rhses`, this is the new macro we create +fn generic_extension<'cx>( + cx: &'cx mut ExtCtxt<'_>, + sp: Span, + def_span: Span, + name: ast::Ident, + transparency: Transparency, + arg: TokenStream, + lhses: &[mbe::TokenTree], + rhses: &[mbe::TokenTree], +) -> Box<dyn MacResult + 'cx> { + if cx.trace_macros() { + let msg = format!("expanding `{}! {{ {} }}`", name, pprust::tts_to_string(arg.clone())); + trace_macros_note(cx, sp, msg); + } + + // Which arm's failure should we report? (the one furthest along) + let mut best_failure: Option<(Token, &str)> = None; + + for (i, lhs) in lhses.iter().enumerate() { + // try each arm's matchers + let lhs_tt = match *lhs { + mbe::TokenTree::Delimited(_, ref delim) => &delim.tts[..], + _ => cx.span_bug(sp, "malformed macro lhs"), + }; + + match parse_tt(cx, lhs_tt, arg.clone()) { + Success(named_matches) => { + let rhs = match rhses[i] { + // ignore delimiters + mbe::TokenTree::Delimited(_, ref delimed) => delimed.tts.clone(), + _ => cx.span_bug(sp, "malformed macro rhs"), + }; + let arm_span = rhses[i].span(); + + let rhs_spans = rhs.iter().map(|t| t.span()).collect::<Vec<_>>(); + // rhs has holes ( `$id` and `$(...)` that need filled) + let mut tts = transcribe(cx, &named_matches, rhs, transparency); + + // Replace all the tokens for the corresponding positions in the macro, to maintain + // proper positions in error reporting, while maintaining the macro_backtrace. + if rhs_spans.len() == tts.len() { + tts = tts.map_enumerated(|i, mut tt| { + let mut sp = rhs_spans[i]; + sp = sp.with_ctxt(tt.span().ctxt()); + tt.set_span(sp); + tt + }); + } + + if cx.trace_macros() { + let msg = format!("to `{}`", pprust::tts_to_string(tts.clone())); + trace_macros_note(cx, sp, msg); + } + + let directory = Directory { + path: Cow::from(cx.current_expansion.module.directory.as_path()), + ownership: cx.current_expansion.directory_ownership, + }; + let mut p = Parser::new(cx.parse_sess(), tts, Some(directory), true, false, None); + p.root_module_name = + cx.current_expansion.module.mod_path.last().map(|id| id.as_str().to_string()); + p.last_type_ascription = cx.current_expansion.prior_type_ascription; + + p.process_potential_macro_variable(); + // Let the context choose how to interpret the result. + // Weird, but useful for X-macros. + return Box::new(ParserAnyMacro { + parser: p, + + // Pass along the original expansion site and the name of the macro + // so we can print a useful error message if the parse of the expanded + // macro leaves unparsed tokens. + site_span: sp, + macro_ident: name, + arm_span, + }); + } + Failure(token, msg) => match best_failure { + Some((ref best_token, _)) if best_token.span.lo() >= token.span.lo() => {} + _ => best_failure = Some((token, msg)), + }, + Error(err_sp, ref msg) => cx.span_fatal(err_sp.substitute_dummy(sp), &msg[..]), + } + } + + let (token, label) = best_failure.expect("ran no matchers"); + let span = token.span.substitute_dummy(sp); + let mut err = cx.struct_span_err(span, &parse_failure_msg(&token)); + err.span_label(span, label); + if !def_span.is_dummy() && cx.source_map().span_to_filename(def_span).is_real() { + err.span_label(cx.source_map().def_span(def_span), "when calling this macro"); + } + + // Check whether there's a missing comma in this macro call, like `println!("{}" a);` + if let Some((arg, comma_span)) = arg.add_comma() { + for lhs in lhses { + // try each arm's matchers + let lhs_tt = match *lhs { + mbe::TokenTree::Delimited(_, ref delim) => &delim.tts[..], + _ => continue, + }; + match parse_tt(cx, lhs_tt, arg.clone()) { + Success(_) => { + if comma_span.is_dummy() { + err.note("you might be missing a comma"); + } else { + err.span_suggestion_short( + comma_span, + "missing comma here", + ", ".to_string(), + Applicability::MachineApplicable, + ); + } + } + _ => {} + } + } + } + err.emit(); + cx.trace_macros_diag(); + DummyResult::any(sp) +} + +// Note that macro-by-example's input is also matched against a token tree: +// $( $lhs:tt => $rhs:tt );+ +// +// Holy self-referential! + +/// Converts a macro item into a syntax extension. +pub fn compile_declarative_macro( + sess: &ParseSess, + features: &Features, + def: &ast::Item, + edition: Edition, +) -> SyntaxExtension { + let diag = &sess.span_diagnostic; + let lhs_nm = ast::Ident::new(sym::lhs, def.span); + let rhs_nm = ast::Ident::new(sym::rhs, def.span); + let tt_spec = ast::Ident::new(sym::tt, def.span); + + // Parse the macro_rules! invocation + let body = match def.kind { + ast::ItemKind::MacroDef(ref body) => body, + _ => unreachable!(), + }; + + // The pattern that macro_rules matches. + // The grammar for macro_rules! is: + // $( $lhs:tt => $rhs:tt );+ + // ...quasiquoting this would be nice. + // These spans won't matter, anyways + let argument_gram = vec![ + mbe::TokenTree::Sequence( + DelimSpan::dummy(), + Lrc::new(mbe::SequenceRepetition { + tts: vec![ + mbe::TokenTree::MetaVarDecl(def.span, lhs_nm, tt_spec), + mbe::TokenTree::token(token::FatArrow, def.span), + mbe::TokenTree::MetaVarDecl(def.span, rhs_nm, tt_spec), + ], + separator: Some(Token::new( + if body.legacy { token::Semi } else { token::Comma }, + def.span, + )), + kleene: mbe::KleeneToken::new(mbe::KleeneOp::OneOrMore, def.span), + num_captures: 2, + }), + ), + // to phase into semicolon-termination instead of semicolon-separation + mbe::TokenTree::Sequence( + DelimSpan::dummy(), + Lrc::new(mbe::SequenceRepetition { + tts: vec![mbe::TokenTree::token( + if body.legacy { token::Semi } else { token::Comma }, + def.span, + )], + separator: None, + kleene: mbe::KleeneToken::new(mbe::KleeneOp::ZeroOrMore, def.span), + num_captures: 0, + }), + ), + ]; + + let argument_map = match parse(sess, body.stream(), &argument_gram, None, true) { + Success(m) => m, + Failure(token, msg) => { + let s = parse_failure_msg(&token); + let sp = token.span.substitute_dummy(def.span); + let mut err = sess.span_diagnostic.struct_span_fatal(sp, &s); + err.span_label(sp, msg); + err.emit(); + FatalError.raise(); + } + Error(sp, s) => { + sess.span_diagnostic.span_fatal(sp.substitute_dummy(def.span), &s).raise(); + } + }; + + let mut valid = true; + + // Extract the arguments: + let lhses = match argument_map[&lhs_nm] { + MatchedSeq(ref s, _) => s + .iter() + .map(|m| { + if let MatchedNonterminal(ref nt) = *m { + if let NtTT(ref tt) = **nt { + let tt = mbe::quoted::parse( + tt.clone().into(), + true, + sess, + ) + .pop() + .unwrap(); + valid &= check_lhs_nt_follows(sess, features, &def.attrs, &tt); + return tt; + } + } + sess.span_diagnostic.span_bug(def.span, "wrong-structured lhs") + }) + .collect::<Vec<mbe::TokenTree>>(), + _ => sess.span_diagnostic.span_bug(def.span, "wrong-structured lhs"), + }; + + let rhses = match argument_map[&rhs_nm] { + MatchedSeq(ref s, _) => s + .iter() + .map(|m| { + if let MatchedNonterminal(ref nt) = *m { + if let NtTT(ref tt) = **nt { + return mbe::quoted::parse( + tt.clone().into(), + false, + sess, + ) + .pop() + .unwrap(); + } + } + sess.span_diagnostic.span_bug(def.span, "wrong-structured lhs") + }) + .collect::<Vec<mbe::TokenTree>>(), + _ => sess.span_diagnostic.span_bug(def.span, "wrong-structured rhs"), + }; + + for rhs in &rhses { + valid &= check_rhs(sess, rhs); + } + + // don't abort iteration early, so that errors for multiple lhses can be reported + for lhs in &lhses { + valid &= check_lhs_no_empty_seq(sess, slice::from_ref(lhs)); + } + + // We use CRATE_NODE_ID instead of `def.id` otherwise we may emit buffered lints for a node id + // that is not lint-checked and trigger the "failed to process buffered lint here" bug. + valid &= macro_check::check_meta_variables(sess, ast::CRATE_NODE_ID, def.span, &lhses, &rhses); + + let (transparency, transparency_error) = attr::find_transparency(&def.attrs, body.legacy); + match transparency_error { + Some(TransparencyError::UnknownTransparency(value, span)) => + diag.span_err(span, &format!("unknown macro transparency: `{}`", value)), + Some(TransparencyError::MultipleTransparencyAttrs(old_span, new_span)) => + diag.span_err(vec![old_span, new_span], "multiple macro transparency attributes"), + None => {} + } + + let expander: Box<_> = Box::new(MacroRulesMacroExpander { + name: def.ident, span: def.span, transparency, lhses, rhses, valid + }); + + SyntaxExtension::new( + sess, + SyntaxExtensionKind::LegacyBang(expander), + def.span, + Vec::new(), + edition, + def.ident.name, + &def.attrs, + ) +} + +fn check_lhs_nt_follows( + sess: &ParseSess, + features: &Features, + attrs: &[ast::Attribute], + lhs: &mbe::TokenTree, +) -> bool { + // lhs is going to be like TokenTree::Delimited(...), where the + // entire lhs is those tts. Or, it can be a "bare sequence", not wrapped in parens. + if let mbe::TokenTree::Delimited(_, ref tts) = *lhs { + check_matcher(sess, features, attrs, &tts.tts) + } else { + let msg = "invalid macro matcher; matchers must be contained in balanced delimiters"; + sess.span_diagnostic.span_err(lhs.span(), msg); + false + } + // we don't abort on errors on rejection, the driver will do that for us + // after parsing/expansion. we can report every error in every macro this way. +} + +/// Checks that the lhs contains no repetition which could match an empty token +/// tree, because then the matcher would hang indefinitely. +fn check_lhs_no_empty_seq(sess: &ParseSess, tts: &[mbe::TokenTree]) -> bool { + use mbe::TokenTree; + for tt in tts { + match *tt { + TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => (), + TokenTree::Delimited(_, ref del) => { + if !check_lhs_no_empty_seq(sess, &del.tts) { + return false; + } + } + TokenTree::Sequence(span, ref seq) => { + if seq.separator.is_none() + && seq.tts.iter().all(|seq_tt| match *seq_tt { + TokenTree::MetaVarDecl(_, _, id) => id.name == sym::vis, + TokenTree::Sequence(_, ref sub_seq) => { + sub_seq.kleene.op == mbe::KleeneOp::ZeroOrMore + || sub_seq.kleene.op == mbe::KleeneOp::ZeroOrOne + } + _ => false, + }) + { + let sp = span.entire(); + sess.span_diagnostic.span_err(sp, "repetition matches empty token tree"); + return false; + } + if !check_lhs_no_empty_seq(sess, &seq.tts) { + return false; + } + } + } + } + + true +} + +fn check_rhs(sess: &ParseSess, rhs: &mbe::TokenTree) -> bool { + match *rhs { + mbe::TokenTree::Delimited(..) => return true, + _ => sess.span_diagnostic.span_err(rhs.span(), "macro rhs must be delimited"), + } + false +} + +fn check_matcher( + sess: &ParseSess, + features: &Features, + attrs: &[ast::Attribute], + matcher: &[mbe::TokenTree], +) -> bool { + let first_sets = FirstSets::new(matcher); + let empty_suffix = TokenSet::empty(); + let err = sess.span_diagnostic.err_count(); + check_matcher_core(sess, features, attrs, &first_sets, matcher, &empty_suffix); + err == sess.span_diagnostic.err_count() +} + +// `The FirstSets` for a matcher is a mapping from subsequences in the +// matcher to the FIRST set for that subsequence. +// +// This mapping is partially precomputed via a backwards scan over the +// token trees of the matcher, which provides a mapping from each +// repetition sequence to its *first* set. +// +// (Hypothetically, sequences should be uniquely identifiable via their +// spans, though perhaps that is false, e.g., for macro-generated macros +// that do not try to inject artificial span information. My plan is +// to try to catch such cases ahead of time and not include them in +// the precomputed mapping.) +struct FirstSets { + // this maps each TokenTree::Sequence `$(tt ...) SEP OP` that is uniquely identified by its + // span in the original matcher to the First set for the inner sequence `tt ...`. + // + // If two sequences have the same span in a matcher, then map that + // span to None (invalidating the mapping here and forcing the code to + // use a slow path). + first: FxHashMap<Span, Option<TokenSet>>, +} + +impl FirstSets { + fn new(tts: &[mbe::TokenTree]) -> FirstSets { + use mbe::TokenTree; + + let mut sets = FirstSets { first: FxHashMap::default() }; + build_recur(&mut sets, tts); + return sets; + + // walks backward over `tts`, returning the FIRST for `tts` + // and updating `sets` at the same time for all sequence + // substructure we find within `tts`. + fn build_recur(sets: &mut FirstSets, tts: &[TokenTree]) -> TokenSet { + let mut first = TokenSet::empty(); + for tt in tts.iter().rev() { + match *tt { + TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => { + first.replace_with(tt.clone()); + } + TokenTree::Delimited(span, ref delimited) => { + build_recur(sets, &delimited.tts[..]); + first.replace_with(delimited.open_tt(span.open)); + } + TokenTree::Sequence(sp, ref seq_rep) => { + let subfirst = build_recur(sets, &seq_rep.tts[..]); + + match sets.first.entry(sp.entire()) { + Entry::Vacant(vac) => { + vac.insert(Some(subfirst.clone())); + } + Entry::Occupied(mut occ) => { + // if there is already an entry, then a span must have collided. + // This should not happen with typical macro_rules macros, + // but syntax extensions need not maintain distinct spans, + // so distinct syntax trees can be assigned the same span. + // In such a case, the map cannot be trusted; so mark this + // entry as unusable. + occ.insert(None); + } + } + + // If the sequence contents can be empty, then the first + // token could be the separator token itself. + + if let (Some(sep), true) = (&seq_rep.separator, subfirst.maybe_empty) { + first.add_one_maybe(TokenTree::Token(sep.clone())); + } + + // Reverse scan: Sequence comes before `first`. + if subfirst.maybe_empty + || seq_rep.kleene.op == mbe::KleeneOp::ZeroOrMore + || seq_rep.kleene.op == mbe::KleeneOp::ZeroOrOne + { + // If sequence is potentially empty, then + // union them (preserving first emptiness). + first.add_all(&TokenSet { maybe_empty: true, ..subfirst }); + } else { + // Otherwise, sequence guaranteed + // non-empty; replace first. + first = subfirst; + } + } + } + } + + first + } + } + + // walks forward over `tts` until all potential FIRST tokens are + // identified. + fn first(&self, tts: &[mbe::TokenTree]) -> TokenSet { + use mbe::TokenTree; + + let mut first = TokenSet::empty(); + for tt in tts.iter() { + assert!(first.maybe_empty); + match *tt { + TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => { + first.add_one(tt.clone()); + return first; + } + TokenTree::Delimited(span, ref delimited) => { + first.add_one(delimited.open_tt(span.open)); + return first; + } + TokenTree::Sequence(sp, ref seq_rep) => { + let subfirst_owned; + let subfirst = match self.first.get(&sp.entire()) { + Some(&Some(ref subfirst)) => subfirst, + Some(&None) => { + subfirst_owned = self.first(&seq_rep.tts[..]); + &subfirst_owned + } + None => { + panic!("We missed a sequence during FirstSets construction"); + } + }; + + // If the sequence contents can be empty, then the first + // token could be the separator token itself. + if let (Some(sep), true) = (&seq_rep.separator, subfirst.maybe_empty) { + first.add_one_maybe(TokenTree::Token(sep.clone())); + } + + assert!(first.maybe_empty); + first.add_all(subfirst); + if subfirst.maybe_empty + || seq_rep.kleene.op == mbe::KleeneOp::ZeroOrMore + || seq_rep.kleene.op == mbe::KleeneOp::ZeroOrOne + { + // Continue scanning for more first + // tokens, but also make sure we + // restore empty-tracking state. + first.maybe_empty = true; + continue; + } else { + return first; + } + } + } + } + + // we only exit the loop if `tts` was empty or if every + // element of `tts` matches the empty sequence. + assert!(first.maybe_empty); + first + } +} + +// A set of `mbe::TokenTree`s, which may include `TokenTree::Match`s +// (for macro-by-example syntactic variables). It also carries the +// `maybe_empty` flag; that is true if and only if the matcher can +// match an empty token sequence. +// +// The First set is computed on submatchers like `$($a:expr b),* $(c)* d`, +// which has corresponding FIRST = {$a:expr, c, d}. +// Likewise, `$($a:expr b),* $(c)+ d` has FIRST = {$a:expr, c}. +// +// (Notably, we must allow for *-op to occur zero times.) +#[derive(Clone, Debug)] +struct TokenSet { + tokens: Vec<mbe::TokenTree>, + maybe_empty: bool, +} + +impl TokenSet { + // Returns a set for the empty sequence. + fn empty() -> Self { + TokenSet { tokens: Vec::new(), maybe_empty: true } + } + + // Returns the set `{ tok }` for the single-token (and thus + // non-empty) sequence [tok]. + fn singleton(tok: mbe::TokenTree) -> Self { + TokenSet { tokens: vec![tok], maybe_empty: false } + } + + // Changes self to be the set `{ tok }`. + // Since `tok` is always present, marks self as non-empty. + fn replace_with(&mut self, tok: mbe::TokenTree) { + self.tokens.clear(); + self.tokens.push(tok); + self.maybe_empty = false; + } + + // Changes self to be the empty set `{}`; meant for use when + // the particular token does not matter, but we want to + // record that it occurs. + fn replace_with_irrelevant(&mut self) { + self.tokens.clear(); + self.maybe_empty = false; + } + + // Adds `tok` to the set for `self`, marking sequence as non-empy. + fn add_one(&mut self, tok: mbe::TokenTree) { + if !self.tokens.contains(&tok) { + self.tokens.push(tok); + } + self.maybe_empty = false; + } + + // Adds `tok` to the set for `self`. (Leaves `maybe_empty` flag alone.) + fn add_one_maybe(&mut self, tok: mbe::TokenTree) { + if !self.tokens.contains(&tok) { + self.tokens.push(tok); + } + } + + // Adds all elements of `other` to this. + // + // (Since this is a set, we filter out duplicates.) + // + // If `other` is potentially empty, then preserves the previous + // setting of the empty flag of `self`. If `other` is guaranteed + // non-empty, then `self` is marked non-empty. + fn add_all(&mut self, other: &Self) { + for tok in &other.tokens { + if !self.tokens.contains(tok) { + self.tokens.push(tok.clone()); + } + } + if !other.maybe_empty { + self.maybe_empty = false; + } + } +} + +// Checks that `matcher` is internally consistent and that it +// can legally be followed by a token `N`, for all `N` in `follow`. +// (If `follow` is empty, then it imposes no constraint on +// the `matcher`.) +// +// Returns the set of NT tokens that could possibly come last in +// `matcher`. (If `matcher` matches the empty sequence, then +// `maybe_empty` will be set to true.) +// +// Requires that `first_sets` is pre-computed for `matcher`; +// see `FirstSets::new`. +fn check_matcher_core( + sess: &ParseSess, + features: &Features, + attrs: &[ast::Attribute], + first_sets: &FirstSets, + matcher: &[mbe::TokenTree], + follow: &TokenSet, +) -> TokenSet { + use mbe::TokenTree; + + let mut last = TokenSet::empty(); + + // 2. For each token and suffix [T, SUFFIX] in M: + // ensure that T can be followed by SUFFIX, and if SUFFIX may be empty, + // then ensure T can also be followed by any element of FOLLOW. + 'each_token: for i in 0..matcher.len() { + let token = &matcher[i]; + let suffix = &matcher[i + 1..]; + + let build_suffix_first = || { + let mut s = first_sets.first(suffix); + if s.maybe_empty { + s.add_all(follow); + } + s + }; + + // (we build `suffix_first` on demand below; you can tell + // which cases are supposed to fall through by looking for the + // initialization of this variable.) + let suffix_first; + + // First, update `last` so that it corresponds to the set + // of NT tokens that might end the sequence `... token`. + match *token { + TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => { + let can_be_followed_by_any; + if let Err(bad_frag) = has_legal_fragment_specifier(sess, features, attrs, token) { + let msg = format!("invalid fragment specifier `{}`", bad_frag); + sess.span_diagnostic + .struct_span_err(token.span(), &msg) + .help(VALID_FRAGMENT_NAMES_MSG) + .emit(); + // (This eliminates false positives and duplicates + // from error messages.) + can_be_followed_by_any = true; + } else { + can_be_followed_by_any = token_can_be_followed_by_any(token); + } + + if can_be_followed_by_any { + // don't need to track tokens that work with any, + last.replace_with_irrelevant(); + // ... and don't need to check tokens that can be + // followed by anything against SUFFIX. + continue 'each_token; + } else { + last.replace_with(token.clone()); + suffix_first = build_suffix_first(); + } + } + TokenTree::Delimited(span, ref d) => { + let my_suffix = TokenSet::singleton(d.close_tt(span.close)); + check_matcher_core(sess, features, attrs, first_sets, &d.tts, &my_suffix); + // don't track non NT tokens + last.replace_with_irrelevant(); + + // also, we don't need to check delimited sequences + // against SUFFIX + continue 'each_token; + } + TokenTree::Sequence(_, ref seq_rep) => { + suffix_first = build_suffix_first(); + // The trick here: when we check the interior, we want + // to include the separator (if any) as a potential + // (but not guaranteed) element of FOLLOW. So in that + // case, we make a temp copy of suffix and stuff + // delimiter in there. + // + // FIXME: Should I first scan suffix_first to see if + // delimiter is already in it before I go through the + // work of cloning it? But then again, this way I may + // get a "tighter" span? + let mut new; + let my_suffix = if let Some(sep) = &seq_rep.separator { + new = suffix_first.clone(); + new.add_one_maybe(TokenTree::Token(sep.clone())); + &new + } else { + &suffix_first + }; + + // At this point, `suffix_first` is built, and + // `my_suffix` is some TokenSet that we can use + // for checking the interior of `seq_rep`. + let next = + check_matcher_core(sess, features, attrs, first_sets, &seq_rep.tts, my_suffix); + if next.maybe_empty { + last.add_all(&next); + } else { + last = next; + } + + // the recursive call to check_matcher_core already ran the 'each_last + // check below, so we can just keep going forward here. + continue 'each_token; + } + } + + // (`suffix_first` guaranteed initialized once reaching here.) + + // Now `last` holds the complete set of NT tokens that could + // end the sequence before SUFFIX. Check that every one works with `suffix`. + 'each_last: for token in &last.tokens { + if let TokenTree::MetaVarDecl(_, name, frag_spec) = *token { + for next_token in &suffix_first.tokens { + match is_in_follow(next_token, frag_spec.name) { + IsInFollow::Invalid(msg, help) => { + sess.span_diagnostic + .struct_span_err(next_token.span(), &msg) + .help(help) + .emit(); + // don't bother reporting every source of + // conflict for a particular element of `last`. + continue 'each_last; + } + IsInFollow::Yes => {} + IsInFollow::No(possible) => { + let may_be = if last.tokens.len() == 1 && suffix_first.tokens.len() == 1 + { + "is" + } else { + "may be" + }; + + let sp = next_token.span(); + let mut err = sess.span_diagnostic.struct_span_err( + sp, + &format!( + "`${name}:{frag}` {may_be} followed by `{next}`, which \ + is not allowed for `{frag}` fragments", + name = name, + frag = frag_spec, + next = quoted_tt_to_string(next_token), + may_be = may_be + ), + ); + err.span_label( + sp, + format!("not allowed after `{}` fragments", frag_spec), + ); + let msg = "allowed there are: "; + match possible { + &[] => {} + &[t] => { + err.note(&format!( + "only {} is allowed after `{}` fragments", + t, frag_spec, + )); + } + ts => { + err.note(&format!( + "{}{} or {}", + msg, + ts[..ts.len() - 1] + .iter() + .map(|s| *s) + .collect::<Vec<_>>() + .join(", "), + ts[ts.len() - 1], + )); + } + } + err.emit(); + } + } + } + } + } + } + last +} + +fn token_can_be_followed_by_any(tok: &mbe::TokenTree) -> bool { + if let mbe::TokenTree::MetaVarDecl(_, _, frag_spec) = *tok { + frag_can_be_followed_by_any(frag_spec.name) + } else { + // (Non NT's can always be followed by anthing in matchers.) + true + } +} + +/// Returns `true` if a fragment of type `frag` can be followed by any sort of +/// token. We use this (among other things) as a useful approximation +/// for when `frag` can be followed by a repetition like `$(...)*` or +/// `$(...)+`. In general, these can be a bit tricky to reason about, +/// so we adopt a conservative position that says that any fragment +/// specifier which consumes at most one token tree can be followed by +/// a fragment specifier (indeed, these fragments can be followed by +/// ANYTHING without fear of future compatibility hazards). +fn frag_can_be_followed_by_any(frag: Symbol) -> bool { + match frag { + sym::item | // always terminated by `}` or `;` + sym::block | // exactly one token tree + sym::ident | // exactly one token tree + sym::literal | // exactly one token tree + sym::meta | // exactly one token tree + sym::lifetime | // exactly one token tree + sym::tt => // exactly one token tree + true, + + _ => + false, + } +} + +enum IsInFollow { + Yes, + No(&'static [&'static str]), + Invalid(String, &'static str), +} + +/// Returns `true` if `frag` can legally be followed by the token `tok`. For +/// fragments that can consume an unbounded number of tokens, `tok` +/// must be within a well-defined follow set. This is intended to +/// guarantee future compatibility: for example, without this rule, if +/// we expanded `expr` to include a new binary operator, we might +/// break macros that were relying on that binary operator as a +/// separator. +// when changing this do not forget to update doc/book/macros.md! +fn is_in_follow(tok: &mbe::TokenTree, frag: Symbol) -> IsInFollow { + use mbe::TokenTree; + + if let TokenTree::Token(Token { kind: token::CloseDelim(_), .. }) = *tok { + // closing a token tree can never be matched by any fragment; + // iow, we always require that `(` and `)` match, etc. + IsInFollow::Yes + } else { + match frag { + sym::item => { + // since items *must* be followed by either a `;` or a `}`, we can + // accept anything after them + IsInFollow::Yes + } + sym::block => { + // anything can follow block, the braces provide an easy boundary to + // maintain + IsInFollow::Yes + } + sym::stmt | sym::expr => { + const TOKENS: &[&str] = &["`=>`", "`,`", "`;`"]; + match tok { + TokenTree::Token(token) => match token.kind { + FatArrow | Comma | Semi => IsInFollow::Yes, + _ => IsInFollow::No(TOKENS), + }, + _ => IsInFollow::No(TOKENS), + } + } + sym::pat => { + const TOKENS: &[&str] = &["`=>`", "`,`", "`=`", "`|`", "`if`", "`in`"]; + match tok { + TokenTree::Token(token) => match token.kind { + FatArrow | Comma | Eq | BinOp(token::Or) => IsInFollow::Yes, + Ident(name, false) if name == kw::If || name == kw::In => IsInFollow::Yes, + _ => IsInFollow::No(TOKENS), + }, + _ => IsInFollow::No(TOKENS), + } + } + sym::path | sym::ty => { + const TOKENS: &[&str] = &[ + "`{`", "`[`", "`=>`", "`,`", "`>`", "`=`", "`:`", "`;`", "`|`", "`as`", + "`where`", + ]; + match tok { + TokenTree::Token(token) => match token.kind { + OpenDelim(token::DelimToken::Brace) + | OpenDelim(token::DelimToken::Bracket) + | Comma + | FatArrow + | Colon + | Eq + | Gt + | BinOp(token::Shr) + | Semi + | BinOp(token::Or) => IsInFollow::Yes, + Ident(name, false) if name == kw::As || name == kw::Where => { + IsInFollow::Yes + } + _ => IsInFollow::No(TOKENS), + }, + TokenTree::MetaVarDecl(_, _, frag) if frag.name == sym::block => { + IsInFollow::Yes + } + _ => IsInFollow::No(TOKENS), + } + } + sym::ident | sym::lifetime => { + // being a single token, idents and lifetimes are harmless + IsInFollow::Yes + } + sym::literal => { + // literals may be of a single token, or two tokens (negative numbers) + IsInFollow::Yes + } + sym::meta | sym::tt => { + // being either a single token or a delimited sequence, tt is + // harmless + IsInFollow::Yes + } + sym::vis => { + // Explicitly disallow `priv`, on the off chance it comes back. + const TOKENS: &[&str] = &["`,`", "an ident", "a type"]; + match tok { + TokenTree::Token(token) => match token.kind { + Comma => IsInFollow::Yes, + Ident(name, is_raw) if is_raw || name != kw::Priv => IsInFollow::Yes, + _ => { + if token.can_begin_type() { + IsInFollow::Yes + } else { + IsInFollow::No(TOKENS) + } + } + }, + TokenTree::MetaVarDecl(_, _, frag) + if frag.name == sym::ident + || frag.name == sym::ty + || frag.name == sym::path => + { + IsInFollow::Yes + } + _ => IsInFollow::No(TOKENS), + } + } + kw::Invalid => IsInFollow::Yes, + _ => IsInFollow::Invalid( + format!("invalid fragment specifier `{}`", frag), + VALID_FRAGMENT_NAMES_MSG, + ), + } + } +} + +fn has_legal_fragment_specifier( + sess: &ParseSess, + features: &Features, + attrs: &[ast::Attribute], + tok: &mbe::TokenTree, +) -> Result<(), String> { + debug!("has_legal_fragment_specifier({:?})", tok); + if let mbe::TokenTree::MetaVarDecl(_, _, ref frag_spec) = *tok { + let frag_span = tok.span(); + if !is_legal_fragment_specifier(sess, features, attrs, frag_spec.name, frag_span) { + return Err(frag_spec.to_string()); + } + } + Ok(()) +} + +fn is_legal_fragment_specifier( + _sess: &ParseSess, + _features: &Features, + _attrs: &[ast::Attribute], + frag_name: Symbol, + _frag_span: Span, +) -> bool { + /* + * If new fragment specifiers are invented in nightly, `_sess`, + * `_features`, `_attrs`, and `_frag_span` will be useful here + * for checking against feature gates. See past versions of + * this function. + */ + match frag_name { + sym::item + | sym::block + | sym::stmt + | sym::expr + | sym::pat + | sym::lifetime + | sym::path + | sym::ty + | sym::ident + | sym::meta + | sym::tt + | sym::vis + | sym::literal + | kw::Invalid => true, + _ => false, + } +} + +fn quoted_tt_to_string(tt: &mbe::TokenTree) -> String { + match *tt { + mbe::TokenTree::Token(ref token) => pprust::token_to_string(&token), + mbe::TokenTree::MetaVar(_, name) => format!("${}", name), + mbe::TokenTree::MetaVarDecl(_, name, kind) => format!("${}:{}", name, kind), + _ => panic!( + "unexpected mbe::TokenTree::{{Sequence or Delimited}} \ + in follow set checker" + ), + } +} + +/// Use this token tree as a matcher to parse given tts. +fn parse_tt(cx: &ExtCtxt<'_>, mtch: &[mbe::TokenTree], tts: TokenStream) -> NamedParseResult { + // `None` is because we're not interpolating + let directory = Directory { + path: Cow::from(cx.current_expansion.module.directory.as_path()), + ownership: cx.current_expansion.directory_ownership, + }; + parse(cx.parse_sess(), tts, mtch, Some(directory), true) +} + +/// Generates an appropriate parsing failure message. For EOF, this is "unexpected end...". For +/// other tokens, this is "unexpected token...". +fn parse_failure_msg(tok: &Token) -> String { + match tok.kind { + token::Eof => "unexpected end of macro invocation".to_string(), + _ => format!( + "no rules expected the token `{}`", + pprust::token_to_string(tok), + ), + } +} diff --git a/src/libsyntax_expand/mbe/quoted.rs b/src/libsyntax_expand/mbe/quoted.rs new file mode 100644 index 00000000000..cedd59233ad --- /dev/null +++ b/src/libsyntax_expand/mbe/quoted.rs @@ -0,0 +1,264 @@ +use crate::mbe::macro_parser; +use crate::mbe::{TokenTree, KleeneOp, KleeneToken, SequenceRepetition, Delimited}; + +use syntax::ast; +use syntax::parse::token::{self, Token}; +use syntax::print::pprust; +use syntax::sess::ParseSess; +use syntax::symbol::kw; +use syntax::tokenstream; + +use syntax_pos::Span; + +use rustc_data_structures::sync::Lrc; + +/// Takes a `tokenstream::TokenStream` and returns a `Vec<self::TokenTree>`. Specifically, this +/// takes a generic `TokenStream`, such as is used in the rest of the compiler, and returns a +/// collection of `TokenTree` for use in parsing a macro. +/// +/// # Parameters +/// +/// - `input`: a token stream to read from, the contents of which we are parsing. +/// - `expect_matchers`: `parse` can be used to parse either the "patterns" or the "body" of a +/// macro. Both take roughly the same form _except_ that in a pattern, metavars are declared with +/// their "matcher" type. For example `$var:expr` or `$id:ident`. In this example, `expr` and +/// `ident` are "matchers". They are not present in the body of a macro rule -- just in the +/// pattern, so we pass a parameter to indicate whether to expect them or not. +/// - `sess`: the parsing session. Any errors will be emitted to this session. +/// - `features`, `attrs`: language feature flags and attributes so that we know whether to use +/// unstable features or not. +/// - `edition`: which edition are we in. +/// - `macro_node_id`: the NodeId of the macro we are parsing. +/// +/// # Returns +/// +/// A collection of `self::TokenTree`. There may also be some errors emitted to `sess`. +pub(super) fn parse( + input: tokenstream::TokenStream, + expect_matchers: bool, + sess: &ParseSess, +) -> Vec<TokenTree> { + // Will contain the final collection of `self::TokenTree` + let mut result = Vec::new(); + + // For each token tree in `input`, parse the token into a `self::TokenTree`, consuming + // additional trees if need be. + let mut trees = input.trees(); + while let Some(tree) = trees.next() { + // Given the parsed tree, if there is a metavar and we are expecting matchers, actually + // parse out the matcher (i.e., in `$id:ident` this would parse the `:` and `ident`). + let tree = parse_tree( + tree, + &mut trees, + expect_matchers, + sess, + ); + match tree { + TokenTree::MetaVar(start_sp, ident) if expect_matchers => { + let span = match trees.next() { + Some(tokenstream::TokenTree::Token(Token { kind: token::Colon, span })) => { + match trees.next() { + Some(tokenstream::TokenTree::Token(token)) => match token.ident() { + Some((kind, _)) => { + let span = token.span.with_lo(start_sp.lo()); + result.push(TokenTree::MetaVarDecl(span, ident, kind)); + continue; + } + _ => token.span, + }, + tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span), + } + } + tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(start_sp), + }; + sess.missing_fragment_specifiers.borrow_mut().insert(span); + result.push(TokenTree::MetaVarDecl(span, ident, ast::Ident::invalid())); + } + + // Not a metavar or no matchers allowed, so just return the tree + _ => result.push(tree), + } + } + result +} + +/// Takes a `tokenstream::TokenTree` and returns a `self::TokenTree`. Specifically, this takes a +/// generic `TokenTree`, such as is used in the rest of the compiler, and returns a `TokenTree` +/// for use in parsing a macro. +/// +/// Converting the given tree may involve reading more tokens. +/// +/// # Parameters +/// +/// - `tree`: the tree we wish to convert. +/// - `trees`: an iterator over trees. We may need to read more tokens from it in order to finish +/// converting `tree` +/// - `expect_matchers`: same as for `parse` (see above). +/// - `sess`: the parsing session. Any errors will be emitted to this session. +/// - `features`, `attrs`: language feature flags and attributes so that we know whether to use +/// unstable features or not. +fn parse_tree( + tree: tokenstream::TokenTree, + trees: &mut impl Iterator<Item = tokenstream::TokenTree>, + expect_matchers: bool, + sess: &ParseSess, +) -> TokenTree { + // Depending on what `tree` is, we could be parsing different parts of a macro + match tree { + // `tree` is a `$` token. Look at the next token in `trees` + tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }) => match trees.next() { + // `tree` is followed by a delimited set of token trees. This indicates the beginning + // of a repetition sequence in the macro (e.g. `$(pat)*`). + Some(tokenstream::TokenTree::Delimited(span, delim, tts)) => { + // Must have `(` not `{` or `[` + if delim != token::Paren { + let tok = pprust::token_kind_to_string(&token::OpenDelim(delim)); + let msg = format!("expected `(`, found `{}`", tok); + sess.span_diagnostic.span_err(span.entire(), &msg); + } + // Parse the contents of the sequence itself + let sequence = parse( + tts.into(), + expect_matchers, + sess, + ); + // Get the Kleene operator and optional separator + let (separator, kleene) = parse_sep_and_kleene_op(trees, span.entire(), sess); + // Count the number of captured "names" (i.e., named metavars) + let name_captures = macro_parser::count_names(&sequence); + TokenTree::Sequence( + span, + Lrc::new(SequenceRepetition { + tts: sequence, + separator, + kleene, + num_captures: name_captures, + }), + ) + } + + // `tree` is followed by an `ident`. This could be `$meta_var` or the `$crate` special + // metavariable that names the crate of the invocation. + Some(tokenstream::TokenTree::Token(token)) if token.is_ident() => { + let (ident, is_raw) = token.ident().unwrap(); + let span = ident.span.with_lo(span.lo()); + if ident.name == kw::Crate && !is_raw { + TokenTree::token(token::Ident(kw::DollarCrate, is_raw), span) + } else { + TokenTree::MetaVar(span, ident) + } + } + + // `tree` is followed by a random token. This is an error. + Some(tokenstream::TokenTree::Token(token)) => { + let msg = + format!("expected identifier, found `{}`", pprust::token_to_string(&token),); + sess.span_diagnostic.span_err(token.span, &msg); + TokenTree::MetaVar(token.span, ast::Ident::invalid()) + } + + // There are no more tokens. Just return the `$` we already have. + None => TokenTree::token(token::Dollar, span), + }, + + // `tree` is an arbitrary token. Keep it. + tokenstream::TokenTree::Token(token) => TokenTree::Token(token), + + // `tree` is the beginning of a delimited set of tokens (e.g., `(` or `{`). We need to + // descend into the delimited set and further parse it. + tokenstream::TokenTree::Delimited(span, delim, tts) => TokenTree::Delimited( + span, + Lrc::new(Delimited { + delim, + tts: parse( + tts.into(), + expect_matchers, + sess, + ), + }), + ), + } +} + +/// Takes a token and returns `Some(KleeneOp)` if the token is `+` `*` or `?`. Otherwise, return +/// `None`. +fn kleene_op(token: &Token) -> Option<KleeneOp> { + match token.kind { + token::BinOp(token::Star) => Some(KleeneOp::ZeroOrMore), + token::BinOp(token::Plus) => Some(KleeneOp::OneOrMore), + token::Question => Some(KleeneOp::ZeroOrOne), + _ => None, + } +} + +/// Parse the next token tree of the input looking for a KleeneOp. Returns +/// +/// - Ok(Ok((op, span))) if the next token tree is a KleeneOp +/// - Ok(Err(tok, span)) if the next token tree is a token but not a KleeneOp +/// - Err(span) if the next token tree is not a token +fn parse_kleene_op( + input: &mut impl Iterator<Item = tokenstream::TokenTree>, + span: Span, +) -> Result<Result<(KleeneOp, Span), Token>, Span> { + match input.next() { + Some(tokenstream::TokenTree::Token(token)) => match kleene_op(&token) { + Some(op) => Ok(Ok((op, token.span))), + None => Ok(Err(token)), + }, + tree => Err(tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span)), + } +} + +/// Attempt to parse a single Kleene star, possibly with a separator. +/// +/// For example, in a pattern such as `$(a),*`, `a` is the pattern to be repeated, `,` is the +/// separator, and `*` is the Kleene operator. This function is specifically concerned with parsing +/// the last two tokens of such a pattern: namely, the optional separator and the Kleene operator +/// itself. Note that here we are parsing the _macro_ itself, rather than trying to match some +/// stream of tokens in an invocation of a macro. +/// +/// This function will take some input iterator `input` corresponding to `span` and a parsing +/// session `sess`. If the next one (or possibly two) tokens in `input` correspond to a Kleene +/// operator and separator, then a tuple with `(separator, KleeneOp)` is returned. Otherwise, an +/// error with the appropriate span is emitted to `sess` and a dummy value is returned. +fn parse_sep_and_kleene_op( + input: &mut impl Iterator<Item = tokenstream::TokenTree>, + span: Span, + sess: &ParseSess, +) -> (Option<Token>, KleeneToken) { + // We basically look at two token trees here, denoted as #1 and #2 below + let span = match parse_kleene_op(input, span) { + // #1 is a `?`, `+`, or `*` KleeneOp + Ok(Ok((op, span))) => return (None, KleeneToken::new(op, span)), + + // #1 is a separator followed by #2, a KleeneOp + Ok(Err(token)) => match parse_kleene_op(input, token.span) { + // #2 is the `?` Kleene op, which does not take a separator (error) + Ok(Ok((KleeneOp::ZeroOrOne, span))) => { + // Error! + sess.span_diagnostic.span_err( + token.span, + "the `?` macro repetition operator does not take a separator", + ); + + // Return a dummy + return (None, KleeneToken::new(KleeneOp::ZeroOrMore, span)); + } + + // #2 is a KleeneOp :D + Ok(Ok((op, span))) => return (Some(token), KleeneToken::new(op, span)), + + // #2 is a random token or not a token at all :( + Ok(Err(Token { span, .. })) | Err(span) => span, + }, + + // #1 is not a token + Err(span) => span, + }; + + // If we ever get to this point, we have experienced an "unexpected token" error + sess.span_diagnostic.span_err(span, "expected one of: `*`, `+`, or `?`"); + + // Return a dummy + (None, KleeneToken::new(KleeneOp::ZeroOrMore, span)) +} diff --git a/src/libsyntax_expand/mbe/transcribe.rs b/src/libsyntax_expand/mbe/transcribe.rs new file mode 100644 index 00000000000..94523bbf91b --- /dev/null +++ b/src/libsyntax_expand/mbe/transcribe.rs @@ -0,0 +1,399 @@ +use crate::base::ExtCtxt; +use crate::mbe; +use crate::mbe::macro_parser::{MatchedNonterminal, MatchedSeq, NamedMatch}; + +use syntax::ast::{Ident, Mac}; +use syntax::mut_visit::{self, MutVisitor}; +use syntax::parse::token::{self, NtTT, Token}; +use syntax::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndJoint}; + +use smallvec::{smallvec, SmallVec}; + +use errors::pluralise; +use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::sync::Lrc; +use syntax_pos::hygiene::{ExpnId, Transparency}; +use syntax_pos::Span; + +use std::mem; + +// A Marker adds the given mark to the syntax context. +struct Marker(ExpnId, Transparency); + +impl MutVisitor for Marker { + fn visit_span(&mut self, span: &mut Span) { + *span = span.apply_mark(self.0, self.1) + } + + fn visit_mac(&mut self, mac: &mut Mac) { + mut_visit::noop_visit_mac(mac, self) + } +} + +impl Marker { + fn visit_delim_span(&mut self, dspan: &mut DelimSpan) { + self.visit_span(&mut dspan.open); + self.visit_span(&mut dspan.close); + } +} + +/// An iterator over the token trees in a delimited token tree (`{ ... }`) or a sequence (`$(...)`). +enum Frame { + Delimited { forest: Lrc<mbe::Delimited>, idx: usize, span: DelimSpan }, + Sequence { forest: Lrc<mbe::SequenceRepetition>, idx: usize, sep: Option<Token> }, +} + +impl Frame { + /// Construct a new frame around the delimited set of tokens. + fn new(tts: Vec<mbe::TokenTree>) -> Frame { + let forest = Lrc::new(mbe::Delimited { delim: token::NoDelim, tts }); + Frame::Delimited { forest, idx: 0, span: DelimSpan::dummy() } + } +} + +impl Iterator for Frame { + type Item = mbe::TokenTree; + + fn next(&mut self) -> Option<mbe::TokenTree> { + match *self { + Frame::Delimited { ref forest, ref mut idx, .. } => { + *idx += 1; + forest.tts.get(*idx - 1).cloned() + } + Frame::Sequence { ref forest, ref mut idx, .. } => { + *idx += 1; + forest.tts.get(*idx - 1).cloned() + } + } + } +} + +/// This can do Macro-By-Example transcription. +/// - `interp` is a map of meta-variables to the tokens (non-terminals) they matched in the +/// invocation. We are assuming we already know there is a match. +/// - `src` is the RHS of the MBE, that is, the "example" we are filling in. +/// +/// For example, +/// +/// ```rust +/// macro_rules! foo { +/// ($id:ident) => { println!("{}", stringify!($id)); } +/// } +/// +/// foo!(bar); +/// ``` +/// +/// `interp` would contain `$id => bar` and `src` would contain `println!("{}", stringify!($id));`. +/// +/// `transcribe` would return a `TokenStream` containing `println!("{}", stringify!(bar));`. +/// +/// Along the way, we do some additional error checking. +pub(super) fn transcribe( + cx: &ExtCtxt<'_>, + interp: &FxHashMap<Ident, NamedMatch>, + src: Vec<mbe::TokenTree>, + transparency: Transparency, +) -> TokenStream { + // Nothing for us to transcribe... + if src.is_empty() { + return TokenStream::default(); + } + + // We descend into the RHS (`src`), expanding things as we go. This stack contains the things + // we have yet to expand/are still expanding. We start the stack off with the whole RHS. + let mut stack: SmallVec<[Frame; 1]> = smallvec![Frame::new(src)]; + + // As we descend in the RHS, we will need to be able to match nested sequences of matchers. + // `repeats` keeps track of where we are in matching at each level, with the last element being + // the most deeply nested sequence. This is used as a stack. + let mut repeats = Vec::new(); + + // `result` contains resulting token stream from the TokenTree we just finished processing. At + // the end, this will contain the full result of transcription, but at arbitrary points during + // `transcribe`, `result` will contain subsets of the final result. + // + // Specifically, as we descend into each TokenTree, we will push the existing results onto the + // `result_stack` and clear `results`. We will then produce the results of transcribing the + // TokenTree into `results`. Then, as we unwind back out of the `TokenTree`, we will pop the + // `result_stack` and append `results` too it to produce the new `results` up to that point. + // + // Thus, if we try to pop the `result_stack` and it is empty, we have reached the top-level + // again, and we are done transcribing. + let mut result: Vec<TreeAndJoint> = Vec::new(); + let mut result_stack = Vec::new(); + let mut marker = Marker(cx.current_expansion.id, transparency); + + loop { + // Look at the last frame on the stack. + let tree = if let Some(tree) = stack.last_mut().unwrap().next() { + // If it still has a TokenTree we have not looked at yet, use that tree. + tree + } + // The else-case never produces a value for `tree` (it `continue`s or `return`s). + else { + // Otherwise, if we have just reached the end of a sequence and we can keep repeating, + // go back to the beginning of the sequence. + if let Frame::Sequence { idx, sep, .. } = stack.last_mut().unwrap() { + let (repeat_idx, repeat_len) = repeats.last_mut().unwrap(); + *repeat_idx += 1; + if repeat_idx < repeat_len { + *idx = 0; + if let Some(sep) = sep { + result.push(TokenTree::Token(sep.clone()).into()); + } + continue; + } + } + + // We are done with the top of the stack. Pop it. Depending on what it was, we do + // different things. Note that the outermost item must be the delimited, wrapped RHS + // that was passed in originally to `transcribe`. + match stack.pop().unwrap() { + // Done with a sequence. Pop from repeats. + Frame::Sequence { .. } => { + repeats.pop(); + } + + // We are done processing a Delimited. If this is the top-level delimited, we are + // done. Otherwise, we unwind the result_stack to append what we have produced to + // any previous results. + Frame::Delimited { forest, span, .. } => { + if result_stack.is_empty() { + // No results left to compute! We are back at the top-level. + return TokenStream::new(result); + } + + // Step back into the parent Delimited. + let tree = + TokenTree::Delimited(span, forest.delim, TokenStream::new(result).into()); + result = result_stack.pop().unwrap(); + result.push(tree.into()); + } + } + continue; + }; + + // At this point, we know we are in the middle of a TokenTree (the last one on `stack`). + // `tree` contains the next `TokenTree` to be processed. + match tree { + // We are descending into a sequence. We first make sure that the matchers in the RHS + // and the matches in `interp` have the same shape. Otherwise, either the caller or the + // macro writer has made a mistake. + seq @ mbe::TokenTree::Sequence(..) => { + match lockstep_iter_size(&seq, interp, &repeats) { + LockstepIterSize::Unconstrained => { + cx.span_fatal( + seq.span(), /* blame macro writer */ + "attempted to repeat an expression containing no syntax variables \ + matched as repeating at this depth", + ); + } + + LockstepIterSize::Contradiction(ref msg) => { + // FIXME: this really ought to be caught at macro definition time... It + // happens when two meta-variables are used in the same repetition in a + // sequence, but they come from different sequence matchers and repeat + // different amounts. + cx.span_fatal(seq.span(), &msg[..]); + } + + LockstepIterSize::Constraint(len, _) => { + // We do this to avoid an extra clone above. We know that this is a + // sequence already. + let (sp, seq) = if let mbe::TokenTree::Sequence(sp, seq) = seq { + (sp, seq) + } else { + unreachable!() + }; + + // Is the repetition empty? + if len == 0 { + if seq.kleene.op == mbe::KleeneOp::OneOrMore { + // FIXME: this really ought to be caught at macro definition + // time... It happens when the Kleene operator in the matcher and + // the body for the same meta-variable do not match. + cx.span_fatal(sp.entire(), "this must repeat at least once"); + } + } else { + // 0 is the initial counter (we have done 0 repretitions so far). `len` + // is the total number of reptitions we should generate. + repeats.push((0, len)); + + // The first time we encounter the sequence we push it to the stack. It + // then gets reused (see the beginning of the loop) until we are done + // repeating. + stack.push(Frame::Sequence { + idx: 0, + sep: seq.separator.clone(), + forest: seq, + }); + } + } + } + } + + // Replace the meta-var with the matched token tree from the invocation. + mbe::TokenTree::MetaVar(mut sp, mut ident) => { + // Find the matched nonterminal from the macro invocation, and use it to replace + // the meta-var. + if let Some(cur_matched) = lookup_cur_matched(ident, interp, &repeats) { + if let MatchedNonterminal(ref nt) = cur_matched { + // FIXME #2887: why do we apply a mark when matching a token tree meta-var + // (e.g. `$x:tt`), but not when we are matching any other type of token + // tree? + if let NtTT(ref tt) = **nt { + result.push(tt.clone().into()); + } else { + marker.visit_span(&mut sp); + let token = TokenTree::token(token::Interpolated(nt.clone()), sp); + result.push(token.into()); + } + } else { + // We were unable to descend far enough. This is an error. + cx.span_fatal( + sp, /* blame the macro writer */ + &format!("variable '{}' is still repeating at this depth", ident), + ); + } + } else { + // If we aren't able to match the meta-var, we push it back into the result but + // with modified syntax context. (I believe this supports nested macros). + marker.visit_span(&mut sp); + marker.visit_ident(&mut ident); + result.push(TokenTree::token(token::Dollar, sp).into()); + result.push(TokenTree::Token(Token::from_ast_ident(ident)).into()); + } + } + + // If we are entering a new delimiter, we push its contents to the `stack` to be + // processed, and we push all of the currently produced results to the `result_stack`. + // We will produce all of the results of the inside of the `Delimited` and then we will + // jump back out of the Delimited, pop the result_stack and add the new results back to + // the previous results (from outside the Delimited). + mbe::TokenTree::Delimited(mut span, delimited) => { + marker.visit_delim_span(&mut span); + stack.push(Frame::Delimited { forest: delimited, idx: 0, span }); + result_stack.push(mem::take(&mut result)); + } + + // Nothing much to do here. Just push the token to the result, being careful to + // preserve syntax context. + mbe::TokenTree::Token(token) => { + let mut tt = TokenTree::Token(token); + marker.visit_tt(&mut tt); + result.push(tt.into()); + } + + // There should be no meta-var declarations in the invocation of a macro. + mbe::TokenTree::MetaVarDecl(..) => panic!("unexpected `TokenTree::MetaVarDecl"), + } + } +} + +/// Lookup the meta-var named `ident` and return the matched token tree from the invocation using +/// the set of matches `interpolations`. +/// +/// See the definition of `repeats` in the `transcribe` function. `repeats` is used to descend +/// into the right place in nested matchers. If we attempt to descend too far, the macro writer has +/// made a mistake, and we return `None`. +fn lookup_cur_matched<'a>( + ident: Ident, + interpolations: &'a FxHashMap<Ident, NamedMatch>, + repeats: &[(usize, usize)], +) -> Option<&'a NamedMatch> { + interpolations.get(&ident).map(|matched| { + let mut matched = matched; + for &(idx, _) in repeats { + match matched { + MatchedNonterminal(_) => break, + MatchedSeq(ref ads, _) => matched = ads.get(idx).unwrap(), + } + } + + matched + }) +} + +/// An accumulator over a TokenTree to be used with `fold`. During transcription, we need to make +/// sure that the size of each sequence and all of its nested sequences are the same as the sizes +/// of all the matched (nested) sequences in the macro invocation. If they don't match, somebody +/// has made a mistake (either the macro writer or caller). +#[derive(Clone)] +enum LockstepIterSize { + /// No constraints on length of matcher. This is true for any TokenTree variants except a + /// `MetaVar` with an actual `MatchedSeq` (as opposed to a `MatchedNonterminal`). + Unconstrained, + + /// A `MetaVar` with an actual `MatchedSeq`. The length of the match and the name of the + /// meta-var are returned. + Constraint(usize, Ident), + + /// Two `Constraint`s on the same sequence had different lengths. This is an error. + Contradiction(String), +} + +impl LockstepIterSize { + /// Find incompatibilities in matcher/invocation sizes. + /// - `Unconstrained` is compatible with everything. + /// - `Contradiction` is incompatible with everything. + /// - `Constraint(len)` is only compatible with other constraints of the same length. + fn with(self, other: LockstepIterSize) -> LockstepIterSize { + match self { + LockstepIterSize::Unconstrained => other, + LockstepIterSize::Contradiction(_) => self, + LockstepIterSize::Constraint(l_len, ref l_id) => match other { + LockstepIterSize::Unconstrained => self, + LockstepIterSize::Contradiction(_) => other, + LockstepIterSize::Constraint(r_len, _) if l_len == r_len => self, + LockstepIterSize::Constraint(r_len, r_id) => { + let msg = format!( + "meta-variable `{}` repeats {} time{}, but `{}` repeats {} time{}", + l_id, + l_len, + pluralise!(l_len), + r_id, + r_len, + pluralise!(r_len), + ); + LockstepIterSize::Contradiction(msg) + } + }, + } + } +} + +/// Given a `tree`, make sure that all sequences have the same length as the matches for the +/// appropriate meta-vars in `interpolations`. +/// +/// Note that if `repeats` does not match the exact correct depth of a meta-var, +/// `lookup_cur_matched` will return `None`, which is why this still works even in the presnece of +/// multiple nested matcher sequences. +fn lockstep_iter_size( + tree: &mbe::TokenTree, + interpolations: &FxHashMap<Ident, NamedMatch>, + repeats: &[(usize, usize)], +) -> LockstepIterSize { + use mbe::TokenTree; + match *tree { + TokenTree::Delimited(_, ref delimed) => { + delimed.tts.iter().fold(LockstepIterSize::Unconstrained, |size, tt| { + size.with(lockstep_iter_size(tt, interpolations, repeats)) + }) + } + TokenTree::Sequence(_, ref seq) => { + seq.tts.iter().fold(LockstepIterSize::Unconstrained, |size, tt| { + size.with(lockstep_iter_size(tt, interpolations, repeats)) + }) + } + TokenTree::MetaVar(_, name) | TokenTree::MetaVarDecl(_, name, _) => { + match lookup_cur_matched(name, interpolations, repeats) { + Some(matched) => match matched { + MatchedNonterminal(_) => LockstepIterSize::Unconstrained, + MatchedSeq(ref ads, _) => LockstepIterSize::Constraint(ads.len(), name), + }, + _ => LockstepIterSize::Unconstrained, + } + } + TokenTree::Token(..) => LockstepIterSize::Unconstrained, + } +} diff --git a/src/libsyntax_expand/placeholders.rs b/src/libsyntax_expand/placeholders.rs new file mode 100644 index 00000000000..f2c89e14b53 --- /dev/null +++ b/src/libsyntax_expand/placeholders.rs @@ -0,0 +1,350 @@ +use crate::base::ExtCtxt; +use crate::expand::{AstFragment, AstFragmentKind}; + +use syntax::ast::{self, NodeId}; +use syntax::source_map::{DUMMY_SP, dummy_spanned}; +use syntax::tokenstream::TokenStream; +use syntax::mut_visit::*; +use syntax::ptr::P; +use syntax::ThinVec; + +use smallvec::{smallvec, SmallVec}; + +use rustc_data_structures::fx::FxHashMap; + +pub fn placeholder(kind: AstFragmentKind, id: ast::NodeId) -> AstFragment { + fn mac_placeholder() -> ast::Mac { + ast::Mac { + path: ast::Path { span: DUMMY_SP, segments: Vec::new() }, + tts: TokenStream::default().into(), + delim: ast::MacDelimiter::Brace, + span: DUMMY_SP, + prior_type_ascription: None, + } + } + + let ident = ast::Ident::invalid(); + let attrs = Vec::new(); + let generics = ast::Generics::default(); + let vis = dummy_spanned(ast::VisibilityKind::Inherited); + let span = DUMMY_SP; + let expr_placeholder = || P(ast::Expr { + id, span, + attrs: ThinVec::new(), + kind: ast::ExprKind::Mac(mac_placeholder()), + }); + let ty = || P(ast::Ty { + id, + kind: ast::TyKind::Mac(mac_placeholder()), + span, + }); + let pat = || P(ast::Pat { + id, + kind: ast::PatKind::Mac(mac_placeholder()), + span, + }); + + match kind { + AstFragmentKind::Expr => AstFragment::Expr(expr_placeholder()), + AstFragmentKind::OptExpr => AstFragment::OptExpr(Some(expr_placeholder())), + AstFragmentKind::Items => AstFragment::Items(smallvec![P(ast::Item { + id, span, ident, vis, attrs, + kind: ast::ItemKind::Mac(mac_placeholder()), + tokens: None, + })]), + AstFragmentKind::TraitItems => AstFragment::TraitItems(smallvec![ast::TraitItem { + id, span, ident, attrs, generics, + kind: ast::TraitItemKind::Macro(mac_placeholder()), + tokens: None, + }]), + AstFragmentKind::ImplItems => AstFragment::ImplItems(smallvec![ast::ImplItem { + id, span, ident, vis, attrs, generics, + kind: ast::ImplItemKind::Macro(mac_placeholder()), + defaultness: ast::Defaultness::Final, + tokens: None, + }]), + AstFragmentKind::ForeignItems => + AstFragment::ForeignItems(smallvec![ast::ForeignItem { + id, span, ident, vis, attrs, + kind: ast::ForeignItemKind::Macro(mac_placeholder()), + }]), + AstFragmentKind::Pat => AstFragment::Pat(P(ast::Pat { + id, span, kind: ast::PatKind::Mac(mac_placeholder()), + })), + AstFragmentKind::Ty => AstFragment::Ty(P(ast::Ty { + id, span, kind: ast::TyKind::Mac(mac_placeholder()), + })), + AstFragmentKind::Stmts => AstFragment::Stmts(smallvec![{ + let mac = P((mac_placeholder(), ast::MacStmtStyle::Braces, ThinVec::new())); + ast::Stmt { id, span, kind: ast::StmtKind::Mac(mac) } + }]), + AstFragmentKind::Arms => AstFragment::Arms(smallvec![ + ast::Arm { + attrs: Default::default(), + body: expr_placeholder(), + guard: None, + id, + pat: pat(), + span, + is_placeholder: true, + } + ]), + AstFragmentKind::Fields => AstFragment::Fields(smallvec![ + ast::Field { + attrs: Default::default(), + expr: expr_placeholder(), + id, + ident, + is_shorthand: false, + span, + is_placeholder: true, + } + ]), + AstFragmentKind::FieldPats => AstFragment::FieldPats(smallvec![ + ast::FieldPat { + attrs: Default::default(), + id, + ident, + is_shorthand: false, + pat: pat(), + span, + is_placeholder: true, + } + ]), + AstFragmentKind::GenericParams => AstFragment::GenericParams(smallvec![{ + ast::GenericParam { + attrs: Default::default(), + bounds: Default::default(), + id, + ident, + is_placeholder: true, + kind: ast::GenericParamKind::Lifetime, + } + }]), + AstFragmentKind::Params => AstFragment::Params(smallvec![ + ast::Param { + attrs: Default::default(), + id, + pat: pat(), + span, + ty: ty(), + is_placeholder: true, + } + ]), + AstFragmentKind::StructFields => AstFragment::StructFields(smallvec![ + ast::StructField { + attrs: Default::default(), + id, + ident: None, + span, + ty: ty(), + vis, + is_placeholder: true, + } + ]), + AstFragmentKind::Variants => AstFragment::Variants(smallvec![ + ast::Variant { + attrs: Default::default(), + data: ast::VariantData::Struct(Default::default(), false), + disr_expr: None, + id, + ident, + span, + is_placeholder: true, + } + ]) + } +} + +pub struct PlaceholderExpander<'a, 'b> { + expanded_fragments: FxHashMap<ast::NodeId, AstFragment>, + cx: &'a mut ExtCtxt<'b>, + monotonic: bool, +} + +impl<'a, 'b> PlaceholderExpander<'a, 'b> { + pub fn new(cx: &'a mut ExtCtxt<'b>, monotonic: bool) -> Self { + PlaceholderExpander { + cx, + expanded_fragments: FxHashMap::default(), + monotonic, + } + } + + pub fn add(&mut self, id: ast::NodeId, mut fragment: AstFragment, placeholders: Vec<NodeId>) { + fragment.mut_visit_with(self); + if let AstFragment::Items(mut items) = fragment { + for placeholder in placeholders { + match self.remove(placeholder) { + AstFragment::Items(derived_items) => items.extend(derived_items), + _ => unreachable!(), + } + } + fragment = AstFragment::Items(items); + } + self.expanded_fragments.insert(id, fragment); + } + + fn remove(&mut self, id: ast::NodeId) -> AstFragment { + self.expanded_fragments.remove(&id).unwrap() + } +} + +impl<'a, 'b> MutVisitor for PlaceholderExpander<'a, 'b> { + fn flat_map_arm(&mut self, arm: ast::Arm) -> SmallVec<[ast::Arm; 1]> { + if arm.is_placeholder { + self.remove(arm.id).make_arms() + } else { + noop_flat_map_arm(arm, self) + } + } + + fn flat_map_field(&mut self, field: ast::Field) -> SmallVec<[ast::Field; 1]> { + if field.is_placeholder { + self.remove(field.id).make_fields() + } else { + noop_flat_map_field(field, self) + } + } + + fn flat_map_field_pattern(&mut self, fp: ast::FieldPat) -> SmallVec<[ast::FieldPat; 1]> { + if fp.is_placeholder { + self.remove(fp.id).make_field_patterns() + } else { + noop_flat_map_field_pattern(fp, self) + } + } + + fn flat_map_generic_param( + &mut self, + param: ast::GenericParam + ) -> SmallVec<[ast::GenericParam; 1]> + { + if param.is_placeholder { + self.remove(param.id).make_generic_params() + } else { + noop_flat_map_generic_param(param, self) + } + } + + fn flat_map_param(&mut self, p: ast::Param) -> SmallVec<[ast::Param; 1]> { + if p.is_placeholder { + self.remove(p.id).make_params() + } else { + noop_flat_map_param(p, self) + } + } + + fn flat_map_struct_field(&mut self, sf: ast::StructField) -> SmallVec<[ast::StructField; 1]> { + if sf.is_placeholder { + self.remove(sf.id).make_struct_fields() + } else { + noop_flat_map_struct_field(sf, self) + } + } + + fn flat_map_variant(&mut self, variant: ast::Variant) -> SmallVec<[ast::Variant; 1]> { + if variant.is_placeholder { + self.remove(variant.id).make_variants() + } else { + noop_flat_map_variant(variant, self) + } + } + + fn flat_map_item(&mut self, item: P<ast::Item>) -> SmallVec<[P<ast::Item>; 1]> { + match item.kind { + ast::ItemKind::Mac(_) => return self.remove(item.id).make_items(), + ast::ItemKind::MacroDef(_) => return smallvec![item], + _ => {} + } + + noop_flat_map_item(item, self) + } + + fn flat_map_trait_item(&mut self, item: ast::TraitItem) -> SmallVec<[ast::TraitItem; 1]> { + match item.kind { + ast::TraitItemKind::Macro(_) => self.remove(item.id).make_trait_items(), + _ => noop_flat_map_trait_item(item, self), + } + } + + fn flat_map_impl_item(&mut self, item: ast::ImplItem) -> SmallVec<[ast::ImplItem; 1]> { + match item.kind { + ast::ImplItemKind::Macro(_) => self.remove(item.id).make_impl_items(), + _ => noop_flat_map_impl_item(item, self), + } + } + + fn flat_map_foreign_item(&mut self, item: ast::ForeignItem) -> SmallVec<[ast::ForeignItem; 1]> { + match item.kind { + ast::ForeignItemKind::Macro(_) => self.remove(item.id).make_foreign_items(), + _ => noop_flat_map_foreign_item(item, self), + } + } + + fn visit_expr(&mut self, expr: &mut P<ast::Expr>) { + match expr.kind { + ast::ExprKind::Mac(_) => *expr = self.remove(expr.id).make_expr(), + _ => noop_visit_expr(expr, self), + } + } + + fn filter_map_expr(&mut self, expr: P<ast::Expr>) -> Option<P<ast::Expr>> { + match expr.kind { + ast::ExprKind::Mac(_) => self.remove(expr.id).make_opt_expr(), + _ => noop_filter_map_expr(expr, self), + } + } + + fn flat_map_stmt(&mut self, stmt: ast::Stmt) -> SmallVec<[ast::Stmt; 1]> { + let (style, mut stmts) = match stmt.kind { + ast::StmtKind::Mac(mac) => (mac.1, self.remove(stmt.id).make_stmts()), + _ => return noop_flat_map_stmt(stmt, self), + }; + + if style == ast::MacStmtStyle::Semicolon { + if let Some(stmt) = stmts.pop() { + stmts.push(stmt.add_trailing_semicolon()); + } + } + + stmts + } + + fn visit_pat(&mut self, pat: &mut P<ast::Pat>) { + match pat.kind { + ast::PatKind::Mac(_) => *pat = self.remove(pat.id).make_pat(), + _ => noop_visit_pat(pat, self), + } + } + + fn visit_ty(&mut self, ty: &mut P<ast::Ty>) { + match ty.kind { + ast::TyKind::Mac(_) => *ty = self.remove(ty.id).make_ty(), + _ => noop_visit_ty(ty, self), + } + } + + fn visit_block(&mut self, block: &mut P<ast::Block>) { + noop_visit_block(block, self); + + for stmt in block.stmts.iter_mut() { + if self.monotonic { + assert_eq!(stmt.id, ast::DUMMY_NODE_ID); + stmt.id = self.cx.resolver.next_node_id(); + } + } + } + + fn visit_mod(&mut self, module: &mut ast::Mod) { + noop_visit_mod(module, self); + module.items.retain(|item| match item.kind { + ast::ItemKind::Mac(_) if !self.cx.ecfg.keep_macs => false, // remove macro definitions + _ => true, + }); + } + + fn visit_mac(&mut self, _mac: &mut ast::Mac) { + // Do nothing. + } +} diff --git a/src/libsyntax_expand/proc_macro.rs b/src/libsyntax_expand/proc_macro.rs new file mode 100644 index 00000000000..07b618c99a5 --- /dev/null +++ b/src/libsyntax_expand/proc_macro.rs @@ -0,0 +1,215 @@ +use crate::base::{self, *}; +use crate::proc_macro_server; + +use syntax::ast::{self, ItemKind, Attribute, Mac}; +use syntax::attr::{mark_used, mark_known}; +use syntax::errors::{Applicability, FatalError}; +use syntax::parse::{self, token}; +use syntax::symbol::sym; +use syntax::tokenstream::{self, TokenStream}; +use syntax::visit::Visitor; + +use rustc_data_structures::sync::Lrc; +use syntax_pos::{Span, DUMMY_SP}; + +const EXEC_STRATEGY: pm::bridge::server::SameThread = pm::bridge::server::SameThread; + +pub struct BangProcMacro { + pub client: pm::bridge::client::Client< + fn(pm::TokenStream) -> pm::TokenStream, + >, +} + +impl base::ProcMacro for BangProcMacro { + fn expand<'cx>(&self, + ecx: &'cx mut ExtCtxt<'_>, + span: Span, + input: TokenStream) + -> TokenStream { + let server = proc_macro_server::Rustc::new(ecx); + match self.client.run(&EXEC_STRATEGY, server, input) { + Ok(stream) => stream, + Err(e) => { + let msg = "proc macro panicked"; + let mut err = ecx.struct_span_fatal(span, msg); + if let Some(s) = e.as_str() { + err.help(&format!("message: {}", s)); + } + + err.emit(); + FatalError.raise(); + } + } + } +} + +pub struct AttrProcMacro { + pub client: pm::bridge::client::Client<fn(pm::TokenStream, pm::TokenStream) -> pm::TokenStream>, +} + +impl base::AttrProcMacro for AttrProcMacro { + fn expand<'cx>(&self, + ecx: &'cx mut ExtCtxt<'_>, + span: Span, + annotation: TokenStream, + annotated: TokenStream) + -> TokenStream { + let server = proc_macro_server::Rustc::new(ecx); + match self.client.run(&EXEC_STRATEGY, server, annotation, annotated) { + Ok(stream) => stream, + Err(e) => { + let msg = "custom attribute panicked"; + let mut err = ecx.struct_span_fatal(span, msg); + if let Some(s) = e.as_str() { + err.help(&format!("message: {}", s)); + } + + err.emit(); + FatalError.raise(); + } + } + } +} + +pub struct ProcMacroDerive { + pub client: pm::bridge::client::Client<fn(pm::TokenStream) -> pm::TokenStream>, +} + +impl MultiItemModifier for ProcMacroDerive { + fn expand(&self, + ecx: &mut ExtCtxt<'_>, + span: Span, + _meta_item: &ast::MetaItem, + item: Annotatable) + -> Vec<Annotatable> { + let item = match item { + Annotatable::Arm(..) | + Annotatable::Field(..) | + Annotatable::FieldPat(..) | + Annotatable::GenericParam(..) | + Annotatable::Param(..) | + Annotatable::StructField(..) | + Annotatable::Variant(..) + => panic!("unexpected annotatable"), + Annotatable::Item(item) => item, + Annotatable::ImplItem(_) | + Annotatable::TraitItem(_) | + Annotatable::ForeignItem(_) | + Annotatable::Stmt(_) | + Annotatable::Expr(_) => { + ecx.span_err(span, "proc-macro derives may only be \ + applied to a struct, enum, or union"); + return Vec::new() + } + }; + match item.kind { + ItemKind::Struct(..) | + ItemKind::Enum(..) | + ItemKind::Union(..) => {}, + _ => { + ecx.span_err(span, "proc-macro derives may only be \ + applied to a struct, enum, or union"); + return Vec::new() + } + } + + let token = token::Interpolated(Lrc::new(token::NtItem(item))); + let input = tokenstream::TokenTree::token(token, DUMMY_SP).into(); + + let server = proc_macro_server::Rustc::new(ecx); + let stream = match self.client.run(&EXEC_STRATEGY, server, input) { + Ok(stream) => stream, + Err(e) => { + let msg = "proc-macro derive panicked"; + let mut err = ecx.struct_span_fatal(span, msg); + if let Some(s) = e.as_str() { + err.help(&format!("message: {}", s)); + } + + err.emit(); + FatalError.raise(); + } + }; + + let error_count_before = ecx.parse_sess.span_diagnostic.err_count(); + let msg = "proc-macro derive produced unparseable tokens"; + + let mut parser = parse::stream_to_parser(ecx.parse_sess, stream, Some("proc-macro derive")); + let mut items = vec![]; + + loop { + match parser.parse_item() { + Ok(None) => break, + Ok(Some(item)) => { + items.push(Annotatable::Item(item)) + } + Err(mut err) => { + // FIXME: handle this better + err.cancel(); + ecx.struct_span_fatal(span, msg).emit(); + FatalError.raise(); + } + } + } + + + // fail if there have been errors emitted + if ecx.parse_sess.span_diagnostic.err_count() > error_count_before { + ecx.struct_span_fatal(span, msg).emit(); + FatalError.raise(); + } + + items + } +} + +crate struct MarkAttrs<'a>(crate &'a [ast::Name]); + +impl<'a> Visitor<'a> for MarkAttrs<'a> { + fn visit_attribute(&mut self, attr: &Attribute) { + if let Some(ident) = attr.ident() { + if self.0.contains(&ident.name) { + mark_used(attr); + mark_known(attr); + } + } + } + + fn visit_mac(&mut self, _mac: &Mac) {} +} + +pub fn is_proc_macro_attr(attr: &Attribute) -> bool { + [sym::proc_macro, sym::proc_macro_attribute, sym::proc_macro_derive] + .iter().any(|kind| attr.check_name(*kind)) +} + +crate fn collect_derives(cx: &mut ExtCtxt<'_>, attrs: &mut Vec<ast::Attribute>) -> Vec<ast::Path> { + let mut result = Vec::new(); + attrs.retain(|attr| { + if attr.path != sym::derive { + return true; + } + if !attr.is_meta_item_list() { + cx.struct_span_err(attr.span, "malformed `derive` attribute input") + .span_suggestion( + attr.span, + "missing traits to be derived", + "#[derive(Trait1, Trait2, ...)]".to_owned(), + Applicability::HasPlaceholders, + ).emit(); + return false; + } + + match attr.parse_derive_paths(cx.parse_sess) { + Ok(traits) => { + result.extend(traits); + true + } + Err(mut e) => { + e.emit(); + false + } + } + }); + result +} diff --git a/src/libsyntax_expand/proc_macro_server.rs b/src/libsyntax_expand/proc_macro_server.rs new file mode 100644 index 00000000000..4ce99cfe73b --- /dev/null +++ b/src/libsyntax_expand/proc_macro_server.rs @@ -0,0 +1,713 @@ +use crate::base::ExtCtxt; + +use syntax::ast; +use syntax::parse::{self, token}; +use syntax::parse::lexer::comments; +use syntax::print::pprust; +use syntax::sess::ParseSess; +use syntax::tokenstream::{self, DelimSpan, IsJoint::*, TokenStream, TreeAndJoint}; + +use errors::Diagnostic; +use rustc_data_structures::sync::Lrc; +use syntax_pos::{BytePos, FileName, MultiSpan, Pos, SourceFile, Span}; +use syntax_pos::symbol::{kw, sym, Symbol}; + +use pm::{Delimiter, Level, LineColumn, Spacing}; +use pm::bridge::{server, TokenTree}; +use std::{ascii, panic}; +use std::ops::Bound; + +trait FromInternal<T> { + fn from_internal(x: T) -> Self; +} + +trait ToInternal<T> { + fn to_internal(self) -> T; +} + +impl FromInternal<token::DelimToken> for Delimiter { + fn from_internal(delim: token::DelimToken) -> Delimiter { + match delim { + token::Paren => Delimiter::Parenthesis, + token::Brace => Delimiter::Brace, + token::Bracket => Delimiter::Bracket, + token::NoDelim => Delimiter::None, + } + } +} + +impl ToInternal<token::DelimToken> for Delimiter { + fn to_internal(self) -> token::DelimToken { + match self { + Delimiter::Parenthesis => token::Paren, + Delimiter::Brace => token::Brace, + Delimiter::Bracket => token::Bracket, + Delimiter::None => token::NoDelim, + } + } +} + +impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec<Self>)> + for TokenTree<Group, Punct, Ident, Literal> +{ + fn from_internal(((tree, is_joint), sess, stack): (TreeAndJoint, &ParseSess, &mut Vec<Self>)) + -> Self { + use syntax::parse::token::*; + + let joint = is_joint == Joint; + let Token { kind, span } = match tree { + tokenstream::TokenTree::Delimited(span, delim, tts) => { + let delimiter = Delimiter::from_internal(delim); + return TokenTree::Group(Group { + delimiter, + stream: tts.into(), + span, + }); + } + tokenstream::TokenTree::Token(token) => token, + }; + + macro_rules! tt { + ($ty:ident { $($field:ident $(: $value:expr)*),+ $(,)? }) => ( + TokenTree::$ty(self::$ty { + $($field $(: $value)*,)+ + span, + }) + ); + ($ty:ident::$method:ident($($value:expr),*)) => ( + TokenTree::$ty(self::$ty::$method($($value,)* span)) + ); + } + macro_rules! op { + ($a:expr) => { + tt!(Punct::new($a, joint)) + }; + ($a:expr, $b:expr) => {{ + stack.push(tt!(Punct::new($b, joint))); + tt!(Punct::new($a, true)) + }}; + ($a:expr, $b:expr, $c:expr) => {{ + stack.push(tt!(Punct::new($c, joint))); + stack.push(tt!(Punct::new($b, true))); + tt!(Punct::new($a, true)) + }}; + } + + match kind { + Eq => op!('='), + Lt => op!('<'), + Le => op!('<', '='), + EqEq => op!('=', '='), + Ne => op!('!', '='), + Ge => op!('>', '='), + Gt => op!('>'), + AndAnd => op!('&', '&'), + OrOr => op!('|', '|'), + Not => op!('!'), + Tilde => op!('~'), + BinOp(Plus) => op!('+'), + BinOp(Minus) => op!('-'), + BinOp(Star) => op!('*'), + BinOp(Slash) => op!('/'), + BinOp(Percent) => op!('%'), + BinOp(Caret) => op!('^'), + BinOp(And) => op!('&'), + BinOp(Or) => op!('|'), + BinOp(Shl) => op!('<', '<'), + BinOp(Shr) => op!('>', '>'), + BinOpEq(Plus) => op!('+', '='), + BinOpEq(Minus) => op!('-', '='), + BinOpEq(Star) => op!('*', '='), + BinOpEq(Slash) => op!('/', '='), + BinOpEq(Percent) => op!('%', '='), + BinOpEq(Caret) => op!('^', '='), + BinOpEq(And) => op!('&', '='), + BinOpEq(Or) => op!('|', '='), + BinOpEq(Shl) => op!('<', '<', '='), + BinOpEq(Shr) => op!('>', '>', '='), + At => op!('@'), + Dot => op!('.'), + DotDot => op!('.', '.'), + DotDotDot => op!('.', '.', '.'), + DotDotEq => op!('.', '.', '='), + Comma => op!(','), + Semi => op!(';'), + Colon => op!(':'), + ModSep => op!(':', ':'), + RArrow => op!('-', '>'), + LArrow => op!('<', '-'), + FatArrow => op!('=', '>'), + Pound => op!('#'), + Dollar => op!('$'), + Question => op!('?'), + SingleQuote => op!('\''), + + Ident(name, false) if name == kw::DollarCrate => tt!(Ident::dollar_crate()), + Ident(name, is_raw) => tt!(Ident::new(name, is_raw)), + Lifetime(name) => { + let ident = ast::Ident::new(name, span).without_first_quote(); + stack.push(tt!(Ident::new(ident.name, false))); + tt!(Punct::new('\'', true)) + } + Literal(lit) => tt!(Literal { lit }), + DocComment(c) => { + let style = comments::doc_comment_style(&c.as_str()); + let stripped = comments::strip_doc_comment_decoration(&c.as_str()); + let mut escaped = String::new(); + for ch in stripped.chars() { + escaped.extend(ch.escape_debug()); + } + let stream = vec![ + Ident(sym::doc, false), + Eq, + TokenKind::lit(token::Str, Symbol::intern(&escaped), None), + ] + .into_iter() + .map(|kind| tokenstream::TokenTree::token(kind, span)) + .collect(); + stack.push(TokenTree::Group(Group { + delimiter: Delimiter::Bracket, + stream, + span: DelimSpan::from_single(span), + })); + if style == ast::AttrStyle::Inner { + stack.push(tt!(Punct::new('!', false))); + } + tt!(Punct::new('#', false)) + } + + Interpolated(nt) => { + let stream = parse::nt_to_tokenstream(&nt, sess, span); + TokenTree::Group(Group { + delimiter: Delimiter::None, + stream, + span: DelimSpan::from_single(span), + }) + } + + OpenDelim(..) | CloseDelim(..) => unreachable!(), + Whitespace | Comment | Shebang(..) | Unknown(..) | Eof => unreachable!(), + } + } +} + +impl ToInternal<TokenStream> for TokenTree<Group, Punct, Ident, Literal> { + fn to_internal(self) -> TokenStream { + use syntax::parse::token::*; + + let (ch, joint, span) = match self { + TokenTree::Punct(Punct { ch, joint, span }) => (ch, joint, span), + TokenTree::Group(Group { + delimiter, + stream, + span, + }) => { + return tokenstream::TokenTree::Delimited( + span, + delimiter.to_internal(), + stream.into(), + ) + .into(); + } + TokenTree::Ident(self::Ident { sym, is_raw, span }) => { + return tokenstream::TokenTree::token(Ident(sym, is_raw), span).into(); + } + TokenTree::Literal(self::Literal { + lit: token::Lit { kind: token::Integer, symbol, suffix }, + span, + }) if symbol.as_str().starts_with("-") => { + let minus = BinOp(BinOpToken::Minus); + let symbol = Symbol::intern(&symbol.as_str()[1..]); + let integer = TokenKind::lit(token::Integer, symbol, suffix); + let a = tokenstream::TokenTree::token(minus, span); + let b = tokenstream::TokenTree::token(integer, span); + return vec![a, b].into_iter().collect(); + } + TokenTree::Literal(self::Literal { + lit: token::Lit { kind: token::Float, symbol, suffix }, + span, + }) if symbol.as_str().starts_with("-") => { + let minus = BinOp(BinOpToken::Minus); + let symbol = Symbol::intern(&symbol.as_str()[1..]); + let float = TokenKind::lit(token::Float, symbol, suffix); + let a = tokenstream::TokenTree::token(minus, span); + let b = tokenstream::TokenTree::token(float, span); + return vec![a, b].into_iter().collect(); + } + TokenTree::Literal(self::Literal { lit, span }) => { + return tokenstream::TokenTree::token(Literal(lit), span).into() + } + }; + + let kind = match ch { + '=' => Eq, + '<' => Lt, + '>' => Gt, + '!' => Not, + '~' => Tilde, + '+' => BinOp(Plus), + '-' => BinOp(Minus), + '*' => BinOp(Star), + '/' => BinOp(Slash), + '%' => BinOp(Percent), + '^' => BinOp(Caret), + '&' => BinOp(And), + '|' => BinOp(Or), + '@' => At, + '.' => Dot, + ',' => Comma, + ';' => Semi, + ':' => Colon, + '#' => Pound, + '$' => Dollar, + '?' => Question, + '\'' => SingleQuote, + _ => unreachable!(), + }; + + let tree = tokenstream::TokenTree::token(kind, span); + TokenStream::new(vec![(tree, if joint { Joint } else { NonJoint })]) + } +} + +impl ToInternal<errors::Level> for Level { + fn to_internal(self) -> errors::Level { + match self { + Level::Error => errors::Level::Error, + Level::Warning => errors::Level::Warning, + Level::Note => errors::Level::Note, + Level::Help => errors::Level::Help, + _ => unreachable!("unknown proc_macro::Level variant: {:?}", self), + } + } +} + +#[derive(Clone)] +pub struct TokenStreamIter { + cursor: tokenstream::Cursor, + stack: Vec<TokenTree<Group, Punct, Ident, Literal>>, +} + +#[derive(Clone)] +pub struct Group { + delimiter: Delimiter, + stream: TokenStream, + span: DelimSpan, +} + +#[derive(Copy, Clone, PartialEq, Eq, Hash)] +pub struct Punct { + ch: char, + // NB. not using `Spacing` here because it doesn't implement `Hash`. + joint: bool, + span: Span, +} + +impl Punct { + fn new(ch: char, joint: bool, span: Span) -> Punct { + const LEGAL_CHARS: &[char] = &['=', '<', '>', '!', '~', '+', '-', '*', '/', '%', '^', + '&', '|', '@', '.', ',', ';', ':', '#', '$', '?', '\'']; + if !LEGAL_CHARS.contains(&ch) { + panic!("unsupported character `{:?}`", ch) + } + Punct { ch, joint, span } + } +} + +#[derive(Copy, Clone, PartialEq, Eq, Hash)] +pub struct Ident { + sym: Symbol, + is_raw: bool, + span: Span, +} + +impl Ident { + fn is_valid(string: &str) -> bool { + let mut chars = string.chars(); + if let Some(start) = chars.next() { + rustc_lexer::is_id_start(start) && chars.all(rustc_lexer::is_id_continue) + } else { + false + } + } + fn new(sym: Symbol, is_raw: bool, span: Span) -> Ident { + let string = sym.as_str(); + if !Self::is_valid(&string) { + panic!("`{:?}` is not a valid identifier", string) + } + if is_raw && !sym.can_be_raw() { + panic!("`{}` cannot be a raw identifier", string); + } + Ident { sym, is_raw, span } + } + fn dollar_crate(span: Span) -> Ident { + // `$crate` is accepted as an ident only if it comes from the compiler. + Ident { sym: kw::DollarCrate, is_raw: false, span } + } +} + +// FIXME(eddyb) `Literal` should not expose internal `Debug` impls. +#[derive(Clone, Debug)] +pub struct Literal { + lit: token::Lit, + span: Span, +} + +pub(crate) struct Rustc<'a> { + sess: &'a ParseSess, + def_site: Span, + call_site: Span, + mixed_site: Span, +} + +impl<'a> Rustc<'a> { + pub fn new(cx: &'a ExtCtxt<'_>) -> Self { + let expn_data = cx.current_expansion.id.expn_data(); + Rustc { + sess: cx.parse_sess, + def_site: cx.with_def_site_ctxt(expn_data.def_site), + call_site: cx.with_call_site_ctxt(expn_data.call_site), + mixed_site: cx.with_mixed_site_ctxt(expn_data.call_site), + } + } + + fn lit(&mut self, kind: token::LitKind, symbol: Symbol, suffix: Option<Symbol>) -> Literal { + Literal { + lit: token::Lit::new(kind, symbol, suffix), + span: server::Span::call_site(self), + } + } +} + +impl server::Types for Rustc<'_> { + type TokenStream = TokenStream; + type TokenStreamBuilder = tokenstream::TokenStreamBuilder; + type TokenStreamIter = TokenStreamIter; + type Group = Group; + type Punct = Punct; + type Ident = Ident; + type Literal = Literal; + type SourceFile = Lrc<SourceFile>; + type MultiSpan = Vec<Span>; + type Diagnostic = Diagnostic; + type Span = Span; +} + +impl server::TokenStream for Rustc<'_> { + fn new(&mut self) -> Self::TokenStream { + TokenStream::default() + } + fn is_empty(&mut self, stream: &Self::TokenStream) -> bool { + stream.is_empty() + } + fn from_str(&mut self, src: &str) -> Self::TokenStream { + parse::parse_stream_from_source_str( + FileName::proc_macro_source_code(src), + src.to_string(), + self.sess, + Some(self.call_site), + ) + } + fn to_string(&mut self, stream: &Self::TokenStream) -> String { + pprust::tts_to_string(stream.clone()) + } + fn from_token_tree( + &mut self, + tree: TokenTree<Self::Group, Self::Punct, Self::Ident, Self::Literal>, + ) -> Self::TokenStream { + tree.to_internal() + } + fn into_iter(&mut self, stream: Self::TokenStream) -> Self::TokenStreamIter { + TokenStreamIter { + cursor: stream.trees(), + stack: vec![], + } + } +} + +impl server::TokenStreamBuilder for Rustc<'_> { + fn new(&mut self) -> Self::TokenStreamBuilder { + tokenstream::TokenStreamBuilder::new() + } + fn push(&mut self, builder: &mut Self::TokenStreamBuilder, stream: Self::TokenStream) { + builder.push(stream); + } + fn build(&mut self, builder: Self::TokenStreamBuilder) -> Self::TokenStream { + builder.build() + } +} + +impl server::TokenStreamIter for Rustc<'_> { + fn next( + &mut self, + iter: &mut Self::TokenStreamIter, + ) -> Option<TokenTree<Self::Group, Self::Punct, Self::Ident, Self::Literal>> { + loop { + let tree = iter.stack.pop().or_else(|| { + let next = iter.cursor.next_with_joint()?; + Some(TokenTree::from_internal((next, self.sess, &mut iter.stack))) + })?; + // HACK: The condition "dummy span + group with empty delimiter" represents an AST + // fragment approximately converted into a token stream. This may happen, for + // example, with inputs to proc macro attributes, including derives. Such "groups" + // need to flattened during iteration over stream's token trees. + // Eventually this needs to be removed in favor of keeping original token trees + // and not doing the roundtrip through AST. + if let TokenTree::Group(ref group) = tree { + if group.delimiter == Delimiter::None && group.span.entire().is_dummy() { + iter.cursor.append(group.stream.clone()); + continue; + } + } + return Some(tree); + } + } +} + +impl server::Group for Rustc<'_> { + fn new(&mut self, delimiter: Delimiter, stream: Self::TokenStream) -> Self::Group { + Group { + delimiter, + stream, + span: DelimSpan::from_single(server::Span::call_site(self)), + } + } + fn delimiter(&mut self, group: &Self::Group) -> Delimiter { + group.delimiter + } + fn stream(&mut self, group: &Self::Group) -> Self::TokenStream { + group.stream.clone() + } + fn span(&mut self, group: &Self::Group) -> Self::Span { + group.span.entire() + } + fn span_open(&mut self, group: &Self::Group) -> Self::Span { + group.span.open + } + fn span_close(&mut self, group: &Self::Group) -> Self::Span { + group.span.close + } + fn set_span(&mut self, group: &mut Self::Group, span: Self::Span) { + group.span = DelimSpan::from_single(span); + } +} + +impl server::Punct for Rustc<'_> { + fn new(&mut self, ch: char, spacing: Spacing) -> Self::Punct { + Punct::new(ch, spacing == Spacing::Joint, server::Span::call_site(self)) + } + fn as_char(&mut self, punct: Self::Punct) -> char { + punct.ch + } + fn spacing(&mut self, punct: Self::Punct) -> Spacing { + if punct.joint { + Spacing::Joint + } else { + Spacing::Alone + } + } + fn span(&mut self, punct: Self::Punct) -> Self::Span { + punct.span + } + fn with_span(&mut self, punct: Self::Punct, span: Self::Span) -> Self::Punct { + Punct { span, ..punct } + } +} + +impl server::Ident for Rustc<'_> { + fn new(&mut self, string: &str, span: Self::Span, is_raw: bool) -> Self::Ident { + Ident::new(Symbol::intern(string), is_raw, span) + } + fn span(&mut self, ident: Self::Ident) -> Self::Span { + ident.span + } + fn with_span(&mut self, ident: Self::Ident, span: Self::Span) -> Self::Ident { + Ident { span, ..ident } + } +} + +impl server::Literal for Rustc<'_> { + // FIXME(eddyb) `Literal` should not expose internal `Debug` impls. + fn debug(&mut self, literal: &Self::Literal) -> String { + format!("{:?}", literal) + } + fn integer(&mut self, n: &str) -> Self::Literal { + self.lit(token::Integer, Symbol::intern(n), None) + } + fn typed_integer(&mut self, n: &str, kind: &str) -> Self::Literal { + self.lit(token::Integer, Symbol::intern(n), Some(Symbol::intern(kind))) + } + fn float(&mut self, n: &str) -> Self::Literal { + self.lit(token::Float, Symbol::intern(n), None) + } + fn f32(&mut self, n: &str) -> Self::Literal { + self.lit(token::Float, Symbol::intern(n), Some(sym::f32)) + } + fn f64(&mut self, n: &str) -> Self::Literal { + self.lit(token::Float, Symbol::intern(n), Some(sym::f64)) + } + fn string(&mut self, string: &str) -> Self::Literal { + let mut escaped = String::new(); + for ch in string.chars() { + escaped.extend(ch.escape_debug()); + } + self.lit(token::Str, Symbol::intern(&escaped), None) + } + fn character(&mut self, ch: char) -> Self::Literal { + let mut escaped = String::new(); + escaped.extend(ch.escape_unicode()); + self.lit(token::Char, Symbol::intern(&escaped), None) + } + fn byte_string(&mut self, bytes: &[u8]) -> Self::Literal { + let string = bytes + .iter() + .cloned() + .flat_map(ascii::escape_default) + .map(Into::<char>::into) + .collect::<String>(); + self.lit(token::ByteStr, Symbol::intern(&string), None) + } + fn span(&mut self, literal: &Self::Literal) -> Self::Span { + literal.span + } + fn set_span(&mut self, literal: &mut Self::Literal, span: Self::Span) { + literal.span = span; + } + fn subspan( + &mut self, + literal: &Self::Literal, + start: Bound<usize>, + end: Bound<usize>, + ) -> Option<Self::Span> { + let span = literal.span; + let length = span.hi().to_usize() - span.lo().to_usize(); + + let start = match start { + Bound::Included(lo) => lo, + Bound::Excluded(lo) => lo + 1, + Bound::Unbounded => 0, + }; + + let end = match end { + Bound::Included(hi) => hi + 1, + Bound::Excluded(hi) => hi, + Bound::Unbounded => length, + }; + + // Bounds check the values, preventing addition overflow and OOB spans. + if start > u32::max_value() as usize + || end > u32::max_value() as usize + || (u32::max_value() - start as u32) < span.lo().to_u32() + || (u32::max_value() - end as u32) < span.lo().to_u32() + || start >= end + || end > length + { + return None; + } + + let new_lo = span.lo() + BytePos::from_usize(start); + let new_hi = span.lo() + BytePos::from_usize(end); + Some(span.with_lo(new_lo).with_hi(new_hi)) + } +} + +impl server::SourceFile for Rustc<'_> { + fn eq(&mut self, file1: &Self::SourceFile, file2: &Self::SourceFile) -> bool { + Lrc::ptr_eq(file1, file2) + } + fn path(&mut self, file: &Self::SourceFile) -> String { + match file.name { + FileName::Real(ref path) => path + .to_str() + .expect("non-UTF8 file path in `proc_macro::SourceFile::path`") + .to_string(), + _ => file.name.to_string(), + } + } + fn is_real(&mut self, file: &Self::SourceFile) -> bool { + file.is_real_file() + } +} + +impl server::MultiSpan for Rustc<'_> { + fn new(&mut self) -> Self::MultiSpan { + vec![] + } + fn push(&mut self, spans: &mut Self::MultiSpan, span: Self::Span) { + spans.push(span) + } +} + +impl server::Diagnostic for Rustc<'_> { + fn new(&mut self, level: Level, msg: &str, spans: Self::MultiSpan) -> Self::Diagnostic { + let mut diag = Diagnostic::new(level.to_internal(), msg); + diag.set_span(MultiSpan::from_spans(spans)); + diag + } + fn sub( + &mut self, + diag: &mut Self::Diagnostic, + level: Level, + msg: &str, + spans: Self::MultiSpan, + ) { + diag.sub(level.to_internal(), msg, MultiSpan::from_spans(spans), None); + } + fn emit(&mut self, diag: Self::Diagnostic) { + self.sess.span_diagnostic.emit_diagnostic(&diag); + } +} + +impl server::Span for Rustc<'_> { + fn debug(&mut self, span: Self::Span) -> String { + format!("{:?} bytes({}..{})", span.ctxt(), span.lo().0, span.hi().0) + } + fn def_site(&mut self) -> Self::Span { + self.def_site + } + fn call_site(&mut self) -> Self::Span { + self.call_site + } + fn mixed_site(&mut self) -> Self::Span { + self.mixed_site + } + fn source_file(&mut self, span: Self::Span) -> Self::SourceFile { + self.sess.source_map().lookup_char_pos(span.lo()).file + } + fn parent(&mut self, span: Self::Span) -> Option<Self::Span> { + span.parent() + } + fn source(&mut self, span: Self::Span) -> Self::Span { + span.source_callsite() + } + fn start(&mut self, span: Self::Span) -> LineColumn { + let loc = self.sess.source_map().lookup_char_pos(span.lo()); + LineColumn { + line: loc.line, + column: loc.col.to_usize(), + } + } + fn end(&mut self, span: Self::Span) -> LineColumn { + let loc = self.sess.source_map().lookup_char_pos(span.hi()); + LineColumn { + line: loc.line, + column: loc.col.to_usize(), + } + } + fn join(&mut self, first: Self::Span, second: Self::Span) -> Option<Self::Span> { + let self_loc = self.sess.source_map().lookup_char_pos(first.lo()); + let other_loc = self.sess.source_map().lookup_char_pos(second.lo()); + + if self_loc.file.name != other_loc.file.name { + return None; + } + + Some(first.to(second)) + } + fn resolved_at(&mut self, span: Self::Span, at: Self::Span) -> Self::Span { + span.with_ctxt(at.ctxt()) + } + fn source_text(&mut self, span: Self::Span) -> Option<String> { + self.sess.source_map().span_to_snippet(span).ok() + } +} |