From 5c93492da914f972fb549e95db778adcbdf70480 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Thu, 10 Oct 2019 07:29:02 +1100 Subject: Remove the `Option` in `TokenStream`. It means an allocation is required to create an empty `TokenStream`, but all other operations are simpler and marginally faster due to not having to check for `None`. Overall it simplifies the code for a negligible performance effect. The commit also removes `TokenStream::empty` by implementing `Default`, which is now possible. --- src/libsyntax_ext/plugin_macro_defs.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libsyntax_ext/plugin_macro_defs.rs') diff --git a/src/libsyntax_ext/plugin_macro_defs.rs b/src/libsyntax_ext/plugin_macro_defs.rs index 315babceae3..62c7e188eba 100644 --- a/src/libsyntax_ext/plugin_macro_defs.rs +++ b/src/libsyntax_ext/plugin_macro_defs.rs @@ -20,7 +20,7 @@ fn plugin_macro_def(name: Name, span: Span) -> P { attr::mk_word_item(Ident::new(sym::rustc_builtin_macro, span))); let parens: TreeAndJoint = TokenTree::Delimited( - DelimSpan::from_single(span), token::Paren, TokenStream::empty() + DelimSpan::from_single(span), token::Paren, TokenStream::default() ).into(); let trees = vec![parens.clone(), TokenTree::token(token::FatArrow, span).into(), parens]; -- cgit 1.4.1-3-g733a5 From d420d719c4c44c3c6d02b5fafba4f2cf5e837dba Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Wed, 16 Oct 2019 10:59:30 +0200 Subject: move syntax::ext to new crate syntax_expand --- Cargo.lock | 27 + src/librustc/Cargo.toml | 1 + src/librustc/hir/def.rs | 2 +- src/librustc/hir/lowering.rs | 6 +- src/librustc/hir/lowering/item.rs | 2 +- src/librustc/hir/map/def_collector.rs | 2 +- src/librustc/hir/map/definitions.rs | 2 +- src/librustc/hir/map/mod.rs | 2 +- src/librustc/ich/hcx.rs | 2 +- src/librustc/ich/impls_syntax.rs | 2 +- src/librustc/lint/mod.rs | 2 +- src/librustc/session/mod.rs | 2 +- src/librustc/ty/mod.rs | 2 +- src/librustc_codegen_llvm/allocator.rs | 2 +- src/librustc_codegen_llvm/lib.rs | 3 +- src/librustc_codegen_ssa/Cargo.toml | 1 + src/librustc_codegen_ssa/back/symbol_export.rs | 2 +- src/librustc_codegen_ssa/back/write.rs | 2 +- src/librustc_codegen_ssa/traits/backend.rs | 2 +- src/librustc_interface/Cargo.toml | 1 + src/librustc_interface/passes.rs | 8 +- src/librustc_metadata/Cargo.toml | 1 + src/librustc_metadata/creader.rs | 2 +- src/librustc_metadata/cstore.rs | 2 +- src/librustc_metadata/decoder.rs | 4 +- src/librustc_metadata/encoder.rs | 2 +- src/librustc_passes/Cargo.toml | 1 + src/librustc_passes/ast_validation.rs | 2 +- src/librustc_plugin/Cargo.toml | 1 + src/librustc_plugin/lib.rs | 2 +- src/librustc_plugin/registry.rs | 4 +- src/librustc_resolve/Cargo.toml | 1 + src/librustc_resolve/build_reduced_graph.rs | 6 +- src/librustc_resolve/diagnostics.rs | 2 +- src/librustc_resolve/late/diagnostics.rs | 2 +- src/librustc_resolve/lib.rs | 14 +- src/librustc_resolve/macros.rs | 10 +- src/librustc_resolve/resolve_imports.rs | 2 +- src/librustdoc/clean/inline.rs | 2 +- src/librustdoc/clean/mod.rs | 2 +- src/librustdoc/doctree.rs | 2 +- src/librustdoc/html/item_type.rs | 2 +- src/librustdoc/html/render.rs | 4 +- src/librustdoc/lib.rs | 1 + src/librustdoc/passes/collect_intra_doc_links.rs | 2 +- src/librustdoc/visit_ast.rs | 2 +- src/libsyntax/attr/mod.rs | 4 +- src/libsyntax/config.rs | 1 + src/libsyntax/ext/allocator.rs | 75 -- src/libsyntax/ext/base.rs | 1188 ----------------- src/libsyntax/ext/build.rs | 639 --------- src/libsyntax/ext/expand.rs | 1543 --------------------- src/libsyntax/ext/mbe.rs | 166 --- src/libsyntax/ext/mbe/macro_check.rs | 626 --------- src/libsyntax/ext/mbe/macro_parser.rs | 943 ------------- src/libsyntax/ext/mbe/macro_rules.rs | 1194 ----------------- src/libsyntax/ext/mbe/quoted.rs | 263 ---- src/libsyntax/ext/mbe/transcribe.rs | 398 ------ src/libsyntax/ext/placeholders.rs | 349 ----- src/libsyntax/ext/proc_macro.rs | 219 --- src/libsyntax/ext/proc_macro_server.rs | 712 ---------- src/libsyntax/feature_gate/check.rs | 2 +- src/libsyntax/feature_gate/mod.rs | 3 +- src/libsyntax/lib.rs | 41 +- src/libsyntax/parse/literal.rs | 2 +- src/libsyntax/parse/parser.rs | 14 +- src/libsyntax/parse/parser/diagnostics.rs | 6 +- src/libsyntax/parse/parser/expr.rs | 2 +- src/libsyntax/parse/parser/item.rs | 6 +- src/libsyntax/parse/parser/pat.rs | 2 +- src/libsyntax/parse/parser/path.rs | 4 +- src/libsyntax/parse/parser/stmt.rs | 6 +- src/libsyntax/parse/parser/ty.rs | 4 +- src/libsyntax/parse/token.rs | 14 +- src/libsyntax/tokenstream.rs | 2 +- src/libsyntax_expand/Cargo.toml | 26 + src/libsyntax_expand/allocator.rs | 75 ++ src/libsyntax_expand/base.rs | 1189 +++++++++++++++++ src/libsyntax_expand/build.rs | 640 +++++++++ src/libsyntax_expand/expand.rs | 1551 ++++++++++++++++++++++ src/libsyntax_expand/lib.rs | 38 + src/libsyntax_expand/mbe.rs | 166 +++ src/libsyntax_expand/mbe/macro_check.rs | 626 +++++++++ src/libsyntax_expand/mbe/macro_parser.rs | 944 +++++++++++++ src/libsyntax_expand/mbe/macro_rules.rs | 1192 +++++++++++++++++ src/libsyntax_expand/mbe/quoted.rs | 264 ++++ src/libsyntax_expand/mbe/transcribe.rs | 399 ++++++ src/libsyntax_expand/placeholders.rs | 350 +++++ src/libsyntax_expand/proc_macro.rs | 215 +++ src/libsyntax_expand/proc_macro_server.rs | 713 ++++++++++ src/libsyntax_ext/Cargo.toml | 1 + src/libsyntax_ext/asm.rs | 2 +- src/libsyntax_ext/assert.rs | 2 +- src/libsyntax_ext/cfg.rs | 2 +- src/libsyntax_ext/cmdline_attrs.rs | 2 +- src/libsyntax_ext/compile_error.rs | 2 +- src/libsyntax_ext/concat.rs | 2 +- src/libsyntax_ext/concat_idents.rs | 2 +- src/libsyntax_ext/deriving/bounds.rs | 2 +- src/libsyntax_ext/deriving/clone.rs | 2 +- src/libsyntax_ext/deriving/cmp/eq.rs | 2 +- src/libsyntax_ext/deriving/cmp/ord.rs | 2 +- src/libsyntax_ext/deriving/cmp/partial_eq.rs | 2 +- src/libsyntax_ext/deriving/cmp/partial_ord.rs | 2 +- src/libsyntax_ext/deriving/debug.rs | 2 +- src/libsyntax_ext/deriving/decodable.rs | 2 +- src/libsyntax_ext/deriving/default.rs | 2 +- src/libsyntax_ext/deriving/encodable.rs | 2 +- src/libsyntax_ext/deriving/generic/mod.rs | 2 +- src/libsyntax_ext/deriving/generic/ty.rs | 2 +- src/libsyntax_ext/deriving/hash.rs | 2 +- src/libsyntax_ext/deriving/mod.rs | 2 +- src/libsyntax_ext/env.rs | 2 +- src/libsyntax_ext/format.rs | 2 +- src/libsyntax_ext/global_allocator.rs | 4 +- src/libsyntax_ext/global_asm.rs | 2 +- src/libsyntax_ext/lib.rs | 6 +- src/libsyntax_ext/log_syntax.rs | 2 +- src/libsyntax_ext/plugin_macro_defs.rs | 2 +- src/libsyntax_ext/proc_macro_harness.rs | 8 +- src/libsyntax_ext/source_util.rs | 5 +- src/libsyntax_ext/standard_library_imports.rs | 6 +- src/libsyntax_ext/test.rs | 2 +- src/libsyntax_ext/test_harness.rs | 4 +- src/libsyntax_ext/trace_macros.rs | 2 +- src/libsyntax_ext/util.rs | 2 +- 126 files changed, 8564 insertions(+), 8494 deletions(-) delete mode 100644 src/libsyntax/ext/allocator.rs delete mode 100644 src/libsyntax/ext/base.rs delete mode 100644 src/libsyntax/ext/build.rs delete mode 100644 src/libsyntax/ext/expand.rs delete mode 100644 src/libsyntax/ext/mbe.rs delete mode 100644 src/libsyntax/ext/mbe/macro_check.rs delete mode 100644 src/libsyntax/ext/mbe/macro_parser.rs delete mode 100644 src/libsyntax/ext/mbe/macro_rules.rs delete mode 100644 src/libsyntax/ext/mbe/quoted.rs delete mode 100644 src/libsyntax/ext/mbe/transcribe.rs delete mode 100644 src/libsyntax/ext/placeholders.rs delete mode 100644 src/libsyntax/ext/proc_macro.rs delete mode 100644 src/libsyntax/ext/proc_macro_server.rs create mode 100644 src/libsyntax_expand/Cargo.toml create mode 100644 src/libsyntax_expand/allocator.rs create mode 100644 src/libsyntax_expand/base.rs create mode 100644 src/libsyntax_expand/build.rs create mode 100644 src/libsyntax_expand/expand.rs create mode 100644 src/libsyntax_expand/lib.rs create mode 100644 src/libsyntax_expand/mbe.rs create mode 100644 src/libsyntax_expand/mbe/macro_check.rs create mode 100644 src/libsyntax_expand/mbe/macro_parser.rs create mode 100644 src/libsyntax_expand/mbe/macro_rules.rs create mode 100644 src/libsyntax_expand/mbe/quoted.rs create mode 100644 src/libsyntax_expand/mbe/transcribe.rs create mode 100644 src/libsyntax_expand/placeholders.rs create mode 100644 src/libsyntax_expand/proc_macro.rs create mode 100644 src/libsyntax_expand/proc_macro_server.rs (limited to 'src/libsyntax_ext/plugin_macro_defs.rs') diff --git a/Cargo.lock b/Cargo.lock index 6d40b198927..a5a7521abde 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3112,6 +3112,7 @@ dependencies = [ "serialize", "smallvec", "syntax", + "syntax_expand", "syntax_pos", ] @@ -3427,6 +3428,7 @@ dependencies = [ "rustc_target", "serialize", "syntax", + "syntax_expand", "syntax_pos", "tempfile", ] @@ -3559,6 +3561,7 @@ dependencies = [ "serialize", "smallvec", "syntax", + "syntax_expand", "syntax_ext", "syntax_pos", "tempfile", @@ -3630,6 +3633,7 @@ dependencies = [ "smallvec", "stable_deref_trait", "syntax", + "syntax_expand", "syntax_pos", ] @@ -3678,6 +3682,7 @@ dependencies = [ "rustc_index", "rustc_target", "syntax", + "syntax_expand", "syntax_pos", ] @@ -3695,6 +3700,7 @@ dependencies = [ "rustc", "rustc_metadata", "syntax", + "syntax_expand", "syntax_pos", ] @@ -3723,6 +3729,7 @@ dependencies = [ "rustc_metadata", "smallvec", "syntax", + "syntax_expand", "syntax_pos", ] @@ -4336,6 +4343,25 @@ dependencies = [ "syntax_pos", ] +[[package]] +name = "syntax_expand" +version = "0.0.0" +dependencies = [ + "bitflags", + "lazy_static 1.3.0", + "log", + "rustc_data_structures", + "rustc_errors", + "rustc_index", + "rustc_lexer", + "rustc_target", + "scoped-tls", + "serialize", + "smallvec", + "syntax", + "syntax_pos", +] + [[package]] name = "syntax_ext" version = "0.0.0" @@ -4347,6 +4373,7 @@ dependencies = [ "rustc_target", "smallvec", "syntax", + "syntax_expand", "syntax_pos", ] diff --git a/src/librustc/Cargo.toml b/src/librustc/Cargo.toml index 9b3609eca3e..cf9f36ca37c 100644 --- a/src/librustc/Cargo.toml +++ b/src/librustc/Cargo.toml @@ -29,6 +29,7 @@ rustc_index = { path = "../librustc_index" } errors = { path = "../librustc_errors", package = "rustc_errors" } rustc_serialize = { path = "../libserialize", package = "serialize" } syntax = { path = "../libsyntax" } +syntax_expand = { path = "../libsyntax_expand" } syntax_pos = { path = "../libsyntax_pos" } backtrace = "0.3.3" parking_lot = "0.9" diff --git a/src/librustc/hir/def.rs b/src/librustc/hir/def.rs index f7d31ca06ee..a071a539e01 100644 --- a/src/librustc/hir/def.rs +++ b/src/librustc/hir/def.rs @@ -6,7 +6,7 @@ use crate::ty; use crate::util::nodemap::DefIdMap; use syntax::ast; -use syntax::ext::base::MacroKind; +use syntax_expand::base::MacroKind; use syntax::ast::NodeId; use syntax_pos::Span; use rustc_macros::HashStable; diff --git a/src/librustc/hir/lowering.rs b/src/librustc/hir/lowering.rs index e9788a55812..d5287fd415b 100644 --- a/src/librustc/hir/lowering.rs +++ b/src/librustc/hir/lowering.rs @@ -64,15 +64,15 @@ use syntax::ast; use syntax::ptr::P as AstP; use syntax::ast::*; use syntax::errors; -use syntax::ext::base::SpecialDerives; -use syntax::ext::hygiene::ExpnId; +use syntax_expand::base::SpecialDerives; use syntax::print::pprust; -use syntax::tokenstream::{TokenStream, TokenTree}; use syntax::parse::token::{self, Nonterminal, Token}; +use syntax::tokenstream::{TokenStream, TokenTree}; use syntax::sess::ParseSess; use syntax::source_map::{respan, ExpnData, ExpnKind, DesugaringKind, Spanned}; use syntax::symbol::{kw, sym, Symbol}; use syntax::visit::{self, Visitor}; +use syntax_pos::hygiene::ExpnId; use syntax_pos::Span; const HIR_ID_COUNTER_LOCKED: u32 = 0xFFFFFFFF; diff --git a/src/librustc/hir/lowering/item.rs b/src/librustc/hir/lowering/item.rs index 548a2fedfff..73d2ac5c134 100644 --- a/src/librustc/hir/lowering/item.rs +++ b/src/librustc/hir/lowering/item.rs @@ -18,7 +18,7 @@ use smallvec::SmallVec; use syntax::attr; use syntax::ast::*; use syntax::visit::{self, Visitor}; -use syntax::ext::base::SpecialDerives; +use syntax_expand::base::SpecialDerives; use syntax::source_map::{respan, DesugaringKind, Spanned}; use syntax::symbol::{kw, sym}; use syntax_pos::Span; diff --git a/src/librustc/hir/map/def_collector.rs b/src/librustc/hir/map/def_collector.rs index 1997e2aab35..fbef95fec7d 100644 --- a/src/librustc/hir/map/def_collector.rs +++ b/src/librustc/hir/map/def_collector.rs @@ -2,7 +2,7 @@ use crate::hir::map::definitions::*; use crate::hir::def_id::DefIndex; use syntax::ast::*; -use syntax::ext::hygiene::ExpnId; +use syntax_expand::hygiene::ExpnId; use syntax::visit; use syntax::symbol::{kw, sym}; use syntax::parse::token::{self, Token}; diff --git a/src/librustc/hir/map/definitions.rs b/src/librustc/hir/map/definitions.rs index 71bf230e37d..cc099fcc40f 100644 --- a/src/librustc/hir/map/definitions.rs +++ b/src/librustc/hir/map/definitions.rs @@ -17,7 +17,7 @@ use std::borrow::Borrow; use std::fmt::Write; use std::hash::Hash; use syntax::ast; -use syntax::ext::hygiene::ExpnId; +use syntax_expand::hygiene::ExpnId; use syntax::symbol::{Symbol, sym, InternedString}; use syntax_pos::{Span, DUMMY_SP}; diff --git a/src/librustc/hir/map/mod.rs b/src/librustc/hir/map/mod.rs index 16c4ab7187d..f839087ec02 100644 --- a/src/librustc/hir/map/mod.rs +++ b/src/librustc/hir/map/mod.rs @@ -20,7 +20,7 @@ use rustc_data_structures::svh::Svh; use rustc_index::vec::IndexVec; use syntax::ast::{self, Name, NodeId}; use syntax::source_map::Spanned; -use syntax::ext::base::MacroKind; +use syntax_expand::base::MacroKind; use syntax_pos::{Span, DUMMY_SP}; pub mod blocks; diff --git a/src/librustc/ich/hcx.rs b/src/librustc/ich/hcx.rs index 3e6b271b834..14d0673ecc0 100644 --- a/src/librustc/ich/hcx.rs +++ b/src/librustc/ich/hcx.rs @@ -13,7 +13,7 @@ use std::cell::RefCell; use syntax::ast; use syntax::source_map::SourceMap; -use syntax::ext::hygiene::SyntaxContext; +use syntax_expand::hygiene::SyntaxContext; use syntax::symbol::Symbol; use syntax::tokenstream::DelimSpan; use syntax_pos::{Span, DUMMY_SP}; diff --git a/src/librustc/ich/impls_syntax.rs b/src/librustc/ich/impls_syntax.rs index 23a2f115e05..defc3fb25bc 100644 --- a/src/librustc/ich/impls_syntax.rs +++ b/src/librustc/ich/impls_syntax.rs @@ -59,7 +59,7 @@ impl_stable_hash_for!(enum ::syntax::ast::AsmDialect { Intel }); -impl_stable_hash_for!(enum ::syntax::ext::base::MacroKind { +impl_stable_hash_for!(enum ::syntax_expand::base::MacroKind { Bang, Attr, Derive, diff --git a/src/librustc/lint/mod.rs b/src/librustc/lint/mod.rs index b31efc24e52..7443cca822a 100644 --- a/src/librustc/lint/mod.rs +++ b/src/librustc/lint/mod.rs @@ -39,7 +39,7 @@ use syntax::ast; use syntax::source_map::{MultiSpan, ExpnKind, DesugaringKind}; use syntax::early_buffered_lints::BufferedEarlyLintId; use syntax::edition::Edition; -use syntax::ext::base::MacroKind; +use syntax_expand::base::MacroKind; use syntax::symbol::{Symbol, sym}; use syntax_pos::Span; diff --git a/src/librustc/session/mod.rs b/src/librustc/session/mod.rs index c59df146629..b65bf2230b3 100644 --- a/src/librustc/session/mod.rs +++ b/src/librustc/session/mod.rs @@ -24,7 +24,7 @@ use errors::emitter::HumanReadableErrorType; use errors::annotate_snippet_emitter_writer::{AnnotateSnippetEmitterWriter}; use syntax::ast::{self, NodeId}; use syntax::edition::Edition; -use syntax::ext::allocator::AllocatorKind; +use syntax_expand::allocator::AllocatorKind; use syntax::feature_gate::{self, AttributeType}; use syntax::json::JsonEmitter; use syntax::source_map; diff --git a/src/librustc/ty/mod.rs b/src/librustc/ty/mod.rs index 65aea7b459f..ab9fb1df682 100644 --- a/src/librustc/ty/mod.rs +++ b/src/librustc/ty/mod.rs @@ -45,7 +45,7 @@ use std::{mem, ptr}; use std::ops::Range; use syntax::ast::{self, Name, Ident, NodeId}; use syntax::attr; -use syntax::ext::hygiene::ExpnId; +use syntax_expand::hygiene::ExpnId; use syntax::symbol::{kw, sym, Symbol, InternedString}; use syntax_pos::Span; diff --git a/src/librustc_codegen_llvm/allocator.rs b/src/librustc_codegen_llvm/allocator.rs index f31765cea4f..8c60c030eac 100644 --- a/src/librustc_codegen_llvm/allocator.rs +++ b/src/librustc_codegen_llvm/allocator.rs @@ -3,7 +3,7 @@ use std::ffi::CString; use crate::attributes; use libc::c_uint; use rustc::ty::TyCtxt; -use syntax::ext::allocator::{AllocatorKind, AllocatorTy, ALLOCATOR_METHODS}; +use syntax_expand::allocator::{AllocatorKind, AllocatorTy, ALLOCATOR_METHODS}; use crate::ModuleLlvm; use crate::llvm::{self, False, True}; diff --git a/src/librustc_codegen_llvm/lib.rs b/src/librustc_codegen_llvm/lib.rs index fde04a68457..9b55bef0c51 100644 --- a/src/librustc_codegen_llvm/lib.rs +++ b/src/librustc_codegen_llvm/lib.rs @@ -39,6 +39,7 @@ extern crate rustc_driver as _; #[macro_use] extern crate log; extern crate syntax; +extern crate syntax_expand; extern crate syntax_pos; extern crate rustc_errors as errors; @@ -48,7 +49,7 @@ use rustc_codegen_ssa::back::lto::{SerializedModule, LtoModuleCodegen, ThinModul use rustc_codegen_ssa::CompiledModule; use errors::{FatalError, Handler}; use rustc::dep_graph::WorkProduct; -use syntax::ext::allocator::AllocatorKind; +use syntax_expand::allocator::AllocatorKind; use syntax_pos::symbol::InternedString; pub use llvm_util::target_features; use std::any::Any; diff --git a/src/librustc_codegen_ssa/Cargo.toml b/src/librustc_codegen_ssa/Cargo.toml index c7d09a423d5..6992f93d999 100644 --- a/src/librustc_codegen_ssa/Cargo.toml +++ b/src/librustc_codegen_ssa/Cargo.toml @@ -21,6 +21,7 @@ tempfile = "3.1" rustc_serialize = { path = "../libserialize", package = "serialize" } syntax = { path = "../libsyntax" } +syntax_expand = { path = "../libsyntax_expand" } syntax_pos = { path = "../libsyntax_pos" } rustc = { path = "../librustc" } rustc_apfloat = { path = "../librustc_apfloat" } diff --git a/src/librustc_codegen_ssa/back/symbol_export.rs b/src/librustc_codegen_ssa/back/symbol_export.rs index 9078f77f1f7..d866a10f069 100644 --- a/src/librustc_codegen_ssa/back/symbol_export.rs +++ b/src/librustc_codegen_ssa/back/symbol_export.rs @@ -14,7 +14,7 @@ use rustc::ty::query::Providers; use rustc::ty::subst::SubstsRef; use rustc::util::nodemap::{FxHashMap, DefIdMap}; use rustc_index::vec::IndexVec; -use syntax::ext::allocator::ALLOCATOR_METHODS; +use syntax_expand::allocator::ALLOCATOR_METHODS; pub type ExportedSymbols = FxHashMap< CrateNum, diff --git a/src/librustc_codegen_ssa/back/write.rs b/src/librustc_codegen_ssa/back/write.rs index 5339134a2a8..856c280cac0 100644 --- a/src/librustc_codegen_ssa/back/write.rs +++ b/src/librustc_codegen_ssa/back/write.rs @@ -27,7 +27,7 @@ use rustc_errors::{Handler, Level, FatalError, DiagnosticId, SourceMapperDyn}; use rustc_errors::emitter::{Emitter}; use rustc_target::spec::MergeFunctions; use syntax::attr; -use syntax::ext::hygiene::ExpnId; +use syntax_expand::hygiene::ExpnId; use syntax_pos::symbol::{Symbol, sym}; use jobserver::{Client, Acquired}; diff --git a/src/librustc_codegen_ssa/traits/backend.rs b/src/librustc_codegen_ssa/traits/backend.rs index a7faf4eaf10..7cae3e9ade5 100644 --- a/src/librustc_codegen_ssa/traits/backend.rs +++ b/src/librustc_codegen_ssa/traits/backend.rs @@ -9,7 +9,7 @@ use rustc::ty::TyCtxt; use rustc_codegen_utils::codegen_backend::CodegenBackend; use std::sync::Arc; use std::sync::mpsc; -use syntax::ext::allocator::AllocatorKind; +use syntax_expand::allocator::AllocatorKind; use syntax_pos::symbol::InternedString; pub trait BackendTypes { diff --git a/src/librustc_interface/Cargo.toml b/src/librustc_interface/Cargo.toml index bed5febb72e..7a0d931ca73 100644 --- a/src/librustc_interface/Cargo.toml +++ b/src/librustc_interface/Cargo.toml @@ -15,6 +15,7 @@ rayon = { version = "0.3.0", package = "rustc-rayon" } smallvec = { version = "0.6.7", features = ["union", "may_dangle"] } syntax = { path = "../libsyntax" } syntax_ext = { path = "../libsyntax_ext" } +syntax_expand = { path = "../libsyntax_expand" } syntax_pos = { path = "../libsyntax_pos" } rustc_serialize = { path = "../libserialize", package = "serialize" } rustc = { path = "../librustc" } diff --git a/src/librustc_interface/passes.rs b/src/librustc_interface/passes.rs index 396c5610251..89de5714695 100644 --- a/src/librustc_interface/passes.rs +++ b/src/librustc_interface/passes.rs @@ -35,7 +35,7 @@ use rustc_traits; use rustc_typeck as typeck; use syntax::{self, ast, visit}; use syntax::early_buffered_lints::BufferedEarlyLint; -use syntax::ext::base::{NamedSyntaxExtension, ExtCtxt}; +use syntax_expand::base::{NamedSyntaxExtension, ExtCtxt}; use syntax::mut_visit::MutVisitor; use syntax::parse::{self, PResult}; use syntax::util::node_count::NodeCounter; @@ -397,12 +397,12 @@ fn configure_and_expand_inner<'a>( // Create the config for macro expansion let features = sess.features_untracked(); - let cfg = syntax::ext::expand::ExpansionConfig { + let cfg = syntax_expand::expand::ExpansionConfig { features: Some(&features), recursion_limit: *sess.recursion_limit.get(), trace_mac: sess.opts.debugging_opts.trace_macros, should_test: sess.opts.test, - ..syntax::ext::expand::ExpansionConfig::default(crate_name.to_string()) + ..syntax_expand::expand::ExpansionConfig::default(crate_name.to_string()) }; let mut ecx = ExtCtxt::new(&sess.parse_sess, cfg, &mut resolver); @@ -559,7 +559,7 @@ pub fn lower_to_hir( // Discard hygiene data, which isn't required after lowering to HIR. if !sess.opts.debugging_opts.keep_hygiene_data { - syntax::ext::hygiene::clear_syntax_context_map(); + syntax_expand::hygiene::clear_syntax_context_map(); } Ok(hir_forest) diff --git a/src/librustc_metadata/Cargo.toml b/src/librustc_metadata/Cargo.toml index 032470e1400..18192e35f8a 100644 --- a/src/librustc_metadata/Cargo.toml +++ b/src/librustc_metadata/Cargo.toml @@ -22,4 +22,5 @@ rustc_index = { path = "../librustc_index" } rustc_serialize = { path = "../libserialize", package = "serialize" } stable_deref_trait = "1.0.0" syntax = { path = "../libsyntax" } +syntax_expand = { path = "../libsyntax_expand" } syntax_pos = { path = "../libsyntax_pos" } diff --git a/src/librustc_metadata/creader.rs b/src/librustc_metadata/creader.rs index 9850121d2ce..6a2da5d4988 100644 --- a/src/librustc_metadata/creader.rs +++ b/src/librustc_metadata/creader.rs @@ -26,7 +26,7 @@ use std::{cmp, fs}; use syntax::ast; use syntax::attr; -use syntax::ext::allocator::{global_allocator_spans, AllocatorKind}; +use syntax_expand::allocator::{global_allocator_spans, AllocatorKind}; use syntax::symbol::{Symbol, sym}; use syntax::{span_err, span_fatal}; use syntax_pos::{Span, DUMMY_SP}; diff --git a/src/librustc_metadata/cstore.rs b/src/librustc_metadata/cstore.rs index a5a458e49a3..9a0b98ffb73 100644 --- a/src/librustc_metadata/cstore.rs +++ b/src/librustc_metadata/cstore.rs @@ -11,7 +11,7 @@ use rustc_index::vec::IndexVec; use rustc::util::nodemap::FxHashMap; use rustc_data_structures::sync::{Lrc, RwLock, Lock, MetadataRef, AtomicCell}; use syntax::ast; -use syntax::ext::base::SyntaxExtension; +use syntax_expand::base::SyntaxExtension; use syntax_pos; use proc_macro::bridge::client::ProcMacro; diff --git a/src/librustc_metadata/decoder.rs b/src/librustc_metadata/decoder.rs index 6969d608d76..37f3726124a 100644 --- a/src/librustc_metadata/decoder.rs +++ b/src/librustc_metadata/decoder.rs @@ -32,11 +32,11 @@ use syntax::attr; use syntax::ast::{self, Ident}; use syntax::source_map::{self, respan, Spanned}; use syntax::symbol::{Symbol, sym}; -use syntax::ext::base::{MacroKind, SyntaxExtensionKind, SyntaxExtension}; +use syntax_expand::base::{MacroKind, SyntaxExtensionKind, SyntaxExtension}; use syntax_pos::{self, Span, BytePos, Pos, DUMMY_SP, symbol::{InternedString}}; use log::debug; use proc_macro::bridge::client::ProcMacro; -use syntax::ext::proc_macro::{AttrProcMacro, ProcMacroDerive, BangProcMacro}; +use syntax_expand::proc_macro::{AttrProcMacro, ProcMacroDerive, BangProcMacro}; crate struct DecodeContext<'a, 'tcx> { opaque: opaque::Decoder<'a>, diff --git a/src/librustc_metadata/encoder.rs b/src/librustc_metadata/encoder.rs index 03a14f88645..ac20284233a 100644 --- a/src/librustc_metadata/encoder.rs +++ b/src/librustc_metadata/encoder.rs @@ -31,7 +31,7 @@ use rustc_data_structures::sync::Lrc; use std::u32; use syntax::ast; use syntax::attr; -use syntax::ext::proc_macro::is_proc_macro_attr; +use syntax_expand::proc_macro::is_proc_macro_attr; use syntax::source_map::Spanned; use syntax::symbol::{kw, sym, Ident, Symbol}; use syntax_pos::{self, FileName, SourceFile, Span}; diff --git a/src/librustc_passes/Cargo.toml b/src/librustc_passes/Cargo.toml index 9d29a230314..118deb560d6 100644 --- a/src/librustc_passes/Cargo.toml +++ b/src/librustc_passes/Cargo.toml @@ -13,6 +13,7 @@ log = "0.4" rustc = { path = "../librustc" } rustc_data_structures = { path = "../librustc_data_structures" } syntax = { path = "../libsyntax" } +syntax_expand = { path = "../libsyntax_expand" } syntax_pos = { path = "../libsyntax_pos" } errors = { path = "../librustc_errors", package = "rustc_errors" } rustc_target = { path = "../librustc_target" } diff --git a/src/librustc_passes/ast_validation.rs b/src/librustc_passes/ast_validation.rs index 43c4f720ad4..74de31263d3 100644 --- a/src/librustc_passes/ast_validation.rs +++ b/src/librustc_passes/ast_validation.rs @@ -14,7 +14,7 @@ use rustc::session::Session; use rustc_data_structures::fx::FxHashMap; use syntax::ast::*; use syntax::attr; -use syntax::ext::proc_macro::is_proc_macro_attr; +use syntax_expand::proc_macro::is_proc_macro_attr; use syntax::feature_gate::is_builtin_attr; use syntax::source_map::Spanned; use syntax::symbol::{kw, sym}; diff --git a/src/librustc_plugin/Cargo.toml b/src/librustc_plugin/Cargo.toml index 3f11430dc82..e8bf4e7ea8f 100644 --- a/src/librustc_plugin/Cargo.toml +++ b/src/librustc_plugin/Cargo.toml @@ -14,4 +14,5 @@ doctest = false rustc = { path = "../librustc" } rustc_metadata = { path = "../librustc_metadata" } syntax = { path = "../libsyntax" } +syntax_expand = { path = "../libsyntax_expand" } syntax_pos = { path = "../libsyntax_pos" } diff --git a/src/librustc_plugin/lib.rs b/src/librustc_plugin/lib.rs index 4e1a47c503e..38738e20630 100644 --- a/src/librustc_plugin/lib.rs +++ b/src/librustc_plugin/lib.rs @@ -21,7 +21,7 @@ //! extern crate syntax_pos; //! //! use rustc_driver::plugin::Registry; -//! use syntax::ext::base::{ExtCtxt, MacResult}; +//! use syntax_expand::base::{ExtCtxt, MacResult}; //! use syntax_pos::Span; //! use syntax::tokenstream::TokenTree; //! diff --git a/src/librustc_plugin/registry.rs b/src/librustc_plugin/registry.rs index bb3c950edae..b826dd91198 100644 --- a/src/librustc_plugin/registry.rs +++ b/src/librustc_plugin/registry.rs @@ -4,8 +4,8 @@ use rustc::lint::{EarlyLintPassObject, LateLintPassObject, LintId, Lint}; use rustc::session::Session; use rustc::util::nodemap::FxHashMap; -use syntax::ext::base::{SyntaxExtension, SyntaxExtensionKind, NamedSyntaxExtension}; -use syntax::ext::base::MacroExpanderFn; +use syntax_expand::base::{SyntaxExtension, SyntaxExtensionKind, NamedSyntaxExtension}; +use syntax_expand::base::MacroExpanderFn; use syntax::symbol::Symbol; use syntax::ast; use syntax::feature_gate::AttributeType; diff --git a/src/librustc_resolve/Cargo.toml b/src/librustc_resolve/Cargo.toml index 936e72ef2c5..06bf3085989 100644 --- a/src/librustc_resolve/Cargo.toml +++ b/src/librustc_resolve/Cargo.toml @@ -14,6 +14,7 @@ doctest = false bitflags = "1.0" log = "0.4" syntax = { path = "../libsyntax" } +syntax_expand = { path = "../libsyntax_expand" } rustc = { path = "../librustc" } arena = { path = "../libarena" } errors = { path = "../librustc_errors", package = "rustc_errors" } diff --git a/src/librustc_resolve/build_reduced_graph.rs b/src/librustc_resolve/build_reduced_graph.rs index 0d5361fe8d8..eadae52c250 100644 --- a/src/librustc_resolve/build_reduced_graph.rs +++ b/src/librustc_resolve/build_reduced_graph.rs @@ -32,9 +32,9 @@ use syntax::attr; use syntax::ast::{self, Block, ForeignItem, ForeignItemKind, Item, ItemKind, NodeId}; use syntax::ast::{MetaItemKind, StmtKind, TraitItem, TraitItemKind}; -use syntax::ext::base::{MacroKind, SyntaxExtension}; -use syntax::ext::expand::AstFragment; -use syntax::ext::hygiene::ExpnId; +use syntax_expand::base::{MacroKind, SyntaxExtension}; +use syntax_expand::expand::AstFragment; +use syntax_expand::hygiene::ExpnId; use syntax::feature_gate::is_builtin_attr; use syntax::parse::token::{self, Token}; use syntax::print::pprust; diff --git a/src/librustc_resolve/diagnostics.rs b/src/librustc_resolve/diagnostics.rs index de875808670..5647d5b2794 100644 --- a/src/librustc_resolve/diagnostics.rs +++ b/src/librustc_resolve/diagnostics.rs @@ -10,7 +10,7 @@ use rustc::session::Session; use rustc::ty::{self, DefIdTree}; use rustc::util::nodemap::FxHashSet; use syntax::ast::{self, Ident, Path}; -use syntax::ext::base::MacroKind; +use syntax_expand::base::MacroKind; use syntax::feature_gate::BUILTIN_ATTRIBUTES; use syntax::source_map::SourceMap; use syntax::struct_span_err; diff --git a/src/librustc_resolve/late/diagnostics.rs b/src/librustc_resolve/late/diagnostics.rs index ace9903a835..412734eabe0 100644 --- a/src/librustc_resolve/late/diagnostics.rs +++ b/src/librustc_resolve/late/diagnostics.rs @@ -13,7 +13,7 @@ use rustc::hir::PrimTy; use rustc::session::config::nightly_options; use rustc::util::nodemap::FxHashSet; use syntax::ast::{self, Expr, ExprKind, Ident, NodeId, Path, Ty, TyKind}; -use syntax::ext::base::MacroKind; +use syntax_expand::base::MacroKind; use syntax::symbol::kw; use syntax::util::lev_distance::find_best_match_for_name; use syntax_pos::Span; diff --git a/src/librustc_resolve/lib.rs b/src/librustc_resolve/lib.rs index e5b0ef89a41..17d8f0f211a 100644 --- a/src/librustc_resolve/lib.rs +++ b/src/librustc_resolve/lib.rs @@ -35,18 +35,16 @@ use rustc::span_bug; use rustc_metadata::creader::CrateLoader; use rustc_metadata::cstore::CStore; -use syntax::ext::hygiene::{ExpnId, Transparency, SyntaxContext}; +use syntax_expand::hygiene::{ExpnId, Transparency, SyntaxContext}; +use syntax_expand::base::{SyntaxExtension, MacroKind, SpecialDerives}; +use syntax::{struct_span_err, unwrap_or}; +use syntax::attr; use syntax::ast::{self, Name, NodeId, Ident, FloatTy, IntTy, UintTy}; -use syntax::ext::base::{SyntaxExtension, MacroKind, SpecialDerives}; +use syntax::ast::{ItemKind, Path, CRATE_NODE_ID, Crate}; use syntax::print::pprust; use syntax::symbol::{kw, sym}; - -use syntax::visit::{self, Visitor}; -use syntax::attr; -use syntax::ast::{CRATE_NODE_ID, Crate}; -use syntax::ast::{ItemKind, Path}; -use syntax::{struct_span_err, unwrap_or}; use syntax::source_map::Spanned; +use syntax::visit::{self, Visitor}; use syntax_pos::{Span, DUMMY_SP}; use errors::{Applicability, DiagnosticBuilder}; diff --git a/src/librustc_resolve/macros.rs b/src/librustc_resolve/macros.rs index 49186088fa0..c91a0b2ed98 100644 --- a/src/librustc_resolve/macros.rs +++ b/src/librustc_resolve/macros.rs @@ -14,11 +14,11 @@ use rustc::{ty, lint, span_bug}; use syntax::ast::{self, NodeId, Ident}; use syntax::attr::StabilityLevel; use syntax::edition::Edition; -use syntax::ext::base::{self, InvocationRes, Indeterminate, SpecialDerives}; -use syntax::ext::base::{MacroKind, SyntaxExtension}; -use syntax::ext::expand::{AstFragment, AstFragmentKind, Invocation, InvocationKind}; -use syntax::ext::hygiene::{self, ExpnId, ExpnData, ExpnKind}; -use syntax::ext::compile_declarative_macro; +use syntax_expand::base::{self, InvocationRes, Indeterminate, SpecialDerives}; +use syntax_expand::base::{MacroKind, SyntaxExtension}; +use syntax_expand::expand::{AstFragment, AstFragmentKind, Invocation, InvocationKind}; +use syntax_expand::hygiene::{self, ExpnId, ExpnData, ExpnKind}; +use syntax_expand::compile_declarative_macro; use syntax::feature_gate::{emit_feature_err, is_builtin_attr_name}; use syntax::feature_gate::GateIssue; use syntax::print::pprust; diff --git a/src/librustc_resolve/resolve_imports.rs b/src/librustc_resolve/resolve_imports.rs index 56fd2da2576..424bf31a785 100644 --- a/src/librustc_resolve/resolve_imports.rs +++ b/src/librustc_resolve/resolve_imports.rs @@ -28,7 +28,7 @@ use rustc::util::nodemap::FxHashSet; use rustc::{bug, span_bug}; use syntax::ast::{Ident, Name, NodeId, CRATE_NODE_ID}; -use syntax::ext::hygiene::ExpnId; +use syntax_expand::hygiene::ExpnId; use syntax::symbol::kw; use syntax::util::lev_distance::find_best_match_for_name; use syntax::{struct_span_err, unwrap_or}; diff --git a/src/librustdoc/clean/inline.rs b/src/librustdoc/clean/inline.rs index 532c5f67bf3..63350b2a5d2 100644 --- a/src/librustdoc/clean/inline.rs +++ b/src/librustdoc/clean/inline.rs @@ -3,7 +3,7 @@ use std::iter::once; use syntax::ast; -use syntax::ext::base::MacroKind; +use syntax_expand::base::MacroKind; use syntax::symbol::sym; use syntax_pos::Span; diff --git a/src/librustdoc/clean/mod.rs b/src/librustdoc/clean/mod.rs index 212a09ee6e6..ba00631dc6c 100644 --- a/src/librustdoc/clean/mod.rs +++ b/src/librustdoc/clean/mod.rs @@ -28,7 +28,7 @@ use rustc::ty::layout::VariantIdx; use rustc::util::nodemap::{FxHashMap, FxHashSet}; use syntax::ast::{self, AttrStyle, Ident}; use syntax::attr; -use syntax::ext::base::MacroKind; +use syntax_expand::base::MacroKind; use syntax::source_map::DUMMY_SP; use syntax::symbol::{Symbol, kw, sym}; use syntax::symbol::InternedString; diff --git a/src/librustdoc/doctree.rs b/src/librustdoc/doctree.rs index 6e453561f6d..0dc094ae329 100644 --- a/src/librustdoc/doctree.rs +++ b/src/librustdoc/doctree.rs @@ -4,7 +4,7 @@ pub use self::StructType::*; use syntax::ast; use syntax::ast::Name; -use syntax::ext::base::MacroKind; +use syntax_expand::base::MacroKind; use syntax_pos::{self, Span}; use rustc::hir; diff --git a/src/librustdoc/html/item_type.rs b/src/librustdoc/html/item_type.rs index 5fb9afd6c49..e015739b03c 100644 --- a/src/librustdoc/html/item_type.rs +++ b/src/librustdoc/html/item_type.rs @@ -1,7 +1,7 @@ //! Item types. use std::fmt; -use syntax::ext::base::MacroKind; +use syntax_expand::base::MacroKind; use crate::clean; /// Item type. Corresponds to `clean::ItemEnum` variants. diff --git a/src/librustdoc/html/render.rs b/src/librustdoc/html/render.rs index b726ad1e0d6..72a72e89281 100644 --- a/src/librustdoc/html/render.rs +++ b/src/librustdoc/html/render.rs @@ -45,11 +45,11 @@ use errors; use serialize::json::{ToJson, Json, as_json}; use syntax::ast; use syntax::edition::Edition; -use syntax::ext::base::MacroKind; +use syntax::feature_gate::UnstableFeatures; use syntax::print::pprust; use syntax::source_map::FileName; -use syntax::feature_gate::UnstableFeatures; use syntax::symbol::{Symbol, sym}; +use syntax_expand::base::MacroKind; use rustc::hir::def_id::DefId; use rustc::middle::privacy::AccessLevels; use rustc::middle::stability; diff --git a/src/librustdoc/lib.rs b/src/librustdoc/lib.rs index 6bcb4a817d7..8cd32a3d1b5 100644 --- a/src/librustdoc/lib.rs +++ b/src/librustdoc/lib.rs @@ -34,6 +34,7 @@ extern crate rustc_typeck; extern crate rustc_lexer; extern crate serialize; extern crate syntax; +extern crate syntax_expand; extern crate syntax_pos; extern crate test as testing; #[macro_use] extern crate log; diff --git a/src/librustdoc/passes/collect_intra_doc_links.rs b/src/librustdoc/passes/collect_intra_doc_links.rs index 9186ed51420..4270b162baf 100644 --- a/src/librustdoc/passes/collect_intra_doc_links.rs +++ b/src/librustdoc/passes/collect_intra_doc_links.rs @@ -7,7 +7,7 @@ use rustc::ty; use rustc_resolve::ParentScope; use syntax; use syntax::ast::{self, Ident}; -use syntax::ext::base::SyntaxExtensionKind; +use syntax_expand::base::SyntaxExtensionKind; use syntax::feature_gate::UnstableFeatures; use syntax::symbol::Symbol; use syntax_pos::DUMMY_SP; diff --git a/src/librustdoc/visit_ast.rs b/src/librustdoc/visit_ast.rs index b6a90e1fb98..70c30687dab 100644 --- a/src/librustdoc/visit_ast.rs +++ b/src/librustdoc/visit_ast.rs @@ -8,7 +8,7 @@ use rustc::middle::privacy::AccessLevel; use rustc::util::nodemap::{FxHashSet, FxHashMap}; use rustc::ty::TyCtxt; use syntax::ast; -use syntax::ext::base::MacroKind; +use syntax_expand::base::MacroKind; use syntax::source_map::Spanned; use syntax::symbol::sym; use syntax_pos::{self, Span}; diff --git a/src/libsyntax/attr/mod.rs b/src/libsyntax/attr/mod.rs index 65257c7e985..402c2cad72f 100644 --- a/src/libsyntax/attr/mod.rs +++ b/src/libsyntax/attr/mod.rs @@ -298,14 +298,14 @@ impl Attribute { Ok(result) } - crate fn parse_derive_paths<'a>(&self, sess: &'a ParseSess) -> PResult<'a, Vec> { + pub fn parse_derive_paths<'a>(&self, sess: &'a ParseSess) -> PResult<'a, Vec> { if self.tokens.is_empty() { return Ok(Vec::new()); } self.parse(sess, |p| p.parse_derive_paths()) } - crate fn parse_meta<'a>(&self, sess: &'a ParseSess) -> PResult<'a, MetaItem> { + pub fn parse_meta<'a>(&self, sess: &'a ParseSess) -> PResult<'a, MetaItem> { Ok(MetaItem { path: self.path.clone(), kind: self.parse(sess, |parser| parser.parse_meta_item_kind())?, diff --git a/src/libsyntax/config.rs b/src/libsyntax/config.rs index ee5a4eeb3a3..54dc95291d6 100644 --- a/src/libsyntax/config.rs +++ b/src/libsyntax/config.rs @@ -56,6 +56,7 @@ pub fn features(mut krate: ast::Crate, sess: &ParseSess, edition: Edition, (krate, features) } +#[macro_export] macro_rules! configure { ($this:ident, $node:ident) => { match $this.configure($node) { diff --git a/src/libsyntax/ext/allocator.rs b/src/libsyntax/ext/allocator.rs deleted file mode 100644 index 99aeb5414c5..00000000000 --- a/src/libsyntax/ext/allocator.rs +++ /dev/null @@ -1,75 +0,0 @@ -use crate::{ast, attr, visit}; -use crate::symbol::{sym, Symbol}; -use syntax_pos::Span; - -#[derive(Clone, Copy)] -pub enum AllocatorKind { - Global, - DefaultLib, - DefaultExe, -} - -impl AllocatorKind { - pub fn fn_name(&self, base: &str) -> String { - match *self { - AllocatorKind::Global => format!("__rg_{}", base), - AllocatorKind::DefaultLib => format!("__rdl_{}", base), - AllocatorKind::DefaultExe => format!("__rde_{}", base), - } - } -} - -pub enum AllocatorTy { - Layout, - Ptr, - ResultPtr, - Unit, - Usize, -} - -pub struct AllocatorMethod { - pub name: &'static str, - pub inputs: &'static [AllocatorTy], - pub output: AllocatorTy, -} - -pub static ALLOCATOR_METHODS: &[AllocatorMethod] = &[ - AllocatorMethod { - name: "alloc", - inputs: &[AllocatorTy::Layout], - output: AllocatorTy::ResultPtr, - }, - AllocatorMethod { - name: "dealloc", - inputs: &[AllocatorTy::Ptr, AllocatorTy::Layout], - output: AllocatorTy::Unit, - }, - AllocatorMethod { - name: "realloc", - inputs: &[AllocatorTy::Ptr, AllocatorTy::Layout, AllocatorTy::Usize], - output: AllocatorTy::ResultPtr, - }, - AllocatorMethod { - name: "alloc_zeroed", - inputs: &[AllocatorTy::Layout], - output: AllocatorTy::ResultPtr, - }, -]; - -pub fn global_allocator_spans(krate: &ast::Crate) -> Vec { - struct Finder { name: Symbol, spans: Vec } - impl<'ast> visit::Visitor<'ast> for Finder { - fn visit_item(&mut self, item: &'ast ast::Item) { - if item.ident.name == self.name && - attr::contains_name(&item.attrs, sym::rustc_std_internal_symbol) { - self.spans.push(item.span); - } - visit::walk_item(self, item) - } - } - - let name = Symbol::intern(&AllocatorKind::Global.fn_name("alloc")); - let mut f = Finder { name, spans: Vec::new() }; - visit::walk_crate(&mut f, krate); - f.spans -} diff --git a/src/libsyntax/ext/base.rs b/src/libsyntax/ext/base.rs deleted file mode 100644 index 01be5642d5c..00000000000 --- a/src/libsyntax/ext/base.rs +++ /dev/null @@ -1,1188 +0,0 @@ -use crate::ast::{self, NodeId, Attribute, Name, PatKind}; -use crate::attr::{self, HasAttrs, Stability, Deprecation}; -use crate::source_map::SourceMap; -use crate::edition::Edition; -use crate::ext::expand::{self, AstFragment, Invocation}; -use crate::ext::hygiene::ExpnId; -use crate::mut_visit::{self, MutVisitor}; -use crate::parse::{self, parser, DirectoryOwnership}; -use crate::parse::token; -use crate::ptr::P; -use crate::sess::ParseSess; -use crate::symbol::{kw, sym, Ident, Symbol}; -use crate::{ThinVec, MACRO_ARGUMENTS}; -use crate::tokenstream::{self, TokenStream}; -use crate::visit::Visitor; - -use errors::{DiagnosticBuilder, DiagnosticId}; -use smallvec::{smallvec, SmallVec}; -use syntax_pos::{FileName, Span, MultiSpan, DUMMY_SP}; -use syntax_pos::hygiene::{AstPass, ExpnData, ExpnKind}; - -use rustc_data_structures::fx::FxHashMap; -use rustc_data_structures::sync::{self, Lrc}; -use std::iter; -use std::path::PathBuf; -use std::rc::Rc; -use std::default::Default; - -pub use syntax_pos::hygiene::MacroKind; - -#[derive(Debug,Clone)] -pub enum Annotatable { - Item(P), - TraitItem(P), - ImplItem(P), - ForeignItem(P), - Stmt(P), - Expr(P), - Arm(ast::Arm), - Field(ast::Field), - FieldPat(ast::FieldPat), - GenericParam(ast::GenericParam), - Param(ast::Param), - StructField(ast::StructField), - Variant(ast::Variant), -} - -impl HasAttrs for Annotatable { - fn attrs(&self) -> &[Attribute] { - match *self { - Annotatable::Item(ref item) => &item.attrs, - Annotatable::TraitItem(ref trait_item) => &trait_item.attrs, - Annotatable::ImplItem(ref impl_item) => &impl_item.attrs, - Annotatable::ForeignItem(ref foreign_item) => &foreign_item.attrs, - Annotatable::Stmt(ref stmt) => stmt.attrs(), - Annotatable::Expr(ref expr) => &expr.attrs, - Annotatable::Arm(ref arm) => &arm.attrs, - Annotatable::Field(ref field) => &field.attrs, - Annotatable::FieldPat(ref fp) => &fp.attrs, - Annotatable::GenericParam(ref gp) => &gp.attrs, - Annotatable::Param(ref p) => &p.attrs, - Annotatable::StructField(ref sf) => &sf.attrs, - Annotatable::Variant(ref v) => &v.attrs(), - } - } - - fn visit_attrs)>(&mut self, f: F) { - match self { - Annotatable::Item(item) => item.visit_attrs(f), - Annotatable::TraitItem(trait_item) => trait_item.visit_attrs(f), - Annotatable::ImplItem(impl_item) => impl_item.visit_attrs(f), - Annotatable::ForeignItem(foreign_item) => foreign_item.visit_attrs(f), - Annotatable::Stmt(stmt) => stmt.visit_attrs(f), - Annotatable::Expr(expr) => expr.visit_attrs(f), - Annotatable::Arm(arm) => arm.visit_attrs(f), - Annotatable::Field(field) => field.visit_attrs(f), - Annotatable::FieldPat(fp) => fp.visit_attrs(f), - Annotatable::GenericParam(gp) => gp.visit_attrs(f), - Annotatable::Param(p) => p.visit_attrs(f), - Annotatable::StructField(sf) => sf.visit_attrs(f), - Annotatable::Variant(v) => v.visit_attrs(f), - } - } -} - -impl Annotatable { - pub fn span(&self) -> Span { - match *self { - Annotatable::Item(ref item) => item.span, - Annotatable::TraitItem(ref trait_item) => trait_item.span, - Annotatable::ImplItem(ref impl_item) => impl_item.span, - Annotatable::ForeignItem(ref foreign_item) => foreign_item.span, - Annotatable::Stmt(ref stmt) => stmt.span, - Annotatable::Expr(ref expr) => expr.span, - Annotatable::Arm(ref arm) => arm.span, - Annotatable::Field(ref field) => field.span, - Annotatable::FieldPat(ref fp) => fp.pat.span, - Annotatable::GenericParam(ref gp) => gp.ident.span, - Annotatable::Param(ref p) => p.span, - Annotatable::StructField(ref sf) => sf.span, - Annotatable::Variant(ref v) => v.span, - } - } - - pub fn visit_with<'a, V: Visitor<'a>>(&'a self, visitor: &mut V) { - match self { - Annotatable::Item(item) => visitor.visit_item(item), - Annotatable::TraitItem(trait_item) => visitor.visit_trait_item(trait_item), - Annotatable::ImplItem(impl_item) => visitor.visit_impl_item(impl_item), - Annotatable::ForeignItem(foreign_item) => visitor.visit_foreign_item(foreign_item), - Annotatable::Stmt(stmt) => visitor.visit_stmt(stmt), - Annotatable::Expr(expr) => visitor.visit_expr(expr), - Annotatable::Arm(arm) => visitor.visit_arm(arm), - Annotatable::Field(field) => visitor.visit_field(field), - Annotatable::FieldPat(fp) => visitor.visit_field_pattern(fp), - Annotatable::GenericParam(gp) => visitor.visit_generic_param(gp), - Annotatable::Param(p) => visitor.visit_param(p), - Annotatable::StructField(sf) =>visitor.visit_struct_field(sf), - Annotatable::Variant(v) => visitor.visit_variant(v), - } - } - - pub fn expect_item(self) -> P { - match self { - Annotatable::Item(i) => i, - _ => panic!("expected Item") - } - } - - pub fn map_item_or(self, mut f: F, mut or: G) -> Annotatable - where F: FnMut(P) -> P, - G: FnMut(Annotatable) -> Annotatable - { - match self { - Annotatable::Item(i) => Annotatable::Item(f(i)), - _ => or(self) - } - } - - pub fn expect_trait_item(self) -> ast::TraitItem { - match self { - Annotatable::TraitItem(i) => i.into_inner(), - _ => panic!("expected Item") - } - } - - pub fn expect_impl_item(self) -> ast::ImplItem { - match self { - Annotatable::ImplItem(i) => i.into_inner(), - _ => panic!("expected Item") - } - } - - pub fn expect_foreign_item(self) -> ast::ForeignItem { - match self { - Annotatable::ForeignItem(i) => i.into_inner(), - _ => panic!("expected foreign item") - } - } - - pub fn expect_stmt(self) -> ast::Stmt { - match self { - Annotatable::Stmt(stmt) => stmt.into_inner(), - _ => panic!("expected statement"), - } - } - - pub fn expect_expr(self) -> P { - match self { - Annotatable::Expr(expr) => expr, - _ => panic!("expected expression"), - } - } - - pub fn expect_arm(self) -> ast::Arm { - match self { - Annotatable::Arm(arm) => arm, - _ => panic!("expected match arm") - } - } - - pub fn expect_field(self) -> ast::Field { - match self { - Annotatable::Field(field) => field, - _ => panic!("expected field") - } - } - - pub fn expect_field_pattern(self) -> ast::FieldPat { - match self { - Annotatable::FieldPat(fp) => fp, - _ => panic!("expected field pattern") - } - } - - pub fn expect_generic_param(self) -> ast::GenericParam { - match self { - Annotatable::GenericParam(gp) => gp, - _ => panic!("expected generic parameter") - } - } - - pub fn expect_param(self) -> ast::Param { - match self { - Annotatable::Param(param) => param, - _ => panic!("expected parameter") - } - } - - pub fn expect_struct_field(self) -> ast::StructField { - match self { - Annotatable::StructField(sf) => sf, - _ => panic!("expected struct field") - } - } - - pub fn expect_variant(self) -> ast::Variant { - match self { - Annotatable::Variant(v) => v, - _ => panic!("expected variant") - } - } - - pub fn derive_allowed(&self) -> bool { - match *self { - Annotatable::Item(ref item) => match item.kind { - ast::ItemKind::Struct(..) | - ast::ItemKind::Enum(..) | - ast::ItemKind::Union(..) => true, - _ => false, - }, - _ => false, - } - } -} - -// `meta_item` is the annotation, and `item` is the item being modified. -// FIXME Decorators should follow the same pattern too. -pub trait MultiItemModifier { - fn expand(&self, - ecx: &mut ExtCtxt<'_>, - span: Span, - meta_item: &ast::MetaItem, - item: Annotatable) - -> Vec; -} - -impl MultiItemModifier for F - where F: Fn(&mut ExtCtxt<'_>, Span, &ast::MetaItem, Annotatable) -> T, - T: Into>, -{ - fn expand(&self, - ecx: &mut ExtCtxt<'_>, - span: Span, - meta_item: &ast::MetaItem, - item: Annotatable) - -> Vec { - (*self)(ecx, span, meta_item, item).into() - } -} - -impl Into> for Annotatable { - fn into(self) -> Vec { - vec![self] - } -} - -pub trait ProcMacro { - fn expand<'cx>(&self, - ecx: &'cx mut ExtCtxt<'_>, - span: Span, - ts: TokenStream) - -> TokenStream; -} - -impl ProcMacro for F - where F: Fn(TokenStream) -> TokenStream -{ - fn expand<'cx>(&self, - _ecx: &'cx mut ExtCtxt<'_>, - _span: Span, - ts: TokenStream) - -> TokenStream { - // FIXME setup implicit context in TLS before calling self. - (*self)(ts) - } -} - -pub trait AttrProcMacro { - fn expand<'cx>(&self, - ecx: &'cx mut ExtCtxt<'_>, - span: Span, - annotation: TokenStream, - annotated: TokenStream) - -> TokenStream; -} - -impl AttrProcMacro for F - where F: Fn(TokenStream, TokenStream) -> TokenStream -{ - fn expand<'cx>(&self, - _ecx: &'cx mut ExtCtxt<'_>, - _span: Span, - annotation: TokenStream, - annotated: TokenStream) - -> TokenStream { - // FIXME setup implicit context in TLS before calling self. - (*self)(annotation, annotated) - } -} - -/// Represents a thing that maps token trees to Macro Results -pub trait TTMacroExpander { - fn expand<'cx>( - &self, - ecx: &'cx mut ExtCtxt<'_>, - span: Span, - input: TokenStream, - ) -> Box; -} - -pub type MacroExpanderFn = - for<'cx> fn(&'cx mut ExtCtxt<'_>, Span, TokenStream) - -> Box; - -impl TTMacroExpander for F - where F: for<'cx> Fn(&'cx mut ExtCtxt<'_>, Span, TokenStream) - -> Box -{ - fn expand<'cx>( - &self, - ecx: &'cx mut ExtCtxt<'_>, - span: Span, - mut input: TokenStream, - ) -> Box { - struct AvoidInterpolatedIdents; - - impl MutVisitor for AvoidInterpolatedIdents { - fn visit_tt(&mut self, tt: &mut tokenstream::TokenTree) { - if let tokenstream::TokenTree::Token(token) = tt { - if let token::Interpolated(nt) = &token.kind { - if let token::NtIdent(ident, is_raw) = **nt { - *tt = tokenstream::TokenTree::token( - token::Ident(ident.name, is_raw), ident.span - ); - } - } - } - mut_visit::noop_visit_tt(tt, self) - } - - fn visit_mac(&mut self, mac: &mut ast::Mac) { - mut_visit::noop_visit_mac(mac, self) - } - } - AvoidInterpolatedIdents.visit_tts(&mut input); - (*self)(ecx, span, input) - } -} - -// Use a macro because forwarding to a simple function has type system issues -macro_rules! make_stmts_default { - ($me:expr) => { - $me.make_expr().map(|e| smallvec![ast::Stmt { - id: ast::DUMMY_NODE_ID, - span: e.span, - kind: ast::StmtKind::Expr(e), - }]) - } -} - -/// The result of a macro expansion. The return values of the various -/// methods are spliced into the AST at the callsite of the macro. -pub trait MacResult { - /// Creates an expression. - fn make_expr(self: Box) -> Option> { - None - } - /// Creates zero or more items. - fn make_items(self: Box) -> Option; 1]>> { - None - } - - /// Creates zero or more impl items. - fn make_impl_items(self: Box) -> Option> { - None - } - - /// Creates zero or more trait items. - fn make_trait_items(self: Box) -> Option> { - None - } - - /// Creates zero or more items in an `extern {}` block - fn make_foreign_items(self: Box) -> Option> { None } - - /// Creates a pattern. - fn make_pat(self: Box) -> Option> { - None - } - - /// Creates zero or more statements. - /// - /// By default this attempts to create an expression statement, - /// returning None if that fails. - fn make_stmts(self: Box) -> Option> { - make_stmts_default!(self) - } - - fn make_ty(self: Box) -> Option> { - None - } - - fn make_arms(self: Box) -> Option> { - None - } - - fn make_fields(self: Box) -> Option> { - None - } - - fn make_field_patterns(self: Box) -> Option> { - None - } - - fn make_generic_params(self: Box) -> Option> { - None - } - - fn make_params(self: Box) -> Option> { - None - } - - fn make_struct_fields(self: Box) -> Option> { - None - } - - fn make_variants(self: Box) -> Option> { - None - } -} - -macro_rules! make_MacEager { - ( $( $fld:ident: $t:ty, )* ) => { - /// `MacResult` implementation for the common case where you've already - /// built each form of AST that you might return. - #[derive(Default)] - pub struct MacEager { - $( - pub $fld: Option<$t>, - )* - } - - impl MacEager { - $( - pub fn $fld(v: $t) -> Box { - Box::new(MacEager { - $fld: Some(v), - ..Default::default() - }) - } - )* - } - } -} - -make_MacEager! { - expr: P, - pat: P, - items: SmallVec<[P; 1]>, - impl_items: SmallVec<[ast::ImplItem; 1]>, - trait_items: SmallVec<[ast::TraitItem; 1]>, - foreign_items: SmallVec<[ast::ForeignItem; 1]>, - stmts: SmallVec<[ast::Stmt; 1]>, - ty: P, -} - -impl MacResult for MacEager { - fn make_expr(self: Box) -> Option> { - self.expr - } - - fn make_items(self: Box) -> Option; 1]>> { - self.items - } - - fn make_impl_items(self: Box) -> Option> { - self.impl_items - } - - fn make_trait_items(self: Box) -> Option> { - self.trait_items - } - - fn make_foreign_items(self: Box) -> Option> { - self.foreign_items - } - - fn make_stmts(self: Box) -> Option> { - match self.stmts.as_ref().map_or(0, |s| s.len()) { - 0 => make_stmts_default!(self), - _ => self.stmts, - } - } - - fn make_pat(self: Box) -> Option> { - if let Some(p) = self.pat { - return Some(p); - } - if let Some(e) = self.expr { - if let ast::ExprKind::Lit(_) = e.kind { - return Some(P(ast::Pat { - id: ast::DUMMY_NODE_ID, - span: e.span, - kind: PatKind::Lit(e), - })); - } - } - None - } - - fn make_ty(self: Box) -> Option> { - self.ty - } -} - -/// Fill-in macro expansion result, to allow compilation to continue -/// after hitting errors. -#[derive(Copy, Clone)] -pub struct DummyResult { - is_error: bool, - span: Span, -} - -impl DummyResult { - /// Creates a default MacResult that can be anything. - /// - /// Use this as a return value after hitting any errors and - /// calling `span_err`. - pub fn any(span: Span) -> Box { - Box::new(DummyResult { is_error: true, span }) - } - - /// Same as `any`, but must be a valid fragment, not error. - pub fn any_valid(span: Span) -> Box { - Box::new(DummyResult { is_error: false, span }) - } - - /// A plain dummy expression. - pub fn raw_expr(sp: Span, is_error: bool) -> P { - P(ast::Expr { - id: ast::DUMMY_NODE_ID, - kind: if is_error { ast::ExprKind::Err } else { ast::ExprKind::Tup(Vec::new()) }, - span: sp, - attrs: ThinVec::new(), - }) - } - - /// A plain dummy pattern. - pub fn raw_pat(sp: Span) -> ast::Pat { - ast::Pat { - id: ast::DUMMY_NODE_ID, - kind: PatKind::Wild, - span: sp, - } - } - - /// A plain dummy type. - pub fn raw_ty(sp: Span, is_error: bool) -> P { - P(ast::Ty { - id: ast::DUMMY_NODE_ID, - kind: if is_error { ast::TyKind::Err } else { ast::TyKind::Tup(Vec::new()) }, - span: sp - }) - } -} - -impl MacResult for DummyResult { - fn make_expr(self: Box) -> Option> { - Some(DummyResult::raw_expr(self.span, self.is_error)) - } - - fn make_pat(self: Box) -> Option> { - Some(P(DummyResult::raw_pat(self.span))) - } - - fn make_items(self: Box) -> Option; 1]>> { - Some(SmallVec::new()) - } - - fn make_impl_items(self: Box) -> Option> { - Some(SmallVec::new()) - } - - fn make_trait_items(self: Box) -> Option> { - Some(SmallVec::new()) - } - - fn make_foreign_items(self: Box) -> Option> { - Some(SmallVec::new()) - } - - fn make_stmts(self: Box) -> Option> { - Some(smallvec![ast::Stmt { - id: ast::DUMMY_NODE_ID, - kind: ast::StmtKind::Expr(DummyResult::raw_expr(self.span, self.is_error)), - span: self.span, - }]) - } - - fn make_ty(self: Box) -> Option> { - Some(DummyResult::raw_ty(self.span, self.is_error)) - } - - fn make_arms(self: Box) -> Option> { - Some(SmallVec::new()) - } - - fn make_fields(self: Box) -> Option> { - Some(SmallVec::new()) - } - - fn make_field_patterns(self: Box) -> Option> { - Some(SmallVec::new()) - } - - fn make_generic_params(self: Box) -> Option> { - Some(SmallVec::new()) - } - - fn make_params(self: Box) -> Option> { - Some(SmallVec::new()) - } - - fn make_struct_fields(self: Box) -> Option> { - Some(SmallVec::new()) - } - - fn make_variants(self: Box) -> Option> { - Some(SmallVec::new()) - } -} - -/// A syntax extension kind. -pub enum SyntaxExtensionKind { - /// A token-based function-like macro. - Bang( - /// An expander with signature TokenStream -> TokenStream. - Box, - ), - - /// An AST-based function-like macro. - LegacyBang( - /// An expander with signature TokenStream -> AST. - Box, - ), - - /// A token-based attribute macro. - Attr( - /// An expander with signature (TokenStream, TokenStream) -> TokenStream. - /// The first TokenSteam is the attribute itself, the second is the annotated item. - /// The produced TokenSteam replaces the input TokenSteam. - Box, - ), - - /// An AST-based attribute macro. - LegacyAttr( - /// An expander with signature (AST, AST) -> AST. - /// The first AST fragment is the attribute itself, the second is the annotated item. - /// The produced AST fragment replaces the input AST fragment. - Box, - ), - - /// A trivial attribute "macro" that does nothing, - /// only keeps the attribute and marks it as inert, - /// thus making it ineligible for further expansion. - NonMacroAttr { - /// Suppresses the `unused_attributes` lint for this attribute. - mark_used: bool, - }, - - /// A token-based derive macro. - Derive( - /// An expander with signature TokenStream -> TokenStream (not yet). - /// The produced TokenSteam is appended to the input TokenSteam. - Box, - ), - - /// An AST-based derive macro. - LegacyDerive( - /// An expander with signature AST -> AST. - /// The produced AST fragment is appended to the input AST fragment. - Box, - ), -} - -/// A struct representing a macro definition in "lowered" form ready for expansion. -pub struct SyntaxExtension { - /// A syntax extension kind. - pub kind: SyntaxExtensionKind, - /// Span of the macro definition. - pub span: Span, - /// Whitelist of unstable features that are treated as stable inside this macro. - pub allow_internal_unstable: Option>, - /// Suppresses the `unsafe_code` lint for code produced by this macro. - pub allow_internal_unsafe: bool, - /// Enables the macro helper hack (`ident!(...)` -> `$crate::ident!(...)`) for this macro. - pub local_inner_macros: bool, - /// The macro's stability info. - pub stability: Option, - /// The macro's deprecation info. - pub deprecation: Option, - /// Names of helper attributes registered by this macro. - pub helper_attrs: Vec, - /// Edition of the crate in which this macro is defined. - pub edition: Edition, - /// Built-in macros have a couple of special properties like availability - /// in `#[no_implicit_prelude]` modules, so we have to keep this flag. - pub is_builtin: bool, - /// We have to identify macros providing a `Copy` impl early for compatibility reasons. - pub is_derive_copy: bool, -} - -impl SyntaxExtension { - /// Returns which kind of macro calls this syntax extension. - pub fn macro_kind(&self) -> MacroKind { - match self.kind { - SyntaxExtensionKind::Bang(..) | - SyntaxExtensionKind::LegacyBang(..) => MacroKind::Bang, - SyntaxExtensionKind::Attr(..) | - SyntaxExtensionKind::LegacyAttr(..) | - SyntaxExtensionKind::NonMacroAttr { .. } => MacroKind::Attr, - SyntaxExtensionKind::Derive(..) | - SyntaxExtensionKind::LegacyDerive(..) => MacroKind::Derive, - } - } - - /// Constructs a syntax extension with default properties. - pub fn default(kind: SyntaxExtensionKind, edition: Edition) -> SyntaxExtension { - SyntaxExtension { - span: DUMMY_SP, - allow_internal_unstable: None, - allow_internal_unsafe: false, - local_inner_macros: false, - stability: None, - deprecation: None, - helper_attrs: Vec::new(), - edition, - is_builtin: false, - is_derive_copy: false, - kind, - } - } - - /// Constructs a syntax extension with the given properties - /// and other properties converted from attributes. - pub fn new( - sess: &ParseSess, - kind: SyntaxExtensionKind, - span: Span, - helper_attrs: Vec, - edition: Edition, - name: Name, - attrs: &[ast::Attribute], - ) -> SyntaxExtension { - let allow_internal_unstable = attr::allow_internal_unstable( - &attrs, &sess.span_diagnostic, - ).map(|features| features.collect::>().into()); - - let mut local_inner_macros = false; - if let Some(macro_export) = attr::find_by_name(attrs, sym::macro_export) { - if let Some(l) = macro_export.meta_item_list() { - local_inner_macros = attr::list_contains_name(&l, sym::local_inner_macros); - } - } - - let is_builtin = attr::contains_name(attrs, sym::rustc_builtin_macro); - - SyntaxExtension { - kind, - span, - allow_internal_unstable, - allow_internal_unsafe: attr::contains_name(attrs, sym::allow_internal_unsafe), - local_inner_macros, - stability: attr::find_stability(&sess, attrs, span), - deprecation: attr::find_deprecation(&sess, attrs, span), - helper_attrs, - edition, - is_builtin, - is_derive_copy: is_builtin && name == sym::Copy, - } - } - - pub fn dummy_bang(edition: Edition) -> SyntaxExtension { - fn expander<'cx>(_: &'cx mut ExtCtxt<'_>, span: Span, _: TokenStream) - -> Box { - DummyResult::any(span) - } - SyntaxExtension::default(SyntaxExtensionKind::LegacyBang(Box::new(expander)), edition) - } - - pub fn dummy_derive(edition: Edition) -> SyntaxExtension { - fn expander(_: &mut ExtCtxt<'_>, _: Span, _: &ast::MetaItem, _: Annotatable) - -> Vec { - Vec::new() - } - SyntaxExtension::default(SyntaxExtensionKind::Derive(Box::new(expander)), edition) - } - - pub fn non_macro_attr(mark_used: bool, edition: Edition) -> SyntaxExtension { - SyntaxExtension::default(SyntaxExtensionKind::NonMacroAttr { mark_used }, edition) - } - - pub fn expn_data(&self, parent: ExpnId, call_site: Span, descr: Symbol) -> ExpnData { - ExpnData { - kind: ExpnKind::Macro(self.macro_kind(), descr), - parent, - call_site, - def_site: self.span, - allow_internal_unstable: self.allow_internal_unstable.clone(), - allow_internal_unsafe: self.allow_internal_unsafe, - local_inner_macros: self.local_inner_macros, - edition: self.edition, - } - } -} - -pub type NamedSyntaxExtension = (Name, SyntaxExtension); - -/// Result of resolving a macro invocation. -pub enum InvocationRes { - Single(Lrc), - DeriveContainer(Vec>), -} - -/// Error type that denotes indeterminacy. -pub struct Indeterminate; - -bitflags::bitflags! { - /// Built-in derives that need some extra tracking beyond the usual macro functionality. - #[derive(Default)] - pub struct SpecialDerives: u8 { - const PARTIAL_EQ = 1 << 0; - const EQ = 1 << 1; - const COPY = 1 << 2; - } -} - -pub trait Resolver { - fn next_node_id(&mut self) -> NodeId; - - fn resolve_dollar_crates(&mut self); - fn visit_ast_fragment_with_placeholders(&mut self, expn_id: ExpnId, fragment: &AstFragment, - extra_placeholders: &[NodeId]); - fn register_builtin_macro(&mut self, ident: ast::Ident, ext: SyntaxExtension); - - fn expansion_for_ast_pass( - &mut self, - call_site: Span, - pass: AstPass, - features: &[Symbol], - parent_module_id: Option, - ) -> ExpnId; - - fn resolve_imports(&mut self); - - fn resolve_macro_invocation( - &mut self, invoc: &Invocation, eager_expansion_root: ExpnId, force: bool - ) -> Result; - - fn check_unused_macros(&self); - - fn has_derives(&self, expn_id: ExpnId, derives: SpecialDerives) -> bool; - fn add_derives(&mut self, expn_id: ExpnId, derives: SpecialDerives); -} - -#[derive(Clone)] -pub struct ModuleData { - pub mod_path: Vec, - pub directory: PathBuf, -} - -#[derive(Clone)] -pub struct ExpansionData { - pub id: ExpnId, - pub depth: usize, - pub module: Rc, - pub directory_ownership: DirectoryOwnership, - pub prior_type_ascription: Option<(Span, bool)>, -} - -/// One of these is made during expansion and incrementally updated as we go; -/// when a macro expansion occurs, the resulting nodes have the `backtrace() -/// -> expn_data` of their expansion context stored into their span. -pub struct ExtCtxt<'a> { - pub parse_sess: &'a ParseSess, - pub ecfg: expand::ExpansionConfig<'a>, - pub root_path: PathBuf, - pub resolver: &'a mut dyn Resolver, - pub current_expansion: ExpansionData, - pub expansions: FxHashMap>, -} - -impl<'a> ExtCtxt<'a> { - pub fn new(parse_sess: &'a ParseSess, - ecfg: expand::ExpansionConfig<'a>, - resolver: &'a mut dyn Resolver) - -> ExtCtxt<'a> { - ExtCtxt { - parse_sess, - ecfg, - root_path: PathBuf::new(), - resolver, - current_expansion: ExpansionData { - id: ExpnId::root(), - depth: 0, - module: Rc::new(ModuleData { mod_path: Vec::new(), directory: PathBuf::new() }), - directory_ownership: DirectoryOwnership::Owned { relative: None }, - prior_type_ascription: None, - }, - expansions: FxHashMap::default(), - } - } - - /// Returns a `Folder` for deeply expanding all macros in an AST node. - pub fn expander<'b>(&'b mut self) -> expand::MacroExpander<'b, 'a> { - expand::MacroExpander::new(self, false) - } - - /// Returns a `Folder` that deeply expands all macros and assigns all `NodeId`s in an AST node. - /// Once `NodeId`s are assigned, the node may not be expanded, removed, or otherwise modified. - pub fn monotonic_expander<'b>(&'b mut self) -> expand::MacroExpander<'b, 'a> { - expand::MacroExpander::new(self, true) - } - pub fn new_parser_from_tts(&self, stream: TokenStream) -> parser::Parser<'a> { - parse::stream_to_parser(self.parse_sess, stream, MACRO_ARGUMENTS) - } - pub fn source_map(&self) -> &'a SourceMap { self.parse_sess.source_map() } - pub fn parse_sess(&self) -> &'a ParseSess { self.parse_sess } - pub fn cfg(&self) -> &ast::CrateConfig { &self.parse_sess.config } - pub fn call_site(&self) -> Span { - self.current_expansion.id.expn_data().call_site - } - - /// Equivalent of `Span::def_site` from the proc macro API, - /// except that the location is taken from the span passed as an argument. - pub fn with_def_site_ctxt(&self, span: Span) -> Span { - span.with_def_site_ctxt(self.current_expansion.id) - } - - /// Equivalent of `Span::call_site` from the proc macro API, - /// except that the location is taken from the span passed as an argument. - pub fn with_call_site_ctxt(&self, span: Span) -> Span { - span.with_call_site_ctxt(self.current_expansion.id) - } - - /// Equivalent of `Span::mixed_site` from the proc macro API, - /// except that the location is taken from the span passed as an argument. - pub fn with_mixed_site_ctxt(&self, span: Span) -> Span { - span.with_mixed_site_ctxt(self.current_expansion.id) - } - - /// Returns span for the macro which originally caused the current expansion to happen. - /// - /// Stops backtracing at include! boundary. - pub fn expansion_cause(&self) -> Option { - let mut expn_id = self.current_expansion.id; - let mut last_macro = None; - loop { - let expn_data = expn_id.expn_data(); - // Stop going up the backtrace once include! is encountered - if expn_data.is_root() || expn_data.kind.descr() == sym::include { - break; - } - expn_id = expn_data.call_site.ctxt().outer_expn(); - last_macro = Some(expn_data.call_site); - } - last_macro - } - - pub fn struct_span_warn>(&self, - sp: S, - msg: &str) - -> DiagnosticBuilder<'a> { - self.parse_sess.span_diagnostic.struct_span_warn(sp, msg) - } - pub fn struct_span_err>(&self, - sp: S, - msg: &str) - -> DiagnosticBuilder<'a> { - self.parse_sess.span_diagnostic.struct_span_err(sp, msg) - } - pub fn struct_span_fatal>(&self, - sp: S, - msg: &str) - -> DiagnosticBuilder<'a> { - self.parse_sess.span_diagnostic.struct_span_fatal(sp, msg) - } - - /// Emit `msg` attached to `sp`, and stop compilation immediately. - /// - /// `span_err` should be strongly preferred where-ever possible: - /// this should *only* be used when: - /// - /// - continuing has a high risk of flow-on errors (e.g., errors in - /// declaring a macro would cause all uses of that macro to - /// complain about "undefined macro"), or - /// - there is literally nothing else that can be done (however, - /// in most cases one can construct a dummy expression/item to - /// substitute; we never hit resolve/type-checking so the dummy - /// value doesn't have to match anything) - pub fn span_fatal>(&self, sp: S, msg: &str) -> ! { - self.parse_sess.span_diagnostic.span_fatal(sp, msg).raise(); - } - - /// Emit `msg` attached to `sp`, without immediately stopping - /// compilation. - /// - /// Compilation will be stopped in the near future (at the end of - /// the macro expansion phase). - pub fn span_err>(&self, sp: S, msg: &str) { - self.parse_sess.span_diagnostic.span_err(sp, msg); - } - pub fn span_err_with_code>(&self, sp: S, msg: &str, code: DiagnosticId) { - self.parse_sess.span_diagnostic.span_err_with_code(sp, msg, code); - } - pub fn span_warn>(&self, sp: S, msg: &str) { - self.parse_sess.span_diagnostic.span_warn(sp, msg); - } - pub fn span_bug>(&self, sp: S, msg: &str) -> ! { - self.parse_sess.span_diagnostic.span_bug(sp, msg); - } - pub fn trace_macros_diag(&mut self) { - for (sp, notes) in self.expansions.iter() { - let mut db = self.parse_sess.span_diagnostic.span_note_diag(*sp, "trace_macro"); - for note in notes { - db.note(note); - } - db.emit(); - } - // Fixme: does this result in errors? - self.expansions.clear(); - } - pub fn bug(&self, msg: &str) -> ! { - self.parse_sess.span_diagnostic.bug(msg); - } - pub fn trace_macros(&self) -> bool { - self.ecfg.trace_mac - } - pub fn set_trace_macros(&mut self, x: bool) { - self.ecfg.trace_mac = x - } - pub fn ident_of(&self, st: &str, sp: Span) -> ast::Ident { - ast::Ident::from_str_and_span(st, sp) - } - pub fn std_path(&self, components: &[Symbol]) -> Vec { - let def_site = self.with_def_site_ctxt(DUMMY_SP); - iter::once(Ident::new(kw::DollarCrate, def_site)) - .chain(components.iter().map(|&s| Ident::with_dummy_span(s))) - .collect() - } - pub fn name_of(&self, st: &str) -> ast::Name { - Symbol::intern(st) - } - - pub fn check_unused_macros(&self) { - self.resolver.check_unused_macros(); - } - - /// Resolves a path mentioned inside Rust code. - /// - /// This unifies the logic used for resolving `include_X!`, and `#[doc(include)]` file paths. - /// - /// Returns an absolute path to the file that `path` refers to. - pub fn resolve_path(&self, path: impl Into, span: Span) -> PathBuf { - let path = path.into(); - - // Relative paths are resolved relative to the file in which they are found - // after macro expansion (that is, they are unhygienic). - if !path.is_absolute() { - let callsite = span.source_callsite(); - let mut result = match self.source_map().span_to_unmapped_path(callsite) { - FileName::Real(path) => path, - FileName::DocTest(path, _) => path, - other => panic!("cannot resolve relative path in non-file source `{}`", other), - }; - result.pop(); - result.push(path); - result - } else { - path - } - } -} - -/// Extracts a string literal from the macro expanded version of `expr`, -/// emitting `err_msg` if `expr` is not a string literal. This does not stop -/// compilation on error, merely emits a non-fatal error and returns `None`. -pub fn expr_to_spanned_string<'a>( - cx: &'a mut ExtCtxt<'_>, - expr: P, - err_msg: &str, -) -> Result<(Symbol, ast::StrStyle, Span), Option>> { - // Perform eager expansion on the expression. - // We want to be able to handle e.g., `concat!("foo", "bar")`. - let expr = cx.expander().fully_expand_fragment(AstFragment::Expr(expr)).make_expr(); - - Err(match expr.kind { - ast::ExprKind::Lit(ref l) => match l.kind { - ast::LitKind::Str(s, style) => return Ok((s, style, expr.span)), - ast::LitKind::Err(_) => None, - _ => Some(cx.struct_span_err(l.span, err_msg)) - }, - ast::ExprKind::Err => None, - _ => Some(cx.struct_span_err(expr.span, err_msg)) - }) -} - -pub fn expr_to_string(cx: &mut ExtCtxt<'_>, expr: P, err_msg: &str) - -> Option<(Symbol, ast::StrStyle)> { - expr_to_spanned_string(cx, expr, err_msg) - .map_err(|err| err.map(|mut err| err.emit())) - .ok() - .map(|(symbol, style, _)| (symbol, style)) -} - -/// Non-fatally assert that `tts` is empty. Note that this function -/// returns even when `tts` is non-empty, macros that *need* to stop -/// compilation should call -/// `cx.parse_sess.span_diagnostic.abort_if_errors()` (this should be -/// done as rarely as possible). -pub fn check_zero_tts(cx: &ExtCtxt<'_>, - sp: Span, - tts: TokenStream, - name: &str) { - if !tts.is_empty() { - cx.span_err(sp, &format!("{} takes no arguments", name)); - } -} - -/// Interpreting `tts` as a comma-separated sequence of expressions, -/// expect exactly one string literal, or emit an error and return `None`. -pub fn get_single_str_from_tts(cx: &mut ExtCtxt<'_>, - sp: Span, - tts: TokenStream, - name: &str) - -> Option { - let mut p = cx.new_parser_from_tts(tts); - if p.token == token::Eof { - cx.span_err(sp, &format!("{} takes 1 argument", name)); - return None - } - let ret = panictry!(p.parse_expr()); - let _ = p.eat(&token::Comma); - - if p.token != token::Eof { - cx.span_err(sp, &format!("{} takes 1 argument", name)); - } - expr_to_string(cx, ret, "argument must be a string literal").map(|(s, _)| { - s.to_string() - }) -} - -/// Extracts comma-separated expressions from `tts`. If there is a -/// parsing error, emit a non-fatal error and return `None`. -pub fn get_exprs_from_tts(cx: &mut ExtCtxt<'_>, - sp: Span, - tts: TokenStream) -> Option>> { - let mut p = cx.new_parser_from_tts(tts); - let mut es = Vec::new(); - while p.token != token::Eof { - let expr = panictry!(p.parse_expr()); - - // Perform eager expansion on the expression. - // We want to be able to handle e.g., `concat!("foo", "bar")`. - let expr = cx.expander().fully_expand_fragment(AstFragment::Expr(expr)).make_expr(); - - es.push(expr); - if p.eat(&token::Comma) { - continue; - } - if p.token != token::Eof { - cx.span_err(sp, "expected token: `,`"); - return None; - } - } - Some(es) -} diff --git a/src/libsyntax/ext/build.rs b/src/libsyntax/ext/build.rs deleted file mode 100644 index 8c5289671c9..00000000000 --- a/src/libsyntax/ext/build.rs +++ /dev/null @@ -1,639 +0,0 @@ -use crate::ast::{self, Ident, Expr, BlockCheckMode, UnOp, PatKind}; -use crate::attr; -use crate::source_map::{respan, Spanned}; -use crate::ext::base::ExtCtxt; -use crate::ptr::P; -use crate::symbol::{kw, sym, Symbol}; -use crate::ThinVec; - -use syntax_pos::{Pos, Span}; - -impl<'a> ExtCtxt<'a> { - pub fn path(&self, span: Span, strs: Vec ) -> ast::Path { - self.path_all(span, false, strs, vec![]) - } - pub fn path_ident(&self, span: Span, id: ast::Ident) -> ast::Path { - self.path(span, vec![id]) - } - pub fn path_global(&self, span: Span, strs: Vec ) -> ast::Path { - self.path_all(span, true, strs, vec![]) - } - pub fn path_all(&self, - span: Span, - global: bool, - mut idents: Vec , - args: Vec) - -> ast::Path { - assert!(!idents.is_empty()); - let add_root = global && !idents[0].is_path_segment_keyword(); - let mut segments = Vec::with_capacity(idents.len() + add_root as usize); - if add_root { - segments.push(ast::PathSegment::path_root(span)); - } - let last_ident = idents.pop().unwrap(); - segments.extend(idents.into_iter().map(|ident| { - ast::PathSegment::from_ident(ident.with_span_pos(span)) - })); - let args = if !args.is_empty() { - ast::AngleBracketedArgs { args, constraints: Vec::new(), span }.into() - } else { - None - }; - segments.push(ast::PathSegment { - ident: last_ident.with_span_pos(span), - id: ast::DUMMY_NODE_ID, - args, - }); - ast::Path { span, segments } - } - - pub fn ty_mt(&self, ty: P, mutbl: ast::Mutability) -> ast::MutTy { - ast::MutTy { - ty, - mutbl, - } - } - - pub fn ty(&self, span: Span, kind: ast::TyKind) -> P { - P(ast::Ty { - id: ast::DUMMY_NODE_ID, - span, - kind, - }) - } - - pub fn ty_path(&self, path: ast::Path) -> P { - self.ty(path.span, ast::TyKind::Path(None, path)) - } - - // Might need to take bounds as an argument in the future, if you ever want - // to generate a bounded existential trait type. - pub fn ty_ident(&self, span: Span, ident: ast::Ident) - -> P { - self.ty_path(self.path_ident(span, ident)) - } - - pub fn anon_const(&self, span: Span, kind: ast::ExprKind) -> ast::AnonConst { - ast::AnonConst { - id: ast::DUMMY_NODE_ID, - value: P(ast::Expr { - id: ast::DUMMY_NODE_ID, - kind, - span, - attrs: ThinVec::new(), - }) - } - } - - pub fn const_ident(&self, span: Span, ident: ast::Ident) -> ast::AnonConst { - self.anon_const(span, ast::ExprKind::Path(None, self.path_ident(span, ident))) - } - - pub fn ty_rptr(&self, - span: Span, - ty: P, - lifetime: Option, - mutbl: ast::Mutability) - -> P { - self.ty(span, - ast::TyKind::Rptr(lifetime, self.ty_mt(ty, mutbl))) - } - - pub fn ty_ptr(&self, - span: Span, - ty: P, - mutbl: ast::Mutability) - -> P { - self.ty(span, - ast::TyKind::Ptr(self.ty_mt(ty, mutbl))) - } - - pub fn typaram(&self, - span: Span, - ident: ast::Ident, - attrs: Vec, - bounds: ast::GenericBounds, - default: Option>) -> ast::GenericParam { - ast::GenericParam { - ident: ident.with_span_pos(span), - id: ast::DUMMY_NODE_ID, - attrs: attrs.into(), - bounds, - kind: ast::GenericParamKind::Type { - default, - }, - is_placeholder: false - } - } - - pub fn trait_ref(&self, path: ast::Path) -> ast::TraitRef { - ast::TraitRef { - path, - ref_id: ast::DUMMY_NODE_ID, - } - } - - pub fn poly_trait_ref(&self, span: Span, path: ast::Path) -> ast::PolyTraitRef { - ast::PolyTraitRef { - bound_generic_params: Vec::new(), - trait_ref: self.trait_ref(path), - span, - } - } - - pub fn trait_bound(&self, path: ast::Path) -> ast::GenericBound { - ast::GenericBound::Trait(self.poly_trait_ref(path.span, path), - ast::TraitBoundModifier::None) - } - - pub fn lifetime(&self, span: Span, ident: ast::Ident) -> ast::Lifetime { - ast::Lifetime { id: ast::DUMMY_NODE_ID, ident: ident.with_span_pos(span) } - } - - pub fn lifetime_def(&self, - span: Span, - ident: ast::Ident, - attrs: Vec, - bounds: ast::GenericBounds) - -> ast::GenericParam { - let lifetime = self.lifetime(span, ident); - ast::GenericParam { - ident: lifetime.ident, - id: lifetime.id, - attrs: attrs.into(), - bounds, - kind: ast::GenericParamKind::Lifetime, - is_placeholder: false - } - } - - pub fn stmt_expr(&self, expr: P) -> ast::Stmt { - ast::Stmt { - id: ast::DUMMY_NODE_ID, - span: expr.span, - kind: ast::StmtKind::Expr(expr), - } - } - - pub fn stmt_let(&self, sp: Span, mutbl: bool, ident: ast::Ident, - ex: P) -> ast::Stmt { - let pat = if mutbl { - let binding_mode = ast::BindingMode::ByValue(ast::Mutability::Mutable); - self.pat_ident_binding_mode(sp, ident, binding_mode) - } else { - self.pat_ident(sp, ident) - }; - let local = P(ast::Local { - pat, - ty: None, - init: Some(ex), - id: ast::DUMMY_NODE_ID, - span: sp, - attrs: ThinVec::new(), - }); - ast::Stmt { - id: ast::DUMMY_NODE_ID, - kind: ast::StmtKind::Local(local), - span: sp, - } - } - - // Generates `let _: Type;`, which is usually used for type assertions. - pub fn stmt_let_type_only(&self, span: Span, ty: P) -> ast::Stmt { - let local = P(ast::Local { - pat: self.pat_wild(span), - ty: Some(ty), - init: None, - id: ast::DUMMY_NODE_ID, - span, - attrs: ThinVec::new(), - }); - ast::Stmt { - id: ast::DUMMY_NODE_ID, - kind: ast::StmtKind::Local(local), - span, - } - } - - pub fn stmt_item(&self, sp: Span, item: P) -> ast::Stmt { - ast::Stmt { - id: ast::DUMMY_NODE_ID, - kind: ast::StmtKind::Item(item), - span: sp, - } - } - - pub fn block_expr(&self, expr: P) -> P { - self.block(expr.span, vec![ast::Stmt { - id: ast::DUMMY_NODE_ID, - span: expr.span, - kind: ast::StmtKind::Expr(expr), - }]) - } - pub fn block(&self, span: Span, stmts: Vec) -> P { - P(ast::Block { - stmts, - id: ast::DUMMY_NODE_ID, - rules: BlockCheckMode::Default, - span, - }) - } - - pub fn expr(&self, span: Span, kind: ast::ExprKind) -> P { - P(ast::Expr { - id: ast::DUMMY_NODE_ID, - kind, - span, - attrs: ThinVec::new(), - }) - } - - pub fn expr_path(&self, path: ast::Path) -> P { - self.expr(path.span, ast::ExprKind::Path(None, path)) - } - - pub fn expr_ident(&self, span: Span, id: ast::Ident) -> P { - self.expr_path(self.path_ident(span, id)) - } - pub fn expr_self(&self, span: Span) -> P { - self.expr_ident(span, Ident::with_dummy_span(kw::SelfLower)) - } - - pub fn expr_binary(&self, sp: Span, op: ast::BinOpKind, - lhs: P, rhs: P) -> P { - self.expr(sp, ast::ExprKind::Binary(Spanned { node: op, span: sp }, lhs, rhs)) - } - - pub fn expr_deref(&self, sp: Span, e: P) -> P { - self.expr(sp, ast::ExprKind::Unary(UnOp::Deref, e)) - } - - pub fn expr_addr_of(&self, sp: Span, e: P) -> P { - self.expr(sp, ast::ExprKind::AddrOf(ast::Mutability::Immutable, e)) - } - - pub fn expr_call( - &self, span: Span, expr: P, args: Vec>, - ) -> P { - self.expr(span, ast::ExprKind::Call(expr, args)) - } - pub fn expr_call_ident(&self, span: Span, id: ast::Ident, - args: Vec>) -> P { - self.expr(span, ast::ExprKind::Call(self.expr_ident(span, id), args)) - } - pub fn expr_call_global(&self, sp: Span, fn_path: Vec , - args: Vec> ) -> P { - let pathexpr = self.expr_path(self.path_global(sp, fn_path)); - self.expr_call(sp, pathexpr, args) - } - pub fn expr_method_call(&self, span: Span, - expr: P, - ident: ast::Ident, - mut args: Vec> ) -> P { - args.insert(0, expr); - let segment = ast::PathSegment::from_ident(ident.with_span_pos(span)); - self.expr(span, ast::ExprKind::MethodCall(segment, args)) - } - pub fn expr_block(&self, b: P) -> P { - self.expr(b.span, ast::ExprKind::Block(b, None)) - } - pub fn field_imm(&self, span: Span, ident: Ident, e: P) -> ast::Field { - ast::Field { - ident: ident.with_span_pos(span), - expr: e, - span, - is_shorthand: false, - attrs: ThinVec::new(), - id: ast::DUMMY_NODE_ID, - is_placeholder: false, - } - } - pub fn expr_struct( - &self, span: Span, path: ast::Path, fields: Vec - ) -> P { - self.expr(span, ast::ExprKind::Struct(path, fields, None)) - } - pub fn expr_struct_ident(&self, span: Span, - id: ast::Ident, fields: Vec) -> P { - self.expr_struct(span, self.path_ident(span, id), fields) - } - - pub fn expr_lit(&self, span: Span, lit_kind: ast::LitKind) -> P { - let lit = ast::Lit::from_lit_kind(lit_kind, span); - self.expr(span, ast::ExprKind::Lit(lit)) - } - pub fn expr_usize(&self, span: Span, i: usize) -> P { - self.expr_lit(span, ast::LitKind::Int(i as u128, - ast::LitIntType::Unsigned(ast::UintTy::Usize))) - } - pub fn expr_u32(&self, sp: Span, u: u32) -> P { - self.expr_lit(sp, ast::LitKind::Int(u as u128, - ast::LitIntType::Unsigned(ast::UintTy::U32))) - } - pub fn expr_bool(&self, sp: Span, value: bool) -> P { - self.expr_lit(sp, ast::LitKind::Bool(value)) - } - - pub fn expr_vec(&self, sp: Span, exprs: Vec>) -> P { - self.expr(sp, ast::ExprKind::Array(exprs)) - } - pub fn expr_vec_slice(&self, sp: Span, exprs: Vec>) -> P { - self.expr_addr_of(sp, self.expr_vec(sp, exprs)) - } - pub fn expr_str(&self, sp: Span, s: Symbol) -> P { - self.expr_lit(sp, ast::LitKind::Str(s, ast::StrStyle::Cooked)) - } - - pub fn expr_cast(&self, sp: Span, expr: P, ty: P) -> P { - self.expr(sp, ast::ExprKind::Cast(expr, ty)) - } - - pub fn expr_some(&self, sp: Span, expr: P) -> P { - let some = self.std_path(&[sym::option, sym::Option, sym::Some]); - self.expr_call_global(sp, some, vec![expr]) - } - - pub fn expr_tuple(&self, sp: Span, exprs: Vec>) -> P { - self.expr(sp, ast::ExprKind::Tup(exprs)) - } - - pub fn expr_fail(&self, span: Span, msg: Symbol) -> P { - let loc = self.source_map().lookup_char_pos(span.lo()); - let expr_file = self.expr_str(span, Symbol::intern(&loc.file.name.to_string())); - let expr_line = self.expr_u32(span, loc.line as u32); - let expr_col = self.expr_u32(span, loc.col.to_usize() as u32 + 1); - let expr_loc_tuple = self.expr_tuple(span, vec![expr_file, expr_line, expr_col]); - let expr_loc_ptr = self.expr_addr_of(span, expr_loc_tuple); - self.expr_call_global( - span, - [sym::std, sym::rt, sym::begin_panic].iter().map(|s| Ident::new(*s, span)).collect(), - vec![ - self.expr_str(span, msg), - expr_loc_ptr]) - } - - pub fn expr_unreachable(&self, span: Span) -> P { - self.expr_fail(span, Symbol::intern("internal error: entered unreachable code")) - } - - pub fn expr_ok(&self, sp: Span, expr: P) -> P { - let ok = self.std_path(&[sym::result, sym::Result, sym::Ok]); - self.expr_call_global(sp, ok, vec![expr]) - } - - pub fn expr_try(&self, sp: Span, head: P) -> P { - let ok = self.std_path(&[sym::result, sym::Result, sym::Ok]); - let ok_path = self.path_global(sp, ok); - let err = self.std_path(&[sym::result, sym::Result, sym::Err]); - let err_path = self.path_global(sp, err); - - let binding_variable = self.ident_of("__try_var", sp); - let binding_pat = self.pat_ident(sp, binding_variable); - let binding_expr = self.expr_ident(sp, binding_variable); - - // `Ok(__try_var)` pattern - let ok_pat = self.pat_tuple_struct(sp, ok_path, vec![binding_pat.clone()]); - - // `Err(__try_var)` (pattern and expression respectively) - let err_pat = self.pat_tuple_struct(sp, err_path.clone(), vec![binding_pat]); - let err_inner_expr = self.expr_call(sp, self.expr_path(err_path), - vec![binding_expr.clone()]); - // `return Err(__try_var)` - let err_expr = self.expr(sp, ast::ExprKind::Ret(Some(err_inner_expr))); - - // `Ok(__try_var) => __try_var` - let ok_arm = self.arm(sp, ok_pat, binding_expr); - // `Err(__try_var) => return Err(__try_var)` - let err_arm = self.arm(sp, err_pat, err_expr); - - // `match head { Ok() => ..., Err() => ... }` - self.expr_match(sp, head, vec![ok_arm, err_arm]) - } - - - pub fn pat(&self, span: Span, kind: PatKind) -> P { - P(ast::Pat { id: ast::DUMMY_NODE_ID, kind, span }) - } - pub fn pat_wild(&self, span: Span) -> P { - self.pat(span, PatKind::Wild) - } - pub fn pat_lit(&self, span: Span, expr: P) -> P { - self.pat(span, PatKind::Lit(expr)) - } - pub fn pat_ident(&self, span: Span, ident: ast::Ident) -> P { - let binding_mode = ast::BindingMode::ByValue(ast::Mutability::Immutable); - self.pat_ident_binding_mode(span, ident, binding_mode) - } - - pub fn pat_ident_binding_mode(&self, - span: Span, - ident: ast::Ident, - bm: ast::BindingMode) -> P { - let pat = PatKind::Ident(bm, ident.with_span_pos(span), None); - self.pat(span, pat) - } - pub fn pat_path(&self, span: Span, path: ast::Path) -> P { - self.pat(span, PatKind::Path(None, path)) - } - pub fn pat_tuple_struct(&self, span: Span, path: ast::Path, - subpats: Vec>) -> P { - self.pat(span, PatKind::TupleStruct(path, subpats)) - } - pub fn pat_struct(&self, span: Span, path: ast::Path, - field_pats: Vec) -> P { - self.pat(span, PatKind::Struct(path, field_pats, false)) - } - pub fn pat_tuple(&self, span: Span, pats: Vec>) -> P { - self.pat(span, PatKind::Tuple(pats)) - } - - pub fn pat_some(&self, span: Span, pat: P) -> P { - let some = self.std_path(&[sym::option, sym::Option, sym::Some]); - let path = self.path_global(span, some); - self.pat_tuple_struct(span, path, vec![pat]) - } - - pub fn pat_none(&self, span: Span) -> P { - let some = self.std_path(&[sym::option, sym::Option, sym::None]); - let path = self.path_global(span, some); - self.pat_path(span, path) - } - - pub fn pat_ok(&self, span: Span, pat: P) -> P { - let some = self.std_path(&[sym::result, sym::Result, sym::Ok]); - let path = self.path_global(span, some); - self.pat_tuple_struct(span, path, vec![pat]) - } - - pub fn pat_err(&self, span: Span, pat: P) -> P { - let some = self.std_path(&[sym::result, sym::Result, sym::Err]); - let path = self.path_global(span, some); - self.pat_tuple_struct(span, path, vec![pat]) - } - - pub fn arm(&self, span: Span, pat: P, expr: P) -> ast::Arm { - ast::Arm { - attrs: vec![], - pat, - guard: None, - body: expr, - span, - id: ast::DUMMY_NODE_ID, - is_placeholder: false, - } - } - - pub fn arm_unreachable(&self, span: Span) -> ast::Arm { - self.arm(span, self.pat_wild(span), self.expr_unreachable(span)) - } - - pub fn expr_match(&self, span: Span, arg: P, arms: Vec) -> P { - self.expr(span, ast::ExprKind::Match(arg, arms)) - } - - pub fn expr_if(&self, span: Span, cond: P, - then: P, els: Option>) -> P { - let els = els.map(|x| self.expr_block(self.block_expr(x))); - self.expr(span, ast::ExprKind::If(cond, self.block_expr(then), els)) - } - - pub fn lambda_fn_decl(&self, - span: Span, - fn_decl: P, - body: P, - fn_decl_span: Span) // span of the `|...|` part - -> P { - self.expr(span, ast::ExprKind::Closure(ast::CaptureBy::Ref, - ast::IsAsync::NotAsync, - ast::Movability::Movable, - fn_decl, - body, - fn_decl_span)) - } - - pub fn lambda(&self, - span: Span, - ids: Vec, - body: P) - -> P { - let fn_decl = self.fn_decl( - ids.iter().map(|id| self.param(span, *id, self.ty(span, ast::TyKind::Infer))).collect(), - ast::FunctionRetTy::Default(span)); - - // FIXME -- We are using `span` as the span of the `|...|` - // part of the lambda, but it probably (maybe?) corresponds to - // the entire lambda body. Probably we should extend the API - // here, but that's not entirely clear. - self.expr(span, ast::ExprKind::Closure(ast::CaptureBy::Ref, - ast::IsAsync::NotAsync, - ast::Movability::Movable, - fn_decl, - body, - span)) - } - - pub fn lambda0(&self, span: Span, body: P) -> P { - self.lambda(span, Vec::new(), body) - } - - pub fn lambda1(&self, span: Span, body: P, ident: ast::Ident) -> P { - self.lambda(span, vec![ident], body) - } - - pub fn lambda_stmts_1(&self, span: Span, stmts: Vec, - ident: ast::Ident) -> P { - self.lambda1(span, self.expr_block(self.block(span, stmts)), ident) - } - - pub fn param(&self, span: Span, ident: ast::Ident, ty: P) -> ast::Param { - let arg_pat = self.pat_ident(span, ident); - ast::Param { - attrs: ThinVec::default(), - id: ast::DUMMY_NODE_ID, - pat: arg_pat, - span, - ty, - is_placeholder: false, - } - } - - // FIXME: unused `self` - pub fn fn_decl(&self, inputs: Vec, output: ast::FunctionRetTy) -> P { - P(ast::FnDecl { - inputs, - output, - }) - } - - pub fn item(&self, span: Span, name: Ident, - attrs: Vec, kind: ast::ItemKind) -> P { - // FIXME: Would be nice if our generated code didn't violate - // Rust coding conventions - P(ast::Item { - ident: name, - attrs, - id: ast::DUMMY_NODE_ID, - kind, - vis: respan(span.shrink_to_lo(), ast::VisibilityKind::Inherited), - span, - tokens: None, - }) - } - - pub fn variant(&self, span: Span, ident: Ident, tys: Vec> ) -> ast::Variant { - let fields: Vec<_> = tys.into_iter().map(|ty| { - ast::StructField { - span: ty.span, - ty, - ident: None, - vis: respan(span.shrink_to_lo(), ast::VisibilityKind::Inherited), - attrs: Vec::new(), - id: ast::DUMMY_NODE_ID, - is_placeholder: false, - } - }).collect(); - - let vdata = if fields.is_empty() { - ast::VariantData::Unit(ast::DUMMY_NODE_ID) - } else { - ast::VariantData::Tuple(fields, ast::DUMMY_NODE_ID) - }; - - ast::Variant { - attrs: Vec::new(), - data: vdata, - disr_expr: None, - id: ast::DUMMY_NODE_ID, - ident, - span, - is_placeholder: false, - } - } - - pub fn item_static(&self, - span: Span, - name: Ident, - ty: P, - mutbl: ast::Mutability, - expr: P) - -> P { - self.item(span, name, Vec::new(), ast::ItemKind::Static(ty, mutbl, expr)) - } - - pub fn item_const(&self, - span: Span, - name: Ident, - ty: P, - expr: P) - -> P { - self.item(span, name, Vec::new(), ast::ItemKind::Const(ty, expr)) - } - - pub fn attribute(&self, mi: ast::MetaItem) -> ast::Attribute { - attr::mk_attr_outer(mi) - } - - pub fn meta_word(&self, sp: Span, w: ast::Name) -> ast::MetaItem { - attr::mk_word_item(Ident::new(w, sp)) - } -} diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs deleted file mode 100644 index 44a51a01710..00000000000 --- a/src/libsyntax/ext/expand.rs +++ /dev/null @@ -1,1543 +0,0 @@ -use crate::ast::{self, AttrItem, Block, Ident, LitKind, NodeId, PatKind, Path}; -use crate::ast::{MacStmtStyle, StmtKind, ItemKind}; -use crate::attr::{self, HasAttrs}; -use crate::source_map::respan; -use crate::config::StripUnconfigured; -use crate::ext::base::*; -use crate::ext::proc_macro::{collect_derives, MarkAttrs}; -use crate::ext::hygiene::{ExpnId, SyntaxContext, ExpnData, ExpnKind}; -use crate::ext::mbe::macro_rules::annotate_err_with_kind; -use crate::ext::placeholders::{placeholder, PlaceholderExpander}; -use crate::feature_gate::{self, Features, GateIssue, is_builtin_attr, emit_feature_err}; -use crate::mut_visit::*; -use crate::parse::{DirectoryOwnership, PResult}; -use crate::parse::token; -use crate::parse::parser::Parser; -use crate::print::pprust; -use crate::ptr::P; -use crate::symbol::{sym, Symbol}; -use crate::tokenstream::{TokenStream, TokenTree}; -use crate::visit::Visitor; -use crate::util::map_in_place::MapInPlace; - -use errors::{Applicability, FatalError}; -use smallvec::{smallvec, SmallVec}; -use syntax_pos::{Span, DUMMY_SP, FileName}; - -use rustc_data_structures::fx::FxHashMap; -use rustc_data_structures::sync::Lrc; -use std::io::ErrorKind; -use std::{iter, mem, slice}; -use std::ops::DerefMut; -use std::rc::Rc; -use std::path::PathBuf; - -macro_rules! ast_fragments { - ( - $($Kind:ident($AstTy:ty) { - $kind_name:expr; - $(one fn $mut_visit_ast:ident; fn $visit_ast:ident;)? - $(many fn $flat_map_ast_elt:ident; fn $visit_ast_elt:ident;)? - fn $make_ast:ident; - })* - ) => { - /// A fragment of AST that can be produced by a single macro expansion. - /// Can also serve as an input and intermediate result for macro expansion operations. - pub enum AstFragment { - OptExpr(Option>), - $($Kind($AstTy),)* - } - - /// "Discriminant" of an AST fragment. - #[derive(Copy, Clone, PartialEq, Eq)] - pub enum AstFragmentKind { - OptExpr, - $($Kind,)* - } - - impl AstFragmentKind { - pub fn name(self) -> &'static str { - match self { - AstFragmentKind::OptExpr => "expression", - $(AstFragmentKind::$Kind => $kind_name,)* - } - } - - fn make_from<'a>(self, result: Box) -> Option { - match self { - AstFragmentKind::OptExpr => - result.make_expr().map(Some).map(AstFragment::OptExpr), - $(AstFragmentKind::$Kind => result.$make_ast().map(AstFragment::$Kind),)* - } - } - } - - impl AstFragment { - pub fn make_opt_expr(self) -> Option> { - match self { - AstFragment::OptExpr(expr) => expr, - _ => panic!("AstFragment::make_* called on the wrong kind of fragment"), - } - } - - $(pub fn $make_ast(self) -> $AstTy { - match self { - AstFragment::$Kind(ast) => ast, - _ => panic!("AstFragment::make_* called on the wrong kind of fragment"), - } - })* - - pub fn mut_visit_with(&mut self, vis: &mut F) { - match self { - AstFragment::OptExpr(opt_expr) => { - visit_clobber(opt_expr, |opt_expr| { - if let Some(expr) = opt_expr { - vis.filter_map_expr(expr) - } else { - None - } - }); - } - $($(AstFragment::$Kind(ast) => vis.$mut_visit_ast(ast),)?)* - $($(AstFragment::$Kind(ast) => - ast.flat_map_in_place(|ast| vis.$flat_map_ast_elt(ast)),)?)* - } - } - - pub fn visit_with<'a, V: Visitor<'a>>(&'a self, visitor: &mut V) { - match *self { - AstFragment::OptExpr(Some(ref expr)) => visitor.visit_expr(expr), - AstFragment::OptExpr(None) => {} - $($(AstFragment::$Kind(ref ast) => visitor.$visit_ast(ast),)?)* - $($(AstFragment::$Kind(ref ast) => for ast_elt in &ast[..] { - visitor.$visit_ast_elt(ast_elt); - })?)* - } - } - } - - impl<'a> MacResult for crate::ext::mbe::macro_rules::ParserAnyMacro<'a> { - $(fn $make_ast(self: Box>) - -> Option<$AstTy> { - Some(self.make(AstFragmentKind::$Kind).$make_ast()) - })* - } - } -} - -ast_fragments! { - Expr(P) { "expression"; one fn visit_expr; fn visit_expr; fn make_expr; } - Pat(P) { "pattern"; one fn visit_pat; fn visit_pat; fn make_pat; } - Ty(P) { "type"; one fn visit_ty; fn visit_ty; fn make_ty; } - Stmts(SmallVec<[ast::Stmt; 1]>) { - "statement"; many fn flat_map_stmt; fn visit_stmt; fn make_stmts; - } - Items(SmallVec<[P; 1]>) { - "item"; many fn flat_map_item; fn visit_item; fn make_items; - } - TraitItems(SmallVec<[ast::TraitItem; 1]>) { - "trait item"; many fn flat_map_trait_item; fn visit_trait_item; fn make_trait_items; - } - ImplItems(SmallVec<[ast::ImplItem; 1]>) { - "impl item"; many fn flat_map_impl_item; fn visit_impl_item; fn make_impl_items; - } - ForeignItems(SmallVec<[ast::ForeignItem; 1]>) { - "foreign item"; - many fn flat_map_foreign_item; - fn visit_foreign_item; - fn make_foreign_items; - } - Arms(SmallVec<[ast::Arm; 1]>) { - "match arm"; many fn flat_map_arm; fn visit_arm; fn make_arms; - } - Fields(SmallVec<[ast::Field; 1]>) { - "field expression"; many fn flat_map_field; fn visit_field; fn make_fields; - } - FieldPats(SmallVec<[ast::FieldPat; 1]>) { - "field pattern"; - many fn flat_map_field_pattern; - fn visit_field_pattern; - fn make_field_patterns; - } - GenericParams(SmallVec<[ast::GenericParam; 1]>) { - "generic parameter"; - many fn flat_map_generic_param; - fn visit_generic_param; - fn make_generic_params; - } - Params(SmallVec<[ast::Param; 1]>) { - "function parameter"; many fn flat_map_param; fn visit_param; fn make_params; - } - StructFields(SmallVec<[ast::StructField; 1]>) { - "field"; - many fn flat_map_struct_field; - fn visit_struct_field; - fn make_struct_fields; - } - Variants(SmallVec<[ast::Variant; 1]>) { - "variant"; many fn flat_map_variant; fn visit_variant; fn make_variants; - } -} - -impl AstFragmentKind { - fn dummy(self, span: Span) -> AstFragment { - self.make_from(DummyResult::any(span)).expect("couldn't create a dummy AST fragment") - } - - fn expect_from_annotatables>(self, items: I) - -> AstFragment { - let mut items = items.into_iter(); - match self { - AstFragmentKind::Arms => - AstFragment::Arms(items.map(Annotatable::expect_arm).collect()), - AstFragmentKind::Fields => - AstFragment::Fields(items.map(Annotatable::expect_field).collect()), - AstFragmentKind::FieldPats => - AstFragment::FieldPats(items.map(Annotatable::expect_field_pattern).collect()), - AstFragmentKind::GenericParams => - AstFragment::GenericParams(items.map(Annotatable::expect_generic_param).collect()), - AstFragmentKind::Params => - AstFragment::Params(items.map(Annotatable::expect_param).collect()), - AstFragmentKind::StructFields => AstFragment::StructFields( - items.map(Annotatable::expect_struct_field).collect() - ), - AstFragmentKind::Variants => - AstFragment::Variants(items.map(Annotatable::expect_variant).collect()), - AstFragmentKind::Items => - AstFragment::Items(items.map(Annotatable::expect_item).collect()), - AstFragmentKind::ImplItems => - AstFragment::ImplItems(items.map(Annotatable::expect_impl_item).collect()), - AstFragmentKind::TraitItems => - AstFragment::TraitItems(items.map(Annotatable::expect_trait_item).collect()), - AstFragmentKind::ForeignItems => - AstFragment::ForeignItems(items.map(Annotatable::expect_foreign_item).collect()), - AstFragmentKind::Stmts => - AstFragment::Stmts(items.map(Annotatable::expect_stmt).collect()), - AstFragmentKind::Expr => AstFragment::Expr( - items.next().expect("expected exactly one expression").expect_expr() - ), - AstFragmentKind::OptExpr => - AstFragment::OptExpr(items.next().map(Annotatable::expect_expr)), - AstFragmentKind::Pat | AstFragmentKind::Ty => - panic!("patterns and types aren't annotatable"), - } - } -} - -pub struct Invocation { - pub kind: InvocationKind, - pub fragment_kind: AstFragmentKind, - pub expansion_data: ExpansionData, -} - -pub enum InvocationKind { - Bang { - mac: ast::Mac, - span: Span, - }, - Attr { - attr: ast::Attribute, - item: Annotatable, - // Required for resolving derive helper attributes. - derives: Vec, - // We temporarily report errors for attribute macros placed after derives - after_derive: bool, - }, - Derive { - path: Path, - item: Annotatable, - }, - /// "Invocation" that contains all derives from an item, - /// broken into multiple `Derive` invocations when expanded. - /// FIXME: Find a way to remove it. - DeriveContainer { - derives: Vec, - item: Annotatable, - }, -} - -impl Invocation { - pub fn span(&self) -> Span { - match &self.kind { - InvocationKind::Bang { span, .. } => *span, - InvocationKind::Attr { attr, .. } => attr.span, - InvocationKind::Derive { path, .. } => path.span, - InvocationKind::DeriveContainer { item, .. } => item.span(), - } - } -} - -pub struct MacroExpander<'a, 'b> { - pub cx: &'a mut ExtCtxt<'b>, - monotonic: bool, // cf. `cx.monotonic_expander()` -} - -impl<'a, 'b> MacroExpander<'a, 'b> { - pub fn new(cx: &'a mut ExtCtxt<'b>, monotonic: bool) -> Self { - MacroExpander { cx, monotonic } - } - - pub fn expand_crate(&mut self, mut krate: ast::Crate) -> ast::Crate { - let mut module = ModuleData { - mod_path: vec![Ident::from_str(&self.cx.ecfg.crate_name)], - directory: match self.cx.source_map().span_to_unmapped_path(krate.span) { - FileName::Real(path) => path, - other => PathBuf::from(other.to_string()), - }, - }; - module.directory.pop(); - self.cx.root_path = module.directory.clone(); - self.cx.current_expansion.module = Rc::new(module); - - let orig_mod_span = krate.module.inner; - - let krate_item = AstFragment::Items(smallvec![P(ast::Item { - attrs: krate.attrs, - span: krate.span, - kind: ast::ItemKind::Mod(krate.module), - ident: Ident::invalid(), - id: ast::DUMMY_NODE_ID, - vis: respan(krate.span.shrink_to_lo(), ast::VisibilityKind::Public), - tokens: None, - })]); - - match self.fully_expand_fragment(krate_item).make_items().pop().map(P::into_inner) { - Some(ast::Item { attrs, kind: ast::ItemKind::Mod(module), .. }) => { - krate.attrs = attrs; - krate.module = module; - }, - None => { - // Resolution failed so we return an empty expansion - krate.attrs = vec![]; - krate.module = ast::Mod { - inner: orig_mod_span, - items: vec![], - inline: true, - }; - }, - _ => unreachable!(), - }; - self.cx.trace_macros_diag(); - krate - } - - // Recursively expand all macro invocations in this AST fragment. - pub fn fully_expand_fragment(&mut self, input_fragment: AstFragment) -> AstFragment { - let orig_expansion_data = self.cx.current_expansion.clone(); - self.cx.current_expansion.depth = 0; - - // Collect all macro invocations and replace them with placeholders. - let (mut fragment_with_placeholders, mut invocations) - = self.collect_invocations(input_fragment, &[]); - - // Optimization: if we resolve all imports now, - // we'll be able to immediately resolve most of imported macros. - self.resolve_imports(); - - // Resolve paths in all invocations and produce output expanded fragments for them, but - // do not insert them into our input AST fragment yet, only store in `expanded_fragments`. - // The output fragments also go through expansion recursively until no invocations are left. - // Unresolved macros produce dummy outputs as a recovery measure. - invocations.reverse(); - let mut expanded_fragments = Vec::new(); - let mut all_derive_placeholders: FxHashMap> = FxHashMap::default(); - let mut undetermined_invocations = Vec::new(); - let (mut progress, mut force) = (false, !self.monotonic); - loop { - let invoc = if let Some(invoc) = invocations.pop() { - invoc - } else { - self.resolve_imports(); - if undetermined_invocations.is_empty() { break } - invocations = mem::take(&mut undetermined_invocations); - force = !mem::replace(&mut progress, false); - continue - }; - - let eager_expansion_root = - if self.monotonic { invoc.expansion_data.id } else { orig_expansion_data.id }; - let res = match self.cx.resolver.resolve_macro_invocation( - &invoc, eager_expansion_root, force - ) { - Ok(res) => res, - Err(Indeterminate) => { - undetermined_invocations.push(invoc); - continue - } - }; - - progress = true; - let ExpansionData { depth, id: expn_id, .. } = invoc.expansion_data; - self.cx.current_expansion = invoc.expansion_data.clone(); - - // FIXME(jseyfried): Refactor out the following logic - let (expanded_fragment, new_invocations) = match res { - InvocationRes::Single(ext) => { - let fragment = self.expand_invoc(invoc, &ext.kind); - self.collect_invocations(fragment, &[]) - } - InvocationRes::DeriveContainer(exts) => { - let (derives, item) = match invoc.kind { - InvocationKind::DeriveContainer { derives, item } => (derives, item), - _ => unreachable!(), - }; - if !item.derive_allowed() { - let attr = attr::find_by_name(item.attrs(), sym::derive) - .expect("`derive` attribute should exist"); - let span = attr.span; - let mut err = self.cx.struct_span_err(span, - "`derive` may only be applied to structs, enums and unions"); - if let ast::AttrStyle::Inner = attr.style { - let trait_list = derives.iter() - .map(|t| pprust::path_to_string(t)) - .collect::>(); - let suggestion = format!("#[derive({})]", trait_list.join(", ")); - err.span_suggestion( - span, "try an outer attribute", suggestion, - // We don't 𝑘𝑛𝑜𝑤 that the following item is an ADT - Applicability::MaybeIncorrect - ); - } - err.emit(); - } - - let mut item = self.fully_configure(item); - item.visit_attrs(|attrs| attrs.retain(|a| a.path != sym::derive)); - let mut helper_attrs = Vec::new(); - let mut has_copy = false; - for ext in exts { - helper_attrs.extend(&ext.helper_attrs); - has_copy |= ext.is_derive_copy; - } - // Mark derive helpers inside this item as known and used. - // FIXME: This is a hack, derive helpers should be integrated with regular name - // resolution instead. For example, helpers introduced by a derive container - // can be in scope for all code produced by that container's expansion. - item.visit_with(&mut MarkAttrs(&helper_attrs)); - if has_copy { - self.cx.resolver.add_derives(invoc.expansion_data.id, SpecialDerives::COPY); - } - - let derive_placeholders = - all_derive_placeholders.entry(invoc.expansion_data.id).or_default(); - derive_placeholders.reserve(derives.len()); - invocations.reserve(derives.len()); - for path in derives { - let expn_id = ExpnId::fresh(None); - derive_placeholders.push(NodeId::placeholder_from_expn_id(expn_id)); - invocations.push(Invocation { - kind: InvocationKind::Derive { path, item: item.clone() }, - fragment_kind: invoc.fragment_kind, - expansion_data: ExpansionData { - id: expn_id, - ..invoc.expansion_data.clone() - }, - }); - } - let fragment = invoc.fragment_kind - .expect_from_annotatables(::std::iter::once(item)); - self.collect_invocations(fragment, derive_placeholders) - } - }; - - if expanded_fragments.len() < depth { - expanded_fragments.push(Vec::new()); - } - expanded_fragments[depth - 1].push((expn_id, expanded_fragment)); - if !self.cx.ecfg.single_step { - invocations.extend(new_invocations.into_iter().rev()); - } - } - - self.cx.current_expansion = orig_expansion_data; - - // Finally incorporate all the expanded macros into the input AST fragment. - let mut placeholder_expander = PlaceholderExpander::new(self.cx, self.monotonic); - while let Some(expanded_fragments) = expanded_fragments.pop() { - for (expn_id, expanded_fragment) in expanded_fragments.into_iter().rev() { - let derive_placeholders = - all_derive_placeholders.remove(&expn_id).unwrap_or_else(Vec::new); - placeholder_expander.add(NodeId::placeholder_from_expn_id(expn_id), - expanded_fragment, derive_placeholders); - } - } - fragment_with_placeholders.mut_visit_with(&mut placeholder_expander); - fragment_with_placeholders - } - - fn resolve_imports(&mut self) { - if self.monotonic { - self.cx.resolver.resolve_imports(); - } - } - - /// Collects all macro invocations reachable at this time in this AST fragment, and replace - /// them with "placeholders" - dummy macro invocations with specially crafted `NodeId`s. - /// Then call into resolver that builds a skeleton ("reduced graph") of the fragment and - /// prepares data for resolving paths of macro invocations. - fn collect_invocations(&mut self, mut fragment: AstFragment, extra_placeholders: &[NodeId]) - -> (AstFragment, Vec) { - // Resolve `$crate`s in the fragment for pretty-printing. - self.cx.resolver.resolve_dollar_crates(); - - let invocations = { - let mut collector = InvocationCollector { - cfg: StripUnconfigured { - sess: self.cx.parse_sess, - features: self.cx.ecfg.features, - }, - cx: self.cx, - invocations: Vec::new(), - monotonic: self.monotonic, - }; - fragment.mut_visit_with(&mut collector); - collector.invocations - }; - - // FIXME: Merge `extra_placeholders` into the `fragment` as regular placeholders. - if self.monotonic { - self.cx.resolver.visit_ast_fragment_with_placeholders( - self.cx.current_expansion.id, &fragment, extra_placeholders); - } - - (fragment, invocations) - } - - fn fully_configure(&mut self, item: Annotatable) -> Annotatable { - let mut cfg = StripUnconfigured { - sess: self.cx.parse_sess, - features: self.cx.ecfg.features, - }; - // Since the item itself has already been configured by the InvocationCollector, - // we know that fold result vector will contain exactly one element - match item { - Annotatable::Item(item) => { - Annotatable::Item(cfg.flat_map_item(item).pop().unwrap()) - } - Annotatable::TraitItem(item) => { - Annotatable::TraitItem( - item.map(|item| cfg.flat_map_trait_item(item).pop().unwrap())) - } - Annotatable::ImplItem(item) => { - Annotatable::ImplItem(item.map(|item| cfg.flat_map_impl_item(item).pop().unwrap())) - } - Annotatable::ForeignItem(item) => { - Annotatable::ForeignItem( - item.map(|item| cfg.flat_map_foreign_item(item).pop().unwrap()) - ) - } - Annotatable::Stmt(stmt) => { - Annotatable::Stmt(stmt.map(|stmt| cfg.flat_map_stmt(stmt).pop().unwrap())) - } - Annotatable::Expr(mut expr) => { - Annotatable::Expr({ cfg.visit_expr(&mut expr); expr }) - } - Annotatable::Arm(arm) => { - Annotatable::Arm(cfg.flat_map_arm(arm).pop().unwrap()) - } - Annotatable::Field(field) => { - Annotatable::Field(cfg.flat_map_field(field).pop().unwrap()) - } - Annotatable::FieldPat(fp) => { - Annotatable::FieldPat(cfg.flat_map_field_pattern(fp).pop().unwrap()) - } - Annotatable::GenericParam(param) => { - Annotatable::GenericParam(cfg.flat_map_generic_param(param).pop().unwrap()) - } - Annotatable::Param(param) => { - Annotatable::Param(cfg.flat_map_param(param).pop().unwrap()) - } - Annotatable::StructField(sf) => { - Annotatable::StructField(cfg.flat_map_struct_field(sf).pop().unwrap()) - } - Annotatable::Variant(v) => { - Annotatable::Variant(cfg.flat_map_variant(v).pop().unwrap()) - } - } - } - - fn expand_invoc(&mut self, invoc: Invocation, ext: &SyntaxExtensionKind) -> AstFragment { - if self.cx.current_expansion.depth > self.cx.ecfg.recursion_limit { - let expn_data = self.cx.current_expansion.id.expn_data(); - let suggested_limit = self.cx.ecfg.recursion_limit * 2; - let mut err = self.cx.struct_span_err(expn_data.call_site, - &format!("recursion limit reached while expanding the macro `{}`", - expn_data.kind.descr())); - err.help(&format!( - "consider adding a `#![recursion_limit=\"{}\"]` attribute to your crate", - suggested_limit)); - err.emit(); - self.cx.trace_macros_diag(); - FatalError.raise(); - } - - let (fragment_kind, span) = (invoc.fragment_kind, invoc.span()); - match invoc.kind { - InvocationKind::Bang { mac, .. } => match ext { - SyntaxExtensionKind::Bang(expander) => { - self.gate_proc_macro_expansion_kind(span, fragment_kind); - let tok_result = expander.expand(self.cx, span, mac.stream()); - self.parse_ast_fragment(tok_result, fragment_kind, &mac.path, span) - } - SyntaxExtensionKind::LegacyBang(expander) => { - let prev = self.cx.current_expansion.prior_type_ascription; - self.cx.current_expansion.prior_type_ascription = mac.prior_type_ascription; - let tok_result = expander.expand(self.cx, span, mac.stream()); - let result = if let Some(result) = fragment_kind.make_from(tok_result) { - result - } else { - let msg = format!( - "non-{kind} macro in {kind} position: {path}", - kind = fragment_kind.name(), - path = pprust::path_to_string(&mac.path), - ); - self.cx.span_err(span, &msg); - self.cx.trace_macros_diag(); - fragment_kind.dummy(span) - }; - self.cx.current_expansion.prior_type_ascription = prev; - result - } - _ => unreachable!() - } - InvocationKind::Attr { attr, mut item, .. } => match ext { - SyntaxExtensionKind::Attr(expander) => { - self.gate_proc_macro_attr_item(span, &item); - let item_tok = TokenTree::token(token::Interpolated(Lrc::new(match item { - Annotatable::Item(item) => token::NtItem(item), - Annotatable::TraitItem(item) => token::NtTraitItem(item.into_inner()), - Annotatable::ImplItem(item) => token::NtImplItem(item.into_inner()), - Annotatable::ForeignItem(item) => token::NtForeignItem(item.into_inner()), - Annotatable::Stmt(stmt) => token::NtStmt(stmt.into_inner()), - Annotatable::Expr(expr) => token::NtExpr(expr), - Annotatable::Arm(..) - | Annotatable::Field(..) - | Annotatable::FieldPat(..) - | Annotatable::GenericParam(..) - | Annotatable::Param(..) - | Annotatable::StructField(..) - | Annotatable::Variant(..) - => panic!("unexpected annotatable"), - })), DUMMY_SP).into(); - let input = self.extract_proc_macro_attr_input(attr.item.tokens, span); - let tok_result = expander.expand(self.cx, span, input, item_tok); - self.parse_ast_fragment(tok_result, fragment_kind, &attr.item.path, span) - } - SyntaxExtensionKind::LegacyAttr(expander) => { - match attr.parse_meta(self.cx.parse_sess) { - Ok(meta) => { - let item = expander.expand(self.cx, span, &meta, item); - fragment_kind.expect_from_annotatables(item) - } - Err(mut err) => { - err.emit(); - fragment_kind.dummy(span) - } - } - } - SyntaxExtensionKind::NonMacroAttr { mark_used } => { - attr::mark_known(&attr); - if *mark_used { - attr::mark_used(&attr); - } - item.visit_attrs(|attrs| attrs.push(attr)); - fragment_kind.expect_from_annotatables(iter::once(item)) - } - _ => unreachable!() - } - InvocationKind::Derive { path, item } => match ext { - SyntaxExtensionKind::Derive(expander) | - SyntaxExtensionKind::LegacyDerive(expander) => { - if !item.derive_allowed() { - return fragment_kind.dummy(span); - } - let meta = ast::MetaItem { kind: ast::MetaItemKind::Word, span, path }; - let items = expander.expand(self.cx, span, &meta, item); - fragment_kind.expect_from_annotatables(items) - } - _ => unreachable!() - } - InvocationKind::DeriveContainer { .. } => unreachable!() - } - } - - fn extract_proc_macro_attr_input(&self, tokens: TokenStream, span: Span) -> TokenStream { - let mut trees = tokens.trees(); - match trees.next() { - Some(TokenTree::Delimited(_, _, tts)) => { - if trees.next().is_none() { - return tts.into() - } - } - Some(TokenTree::Token(..)) => {} - None => return TokenStream::default(), - } - self.cx.span_err(span, "custom attribute invocations must be \ - of the form `#[foo]` or `#[foo(..)]`, the macro name must only be \ - followed by a delimiter token"); - TokenStream::default() - } - - fn gate_proc_macro_attr_item(&self, span: Span, item: &Annotatable) { - let (kind, gate) = match *item { - Annotatable::Item(ref item) => { - match item.kind { - ItemKind::Mod(_) if self.cx.ecfg.proc_macro_hygiene() => return, - ItemKind::Mod(_) => ("modules", sym::proc_macro_hygiene), - _ => return, - } - } - Annotatable::TraitItem(_) => return, - Annotatable::ImplItem(_) => return, - Annotatable::ForeignItem(_) => return, - Annotatable::Stmt(_) | - Annotatable::Expr(_) if self.cx.ecfg.proc_macro_hygiene() => return, - Annotatable::Stmt(_) => ("statements", sym::proc_macro_hygiene), - Annotatable::Expr(_) => ("expressions", sym::proc_macro_hygiene), - Annotatable::Arm(..) - | Annotatable::Field(..) - | Annotatable::FieldPat(..) - | Annotatable::GenericParam(..) - | Annotatable::Param(..) - | Annotatable::StructField(..) - | Annotatable::Variant(..) - => panic!("unexpected annotatable"), - }; - emit_feature_err( - self.cx.parse_sess, - gate, - span, - GateIssue::Language, - &format!("custom attributes cannot be applied to {}", kind), - ); - } - - fn gate_proc_macro_expansion_kind(&self, span: Span, kind: AstFragmentKind) { - let kind = match kind { - AstFragmentKind::Expr | - AstFragmentKind::OptExpr => "expressions", - AstFragmentKind::Pat => "patterns", - AstFragmentKind::Stmts => "statements", - AstFragmentKind::Ty | - AstFragmentKind::Items | - AstFragmentKind::TraitItems | - AstFragmentKind::ImplItems | - AstFragmentKind::ForeignItems => return, - AstFragmentKind::Arms - | AstFragmentKind::Fields - | AstFragmentKind::FieldPats - | AstFragmentKind::GenericParams - | AstFragmentKind::Params - | AstFragmentKind::StructFields - | AstFragmentKind::Variants - => panic!("unexpected AST fragment kind"), - }; - if self.cx.ecfg.proc_macro_hygiene() { - return - } - emit_feature_err( - self.cx.parse_sess, - sym::proc_macro_hygiene, - span, - GateIssue::Language, - &format!("procedural macros cannot be expanded to {}", kind), - ); - } - - fn parse_ast_fragment( - &mut self, - toks: TokenStream, - kind: AstFragmentKind, - path: &Path, - span: Span, - ) -> AstFragment { - let mut parser = self.cx.new_parser_from_tts(toks); - match parser.parse_ast_fragment(kind, false) { - Ok(fragment) => { - parser.ensure_complete_parse(path, kind.name(), span); - fragment - } - Err(mut err) => { - err.set_span(span); - annotate_err_with_kind(&mut err, kind, span); - err.emit(); - self.cx.trace_macros_diag(); - kind.dummy(span) - } - } - } -} - -impl<'a> Parser<'a> { - pub fn parse_ast_fragment(&mut self, kind: AstFragmentKind, macro_legacy_warnings: bool) - -> PResult<'a, AstFragment> { - Ok(match kind { - AstFragmentKind::Items => { - let mut items = SmallVec::new(); - while let Some(item) = self.parse_item()? { - items.push(item); - } - AstFragment::Items(items) - } - AstFragmentKind::TraitItems => { - let mut items = SmallVec::new(); - while self.token != token::Eof { - items.push(self.parse_trait_item(&mut false)?); - } - AstFragment::TraitItems(items) - } - AstFragmentKind::ImplItems => { - let mut items = SmallVec::new(); - while self.token != token::Eof { - items.push(self.parse_impl_item(&mut false)?); - } - AstFragment::ImplItems(items) - } - AstFragmentKind::ForeignItems => { - let mut items = SmallVec::new(); - while self.token != token::Eof { - items.push(self.parse_foreign_item(DUMMY_SP)?); - } - AstFragment::ForeignItems(items) - } - AstFragmentKind::Stmts => { - let mut stmts = SmallVec::new(); - while self.token != token::Eof && - // won't make progress on a `}` - self.token != token::CloseDelim(token::Brace) { - if let Some(stmt) = self.parse_full_stmt(macro_legacy_warnings)? { - stmts.push(stmt); - } - } - AstFragment::Stmts(stmts) - } - AstFragmentKind::Expr => AstFragment::Expr(self.parse_expr()?), - AstFragmentKind::OptExpr => { - if self.token != token::Eof { - AstFragment::OptExpr(Some(self.parse_expr()?)) - } else { - AstFragment::OptExpr(None) - } - }, - AstFragmentKind::Ty => AstFragment::Ty(self.parse_ty()?), - AstFragmentKind::Pat => AstFragment::Pat(self.parse_pat(None)?), - AstFragmentKind::Arms - | AstFragmentKind::Fields - | AstFragmentKind::FieldPats - | AstFragmentKind::GenericParams - | AstFragmentKind::Params - | AstFragmentKind::StructFields - | AstFragmentKind::Variants - => panic!("unexpected AST fragment kind"), - }) - } - - pub fn ensure_complete_parse(&mut self, macro_path: &Path, kind_name: &str, span: Span) { - if self.token != token::Eof { - let msg = format!("macro expansion ignores token `{}` and any following", - self.this_token_to_string()); - // Avoid emitting backtrace info twice. - let def_site_span = self.token.span.with_ctxt(SyntaxContext::root()); - let mut err = self.struct_span_err(def_site_span, &msg); - err.span_label(span, "caused by the macro expansion here"); - let msg = format!( - "the usage of `{}!` is likely invalid in {} context", - pprust::path_to_string(¯o_path), - kind_name, - ); - err.note(&msg); - let semi_span = self.sess.source_map().next_point(span); - - let semi_full_span = semi_span.to(self.sess.source_map().next_point(semi_span)); - match self.sess.source_map().span_to_snippet(semi_full_span) { - Ok(ref snippet) if &snippet[..] != ";" && kind_name == "expression" => { - err.span_suggestion( - semi_span, - "you might be missing a semicolon here", - ";".to_owned(), - Applicability::MaybeIncorrect, - ); - } - _ => {} - } - err.emit(); - } - } -} - -struct InvocationCollector<'a, 'b> { - cx: &'a mut ExtCtxt<'b>, - cfg: StripUnconfigured<'a>, - invocations: Vec, - monotonic: bool, -} - -impl<'a, 'b> InvocationCollector<'a, 'b> { - fn collect(&mut self, fragment_kind: AstFragmentKind, kind: InvocationKind) -> AstFragment { - // Expansion data for all the collected invocations is set upon their resolution, - // with exception of the derive container case which is not resolved and can get - // its expansion data immediately. - let expn_data = match &kind { - InvocationKind::DeriveContainer { item, .. } => Some(ExpnData { - parent: self.cx.current_expansion.id, - ..ExpnData::default( - ExpnKind::Macro(MacroKind::Attr, sym::derive), - item.span(), self.cx.parse_sess.edition, - ) - }), - _ => None, - }; - let expn_id = ExpnId::fresh(expn_data); - self.invocations.push(Invocation { - kind, - fragment_kind, - expansion_data: ExpansionData { - id: expn_id, - depth: self.cx.current_expansion.depth + 1, - ..self.cx.current_expansion.clone() - }, - }); - placeholder(fragment_kind, NodeId::placeholder_from_expn_id(expn_id)) - } - - fn collect_bang(&mut self, mac: ast::Mac, span: Span, kind: AstFragmentKind) -> AstFragment { - self.collect(kind, InvocationKind::Bang { mac, span }) - } - - fn collect_attr(&mut self, - attr: Option, - derives: Vec, - item: Annotatable, - kind: AstFragmentKind, - after_derive: bool) - -> AstFragment { - self.collect(kind, match attr { - Some(attr) => InvocationKind::Attr { attr, item, derives, after_derive }, - None => InvocationKind::DeriveContainer { derives, item }, - }) - } - - fn find_attr_invoc(&self, attrs: &mut Vec, after_derive: &mut bool) - -> Option { - let attr = attrs.iter() - .position(|a| { - if a.path == sym::derive { - *after_derive = true; - } - !attr::is_known(a) && !is_builtin_attr(a) - }) - .map(|i| attrs.remove(i)); - if let Some(attr) = &attr { - if !self.cx.ecfg.custom_inner_attributes() && - attr.style == ast::AttrStyle::Inner && attr.path != sym::test { - emit_feature_err(&self.cx.parse_sess, sym::custom_inner_attributes, - attr.span, GateIssue::Language, - "non-builtin inner attributes are unstable"); - } - } - attr - } - - /// If `item` is an attr invocation, remove and return the macro attribute and derive traits. - fn classify_item(&mut self, item: &mut T) - -> (Option, Vec, /* after_derive */ bool) - where T: HasAttrs, - { - let (mut attr, mut traits, mut after_derive) = (None, Vec::new(), false); - - item.visit_attrs(|mut attrs| { - attr = self.find_attr_invoc(&mut attrs, &mut after_derive); - traits = collect_derives(&mut self.cx, &mut attrs); - }); - - (attr, traits, after_derive) - } - - /// Alternative to `classify_item()` that ignores `#[derive]` so invocations fallthrough - /// to the unused-attributes lint (making it an error on statements and expressions - /// is a breaking change) - fn classify_nonitem(&mut self, nonitem: &mut T) - -> (Option, /* after_derive */ bool) { - let (mut attr, mut after_derive) = (None, false); - - nonitem.visit_attrs(|mut attrs| { - attr = self.find_attr_invoc(&mut attrs, &mut after_derive); - }); - - (attr, after_derive) - } - - fn configure(&mut self, node: T) -> Option { - self.cfg.configure(node) - } - - // Detect use of feature-gated or invalid attributes on macro invocations - // since they will not be detected after macro expansion. - fn check_attributes(&mut self, attrs: &[ast::Attribute]) { - let features = self.cx.ecfg.features.unwrap(); - for attr in attrs.iter() { - feature_gate::check_attribute(attr, self.cx.parse_sess, features); - - // macros are expanded before any lint passes so this warning has to be hardcoded - if attr.path == sym::derive { - self.cx.struct_span_warn(attr.span, "`#[derive]` does nothing on macro invocations") - .note("this may become a hard error in a future release") - .emit(); - } - } - } -} - -impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> { - fn visit_expr(&mut self, expr: &mut P) { - self.cfg.configure_expr(expr); - visit_clobber(expr.deref_mut(), |mut expr| { - self.cfg.configure_expr_kind(&mut expr.kind); - - // ignore derives so they remain unused - let (attr, after_derive) = self.classify_nonitem(&mut expr); - - if attr.is_some() { - // Collect the invoc regardless of whether or not attributes are permitted here - // expansion will eat the attribute so it won't error later. - attr.as_ref().map(|a| self.cfg.maybe_emit_expr_attr_err(a)); - - // AstFragmentKind::Expr requires the macro to emit an expression. - return self.collect_attr(attr, vec![], Annotatable::Expr(P(expr)), - AstFragmentKind::Expr, after_derive) - .make_expr() - .into_inner() - } - - if let ast::ExprKind::Mac(mac) = expr.kind { - self.check_attributes(&expr.attrs); - self.collect_bang(mac, expr.span, AstFragmentKind::Expr) - .make_expr() - .into_inner() - } else { - noop_visit_expr(&mut expr, self); - expr - } - }); - } - - fn flat_map_arm(&mut self, arm: ast::Arm) -> SmallVec<[ast::Arm; 1]> { - let mut arm = configure!(self, arm); - - let (attr, traits, after_derive) = self.classify_item(&mut arm); - if attr.is_some() || !traits.is_empty() { - return self.collect_attr(attr, traits, Annotatable::Arm(arm), - AstFragmentKind::Arms, after_derive) - .make_arms(); - } - - noop_flat_map_arm(arm, self) - } - - fn flat_map_field(&mut self, field: ast::Field) -> SmallVec<[ast::Field; 1]> { - let mut field = configure!(self, field); - - let (attr, traits, after_derive) = self.classify_item(&mut field); - if attr.is_some() || !traits.is_empty() { - return self.collect_attr(attr, traits, Annotatable::Field(field), - AstFragmentKind::Fields, after_derive) - .make_fields(); - } - - noop_flat_map_field(field, self) - } - - fn flat_map_field_pattern(&mut self, fp: ast::FieldPat) -> SmallVec<[ast::FieldPat; 1]> { - let mut fp = configure!(self, fp); - - let (attr, traits, after_derive) = self.classify_item(&mut fp); - if attr.is_some() || !traits.is_empty() { - return self.collect_attr(attr, traits, Annotatable::FieldPat(fp), - AstFragmentKind::FieldPats, after_derive) - .make_field_patterns(); - } - - noop_flat_map_field_pattern(fp, self) - } - - fn flat_map_param(&mut self, p: ast::Param) -> SmallVec<[ast::Param; 1]> { - let mut p = configure!(self, p); - - let (attr, traits, after_derive) = self.classify_item(&mut p); - if attr.is_some() || !traits.is_empty() { - return self.collect_attr(attr, traits, Annotatable::Param(p), - AstFragmentKind::Params, after_derive) - .make_params(); - } - - noop_flat_map_param(p, self) - } - - fn flat_map_struct_field(&mut self, sf: ast::StructField) -> SmallVec<[ast::StructField; 1]> { - let mut sf = configure!(self, sf); - - let (attr, traits, after_derive) = self.classify_item(&mut sf); - if attr.is_some() || !traits.is_empty() { - return self.collect_attr(attr, traits, Annotatable::StructField(sf), - AstFragmentKind::StructFields, after_derive) - .make_struct_fields(); - } - - noop_flat_map_struct_field(sf, self) - } - - fn flat_map_variant(&mut self, variant: ast::Variant) -> SmallVec<[ast::Variant; 1]> { - let mut variant = configure!(self, variant); - - let (attr, traits, after_derive) = self.classify_item(&mut variant); - if attr.is_some() || !traits.is_empty() { - return self.collect_attr(attr, traits, Annotatable::Variant(variant), - AstFragmentKind::Variants, after_derive) - .make_variants(); - } - - noop_flat_map_variant(variant, self) - } - - fn filter_map_expr(&mut self, expr: P) -> Option> { - let expr = configure!(self, expr); - expr.filter_map(|mut expr| { - self.cfg.configure_expr_kind(&mut expr.kind); - - // Ignore derives so they remain unused. - let (attr, after_derive) = self.classify_nonitem(&mut expr); - - if attr.is_some() { - attr.as_ref().map(|a| self.cfg.maybe_emit_expr_attr_err(a)); - - return self.collect_attr(attr, vec![], Annotatable::Expr(P(expr)), - AstFragmentKind::OptExpr, after_derive) - .make_opt_expr() - .map(|expr| expr.into_inner()) - } - - if let ast::ExprKind::Mac(mac) = expr.kind { - self.check_attributes(&expr.attrs); - self.collect_bang(mac, expr.span, AstFragmentKind::OptExpr) - .make_opt_expr() - .map(|expr| expr.into_inner()) - } else { - Some({ noop_visit_expr(&mut expr, self); expr }) - } - }) - } - - fn visit_pat(&mut self, pat: &mut P) { - self.cfg.configure_pat(pat); - match pat.kind { - PatKind::Mac(_) => {} - _ => return noop_visit_pat(pat, self), - } - - visit_clobber(pat, |mut pat| { - match mem::replace(&mut pat.kind, PatKind::Wild) { - PatKind::Mac(mac) => - self.collect_bang(mac, pat.span, AstFragmentKind::Pat).make_pat(), - _ => unreachable!(), - } - }); - } - - fn flat_map_stmt(&mut self, stmt: ast::Stmt) -> SmallVec<[ast::Stmt; 1]> { - let mut stmt = configure!(self, stmt); - - // we'll expand attributes on expressions separately - if !stmt.is_expr() { - let (attr, derives, after_derive) = if stmt.is_item() { - self.classify_item(&mut stmt) - } else { - // ignore derives on non-item statements so it falls through - // to the unused-attributes lint - let (attr, after_derive) = self.classify_nonitem(&mut stmt); - (attr, vec![], after_derive) - }; - - if attr.is_some() || !derives.is_empty() { - return self.collect_attr(attr, derives, Annotatable::Stmt(P(stmt)), - AstFragmentKind::Stmts, after_derive).make_stmts(); - } - } - - if let StmtKind::Mac(mac) = stmt.kind { - let (mac, style, attrs) = mac.into_inner(); - self.check_attributes(&attrs); - let mut placeholder = self.collect_bang(mac, stmt.span, AstFragmentKind::Stmts) - .make_stmts(); - - // If this is a macro invocation with a semicolon, then apply that - // semicolon to the final statement produced by expansion. - if style == MacStmtStyle::Semicolon { - if let Some(stmt) = placeholder.pop() { - placeholder.push(stmt.add_trailing_semicolon()); - } - } - - return placeholder; - } - - // The placeholder expander gives ids to statements, so we avoid folding the id here. - let ast::Stmt { id, kind, span } = stmt; - noop_flat_map_stmt_kind(kind, self).into_iter().map(|kind| { - ast::Stmt { id, kind, span } - }).collect() - - } - - fn visit_block(&mut self, block: &mut P) { - let old_directory_ownership = self.cx.current_expansion.directory_ownership; - self.cx.current_expansion.directory_ownership = DirectoryOwnership::UnownedViaBlock; - noop_visit_block(block, self); - self.cx.current_expansion.directory_ownership = old_directory_ownership; - } - - fn flat_map_item(&mut self, item: P) -> SmallVec<[P; 1]> { - let mut item = configure!(self, item); - - let (attr, traits, after_derive) = self.classify_item(&mut item); - if attr.is_some() || !traits.is_empty() { - return self.collect_attr(attr, traits, Annotatable::Item(item), - AstFragmentKind::Items, after_derive).make_items(); - } - - match item.kind { - ast::ItemKind::Mac(..) => { - self.check_attributes(&item.attrs); - item.and_then(|item| match item.kind { - ItemKind::Mac(mac) => self.collect( - AstFragmentKind::Items, InvocationKind::Bang { mac, span: item.span } - ).make_items(), - _ => unreachable!(), - }) - } - ast::ItemKind::Mod(ast::Mod { inner, .. }) => { - if item.ident == Ident::invalid() { - return noop_flat_map_item(item, self); - } - - let orig_directory_ownership = self.cx.current_expansion.directory_ownership; - let mut module = (*self.cx.current_expansion.module).clone(); - module.mod_path.push(item.ident); - - // Detect if this is an inline module (`mod m { ... }` as opposed to `mod m;`). - // In the non-inline case, `inner` is never the dummy span (cf. `parse_item_mod`). - // Thus, if `inner` is the dummy span, we know the module is inline. - let inline_module = item.span.contains(inner) || inner.is_dummy(); - - if inline_module { - if let Some(path) = attr::first_attr_value_str_by_name(&item.attrs, sym::path) { - self.cx.current_expansion.directory_ownership = - DirectoryOwnership::Owned { relative: None }; - module.directory.push(&*path.as_str()); - } else { - module.directory.push(&*item.ident.as_str()); - } - } else { - let path = self.cx.parse_sess.source_map().span_to_unmapped_path(inner); - let mut path = match path { - FileName::Real(path) => path, - other => PathBuf::from(other.to_string()), - }; - let directory_ownership = match path.file_name().unwrap().to_str() { - Some("mod.rs") => DirectoryOwnership::Owned { relative: None }, - Some(_) => DirectoryOwnership::Owned { - relative: Some(item.ident), - }, - None => DirectoryOwnership::UnownedViaMod(false), - }; - path.pop(); - module.directory = path; - self.cx.current_expansion.directory_ownership = directory_ownership; - } - - let orig_module = - mem::replace(&mut self.cx.current_expansion.module, Rc::new(module)); - let result = noop_flat_map_item(item, self); - self.cx.current_expansion.module = orig_module; - self.cx.current_expansion.directory_ownership = orig_directory_ownership; - result - } - - _ => noop_flat_map_item(item, self), - } - } - - fn flat_map_trait_item(&mut self, item: ast::TraitItem) -> SmallVec<[ast::TraitItem; 1]> { - let mut item = configure!(self, item); - - let (attr, traits, after_derive) = self.classify_item(&mut item); - if attr.is_some() || !traits.is_empty() { - return self.collect_attr(attr, traits, Annotatable::TraitItem(P(item)), - AstFragmentKind::TraitItems, after_derive).make_trait_items() - } - - match item.kind { - ast::TraitItemKind::Macro(mac) => { - let ast::TraitItem { attrs, span, .. } = item; - self.check_attributes(&attrs); - self.collect_bang(mac, span, AstFragmentKind::TraitItems).make_trait_items() - } - _ => noop_flat_map_trait_item(item, self), - } - } - - fn flat_map_impl_item(&mut self, item: ast::ImplItem) -> SmallVec<[ast::ImplItem; 1]> { - let mut item = configure!(self, item); - - let (attr, traits, after_derive) = self.classify_item(&mut item); - if attr.is_some() || !traits.is_empty() { - return self.collect_attr(attr, traits, Annotatable::ImplItem(P(item)), - AstFragmentKind::ImplItems, after_derive).make_impl_items(); - } - - match item.kind { - ast::ImplItemKind::Macro(mac) => { - let ast::ImplItem { attrs, span, .. } = item; - self.check_attributes(&attrs); - self.collect_bang(mac, span, AstFragmentKind::ImplItems).make_impl_items() - } - _ => noop_flat_map_impl_item(item, self), - } - } - - fn visit_ty(&mut self, ty: &mut P) { - match ty.kind { - ast::TyKind::Mac(_) => {} - _ => return noop_visit_ty(ty, self), - }; - - visit_clobber(ty, |mut ty| { - match mem::replace(&mut ty.kind, ast::TyKind::Err) { - ast::TyKind::Mac(mac) => - self.collect_bang(mac, ty.span, AstFragmentKind::Ty).make_ty(), - _ => unreachable!(), - } - }); - } - - fn visit_foreign_mod(&mut self, foreign_mod: &mut ast::ForeignMod) { - self.cfg.configure_foreign_mod(foreign_mod); - noop_visit_foreign_mod(foreign_mod, self); - } - - fn flat_map_foreign_item(&mut self, mut foreign_item: ast::ForeignItem) - -> SmallVec<[ast::ForeignItem; 1]> - { - let (attr, traits, after_derive) = self.classify_item(&mut foreign_item); - - if attr.is_some() || !traits.is_empty() { - return self.collect_attr(attr, traits, Annotatable::ForeignItem(P(foreign_item)), - AstFragmentKind::ForeignItems, after_derive) - .make_foreign_items(); - } - - if let ast::ForeignItemKind::Macro(mac) = foreign_item.kind { - self.check_attributes(&foreign_item.attrs); - return self.collect_bang(mac, foreign_item.span, AstFragmentKind::ForeignItems) - .make_foreign_items(); - } - - noop_flat_map_foreign_item(foreign_item, self) - } - - fn visit_item_kind(&mut self, item: &mut ast::ItemKind) { - match item { - ast::ItemKind::MacroDef(..) => {} - _ => { - self.cfg.configure_item_kind(item); - noop_visit_item_kind(item, self); - } - } - } - - fn flat_map_generic_param( - &mut self, - param: ast::GenericParam - ) -> SmallVec<[ast::GenericParam; 1]> - { - let mut param = configure!(self, param); - - let (attr, traits, after_derive) = self.classify_item(&mut param); - if attr.is_some() || !traits.is_empty() { - return self.collect_attr(attr, traits, Annotatable::GenericParam(param), - AstFragmentKind::GenericParams, after_derive) - .make_generic_params(); - } - - noop_flat_map_generic_param(param, self) - } - - fn visit_attribute(&mut self, at: &mut ast::Attribute) { - // turn `#[doc(include="filename")]` attributes into `#[doc(include(file="filename", - // contents="file contents")]` attributes - if !at.check_name(sym::doc) { - return noop_visit_attribute(at, self); - } - - if let Some(list) = at.meta_item_list() { - if !list.iter().any(|it| it.check_name(sym::include)) { - return noop_visit_attribute(at, self); - } - - let mut items = vec![]; - - for mut it in list { - if !it.check_name(sym::include) { - items.push({ noop_visit_meta_list_item(&mut it, self); it }); - continue; - } - - if let Some(file) = it.value_str() { - let err_count = self.cx.parse_sess.span_diagnostic.err_count(); - self.check_attributes(slice::from_ref(at)); - if self.cx.parse_sess.span_diagnostic.err_count() > err_count { - // avoid loading the file if they haven't enabled the feature - return noop_visit_attribute(at, self); - } - - let filename = self.cx.resolve_path(&*file.as_str(), it.span()); - match self.cx.source_map().load_file(&filename) { - Ok(source_file) => { - let src = source_file.src.as_ref() - .expect("freshly loaded file should have a source"); - let src_interned = Symbol::intern(src.as_str()); - - let include_info = vec![ - ast::NestedMetaItem::MetaItem( - attr::mk_name_value_item_str( - Ident::with_dummy_span(sym::file), - file, - DUMMY_SP, - ), - ), - ast::NestedMetaItem::MetaItem( - attr::mk_name_value_item_str( - Ident::with_dummy_span(sym::contents), - src_interned, - DUMMY_SP, - ), - ), - ]; - - let include_ident = Ident::with_dummy_span(sym::include); - let item = attr::mk_list_item(include_ident, include_info); - items.push(ast::NestedMetaItem::MetaItem(item)); - } - Err(e) => { - let lit = it - .meta_item() - .and_then(|item| item.name_value_literal()) - .unwrap(); - - if e.kind() == ErrorKind::InvalidData { - self.cx - .struct_span_err( - lit.span, - &format!("{} wasn't a utf-8 file", filename.display()), - ) - .span_label(lit.span, "contains invalid utf-8") - .emit(); - } else { - let mut err = self.cx.struct_span_err( - lit.span, - &format!("couldn't read {}: {}", filename.display(), e), - ); - err.span_label(lit.span, "couldn't read file"); - - err.emit(); - } - } - } - } else { - let mut err = self.cx.struct_span_err( - it.span(), - &format!("expected path to external documentation"), - ); - - // Check if the user erroneously used `doc(include(...))` syntax. - let literal = it.meta_item_list().and_then(|list| { - if list.len() == 1 { - list[0].literal().map(|literal| &literal.kind) - } else { - None - } - }); - - let (path, applicability) = match &literal { - Some(LitKind::Str(path, ..)) => { - (path.to_string(), Applicability::MachineApplicable) - } - _ => (String::from(""), Applicability::HasPlaceholders), - }; - - err.span_suggestion( - it.span(), - "provide a file path with `=`", - format!("include = \"{}\"", path), - applicability, - ); - - err.emit(); - } - } - - let meta = attr::mk_list_item(Ident::with_dummy_span(sym::doc), items); - *at = attr::Attribute { - item: AttrItem { path: meta.path, tokens: meta.kind.tokens(meta.span) }, - span: at.span, - id: at.id, - style: at.style, - is_sugared_doc: false, - }; - } else { - noop_visit_attribute(at, self) - } - } - - fn visit_id(&mut self, id: &mut ast::NodeId) { - if self.monotonic { - debug_assert_eq!(*id, ast::DUMMY_NODE_ID); - *id = self.cx.resolver.next_node_id() - } - } - - fn visit_fn_decl(&mut self, mut fn_decl: &mut P) { - self.cfg.configure_fn_decl(&mut fn_decl); - noop_visit_fn_decl(fn_decl, self); - } -} - -pub struct ExpansionConfig<'feat> { - pub crate_name: String, - pub features: Option<&'feat Features>, - pub recursion_limit: usize, - pub trace_mac: bool, - pub should_test: bool, // If false, strip `#[test]` nodes - pub single_step: bool, - pub keep_macs: bool, -} - -impl<'feat> ExpansionConfig<'feat> { - pub fn default(crate_name: String) -> ExpansionConfig<'static> { - ExpansionConfig { - crate_name, - features: None, - recursion_limit: 1024, - trace_mac: false, - should_test: false, - single_step: false, - keep_macs: false, - } - } - - fn proc_macro_hygiene(&self) -> bool { - self.features.map_or(false, |features| features.proc_macro_hygiene) - } - fn custom_inner_attributes(&self) -> bool { - self.features.map_or(false, |features| features.custom_inner_attributes) - } -} diff --git a/src/libsyntax/ext/mbe.rs b/src/libsyntax/ext/mbe.rs deleted file mode 100644 index a87da791c9b..00000000000 --- a/src/libsyntax/ext/mbe.rs +++ /dev/null @@ -1,166 +0,0 @@ -//! This module implements declarative macros: old `macro_rules` and the newer -//! `macro`. Declarative macros are also known as "macro by example", and that's -//! why we call this module `mbe`. For external documentation, prefer the -//! official terminology: "declarative macros". - -crate mod transcribe; -crate mod macro_check; -crate mod macro_parser; -crate mod macro_rules; -crate mod quoted; - -use crate::ast; -use crate::parse::token::{self, Token, TokenKind}; -use crate::tokenstream::{DelimSpan}; - -use syntax_pos::{BytePos, Span}; - -use rustc_data_structures::sync::Lrc; - -/// Contains the sub-token-trees of a "delimited" token tree, such as the contents of `(`. Note -/// that the delimiter itself might be `NoDelim`. -#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug)] -struct Delimited { - delim: token::DelimToken, - tts: Vec, -} - -impl Delimited { - /// Returns a `self::TokenTree` with a `Span` corresponding to the opening delimiter. - fn open_tt(&self, span: Span) -> TokenTree { - let open_span = if span.is_dummy() { - span - } else { - span.with_hi(span.lo() + BytePos(self.delim.len() as u32)) - }; - TokenTree::token(token::OpenDelim(self.delim), open_span) - } - - /// Returns a `self::TokenTree` with a `Span` corresponding to the closing delimiter. - fn close_tt(&self, span: Span) -> TokenTree { - let close_span = if span.is_dummy() { - span - } else { - span.with_lo(span.hi() - BytePos(self.delim.len() as u32)) - }; - TokenTree::token(token::CloseDelim(self.delim), close_span) - } -} - -#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug)] -struct SequenceRepetition { - /// The sequence of token trees - tts: Vec, - /// The optional separator - separator: Option, - /// Whether the sequence can be repeated zero (*), or one or more times (+) - kleene: KleeneToken, - /// The number of `Match`s that appear in the sequence (and subsequences) - num_captures: usize, -} - -#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug, Copy)] -struct KleeneToken { - span: Span, - op: KleeneOp, -} - -impl KleeneToken { - fn new(op: KleeneOp, span: Span) -> KleeneToken { - KleeneToken { span, op } - } -} - -/// A Kleene-style [repetition operator](http://en.wikipedia.org/wiki/Kleene_star) -/// for token sequences. -#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] -enum KleeneOp { - /// Kleene star (`*`) for zero or more repetitions - ZeroOrMore, - /// Kleene plus (`+`) for one or more repetitions - OneOrMore, - /// Kleene optional (`?`) for zero or one reptitions - ZeroOrOne, -} - -/// Similar to `tokenstream::TokenTree`, except that `$i`, `$i:ident`, and `$(...)` -/// are "first-class" token trees. Useful for parsing macros. -#[derive(Debug, Clone, PartialEq, RustcEncodable, RustcDecodable)] -enum TokenTree { - Token(Token), - Delimited(DelimSpan, Lrc), - /// A kleene-style repetition sequence - Sequence(DelimSpan, Lrc), - /// e.g., `$var` - MetaVar(Span, ast::Ident), - /// e.g., `$var:expr`. This is only used in the left hand side of MBE macros. - MetaVarDecl( - Span, - ast::Ident, /* name to bind */ - ast::Ident, /* kind of nonterminal */ - ), -} - -impl TokenTree { - /// Return the number of tokens in the tree. - fn len(&self) -> usize { - match *self { - TokenTree::Delimited(_, ref delimed) => match delimed.delim { - token::NoDelim => delimed.tts.len(), - _ => delimed.tts.len() + 2, - }, - TokenTree::Sequence(_, ref seq) => seq.tts.len(), - _ => 0, - } - } - - /// Returns `true` if the given token tree is delimited. - fn is_delimited(&self) -> bool { - match *self { - TokenTree::Delimited(..) => true, - _ => false, - } - } - - /// Returns `true` if the given token tree is a token of the given kind. - fn is_token(&self, expected_kind: &TokenKind) -> bool { - match self { - TokenTree::Token(Token { kind: actual_kind, .. }) => actual_kind == expected_kind, - _ => false, - } - } - - /// Gets the `index`-th sub-token-tree. This only makes sense for delimited trees and sequences. - fn get_tt(&self, index: usize) -> TokenTree { - match (self, index) { - (&TokenTree::Delimited(_, ref delimed), _) if delimed.delim == token::NoDelim => { - delimed.tts[index].clone() - } - (&TokenTree::Delimited(span, ref delimed), _) => { - if index == 0 { - return delimed.open_tt(span.open); - } - if index == delimed.tts.len() + 1 { - return delimed.close_tt(span.close); - } - delimed.tts[index - 1].clone() - } - (&TokenTree::Sequence(_, ref seq), _) => seq.tts[index].clone(), - _ => panic!("Cannot expand a token tree"), - } - } - - /// Retrieves the `TokenTree`'s span. - fn span(&self) -> Span { - match *self { - TokenTree::Token(Token { span, .. }) - | TokenTree::MetaVar(span, _) - | TokenTree::MetaVarDecl(span, _, _) => span, - TokenTree::Delimited(span, _) | TokenTree::Sequence(span, _) => span.entire(), - } - } - - fn token(kind: TokenKind, span: Span) -> TokenTree { - TokenTree::Token(Token::new(kind, span)) - } -} diff --git a/src/libsyntax/ext/mbe/macro_check.rs b/src/libsyntax/ext/mbe/macro_check.rs deleted file mode 100644 index aabaff4e1c3..00000000000 --- a/src/libsyntax/ext/mbe/macro_check.rs +++ /dev/null @@ -1,626 +0,0 @@ -//! Checks that meta-variables in macro definition are correctly declared and used. -//! -//! # What is checked -//! -//! ## Meta-variables must not be bound twice -//! -//! ``` -//! macro_rules! foo { ($x:tt $x:tt) => { $x }; } -//! ``` -//! -//! This check is sound (no false-negative) and complete (no false-positive). -//! -//! ## Meta-variables must not be free -//! -//! ``` -//! macro_rules! foo { () => { $x }; } -//! ``` -//! -//! This check is also done at macro instantiation but only if the branch is taken. -//! -//! ## Meta-variables must repeat at least as many times as their binder -//! -//! ``` -//! macro_rules! foo { ($($x:tt)*) => { $x }; } -//! ``` -//! -//! This check is also done at macro instantiation but only if the branch is taken. -//! -//! ## Meta-variables must repeat with the same Kleene operators as their binder -//! -//! ``` -//! macro_rules! foo { ($($x:tt)+) => { $($x)* }; } -//! ``` -//! -//! This check is not done at macro instantiation. -//! -//! # Disclaimer -//! -//! In the presence of nested macros (a macro defined in a macro), those checks may have false -//! positives and false negatives. We try to detect those cases by recognizing potential macro -//! definitions in RHSes, but nested macros may be hidden through the use of particular values of -//! meta-variables. -//! -//! ## Examples of false positive -//! -//! False positives can come from cases where we don't recognize a nested macro, because it depends -//! on particular values of meta-variables. In the following example, we think both instances of -//! `$x` are free, which is a correct statement if `$name` is anything but `macro_rules`. But when -//! `$name` is `macro_rules`, like in the instantiation below, then `$x:tt` is actually a binder of -//! the nested macro and `$x` is bound to it. -//! -//! ``` -//! macro_rules! foo { ($name:ident) => { $name! bar { ($x:tt) => { $x }; } }; } -//! foo!(macro_rules); -//! ``` -//! -//! False positives can also come from cases where we think there is a nested macro while there -//! isn't. In the following example, we think `$x` is free, which is incorrect because `bar` is not -//! a nested macro since it is not evaluated as code by `stringify!`. -//! -//! ``` -//! macro_rules! foo { () => { stringify!(macro_rules! bar { () => { $x }; }) }; } -//! ``` -//! -//! ## Examples of false negative -//! -//! False negatives can come from cases where we don't recognize a meta-variable, because it depends -//! on particular values of meta-variables. In the following examples, we don't see that if `$d` is -//! instantiated with `$` then `$d z` becomes `$z` in the nested macro definition and is thus a free -//! meta-variable. Note however, that if `foo` is instantiated, then we would check the definition -//! of `bar` and would see the issue. -//! -//! ``` -//! macro_rules! foo { ($d:tt) => { macro_rules! bar { ($y:tt) => { $d z }; } }; } -//! ``` -//! -//! # How it is checked -//! -//! There are 3 main functions: `check_binders`, `check_occurrences`, and `check_nested_macro`. They -//! all need some kind of environment. -//! -//! ## Environments -//! -//! Environments are used to pass information. -//! -//! ### From LHS to RHS -//! -//! When checking a LHS with `check_binders`, we produce (and use) an environment for binders, -//! namely `Binders`. This is a mapping from binder name to information about that binder: the span -//! of the binder for error messages and the stack of Kleene operators under which it was bound in -//! the LHS. -//! -//! This environment is used by both the LHS and RHS. The LHS uses it to detect duplicate binders. -//! The RHS uses it to detect the other errors. -//! -//! ### From outer macro to inner macro -//! -//! When checking the RHS of an outer macro and we detect a nested macro definition, we push the -//! current state, namely `MacroState`, to an environment of nested macro definitions. Each state -//! stores the LHS binders when entering the macro definition as well as the stack of Kleene -//! operators under which the inner macro is defined in the RHS. -//! -//! This environment is a stack representing the nesting of macro definitions. As such, the stack of -//! Kleene operators under which a meta-variable is repeating is the concatenation of the stacks -//! stored when entering a macro definition starting from the state in which the meta-variable is -//! bound. -use crate::ast::NodeId; -use crate::early_buffered_lints::BufferedEarlyLintId; -use crate::ext::mbe::{KleeneToken, TokenTree}; -use crate::parse::token::TokenKind; -use crate::parse::token::{DelimToken, Token}; -use crate::sess::ParseSess; -use crate::symbol::{kw, sym}; - -use rustc_data_structures::fx::FxHashMap; -use smallvec::SmallVec; -use syntax_pos::{symbol::Ident, MultiSpan, Span}; - -/// Stack represented as linked list. -/// -/// Those are used for environments because they grow incrementally and are not mutable. -enum Stack<'a, T> { - /// Empty stack. - Empty, - /// A non-empty stack. - Push { - /// The top element. - top: T, - /// The previous elements. - prev: &'a Stack<'a, T>, - }, -} - -impl<'a, T> Stack<'a, T> { - /// Returns whether a stack is empty. - fn is_empty(&self) -> bool { - match *self { - Stack::Empty => true, - _ => false, - } - } - - /// Returns a new stack with an element of top. - fn push(&'a self, top: T) -> Stack<'a, T> { - Stack::Push { top, prev: self } - } -} - -impl<'a, T> Iterator for &'a Stack<'a, T> { - type Item = &'a T; - - // Iterates from top to bottom of the stack. - fn next(&mut self) -> Option<&'a T> { - match *self { - Stack::Empty => None, - Stack::Push { ref top, ref prev } => { - *self = prev; - Some(top) - } - } - } -} - -impl From<&Stack<'_, KleeneToken>> for SmallVec<[KleeneToken; 1]> { - fn from(ops: &Stack<'_, KleeneToken>) -> SmallVec<[KleeneToken; 1]> { - let mut ops: SmallVec<[KleeneToken; 1]> = ops.cloned().collect(); - // The stack is innermost on top. We want outermost first. - ops.reverse(); - ops - } -} - -/// Information attached to a meta-variable binder in LHS. -struct BinderInfo { - /// The span of the meta-variable in LHS. - span: Span, - /// The stack of Kleene operators (outermost first). - ops: SmallVec<[KleeneToken; 1]>, -} - -/// An environment of meta-variables to their binder information. -type Binders = FxHashMap; - -/// The state at which we entered a macro definition in the RHS of another macro definition. -struct MacroState<'a> { - /// The binders of the branch where we entered the macro definition. - binders: &'a Binders, - /// The stack of Kleene operators (outermost first) where we entered the macro definition. - ops: SmallVec<[KleeneToken; 1]>, -} - -/// Checks that meta-variables are used correctly in a macro definition. -/// -/// Arguments: -/// - `sess` is used to emit diagnostics and lints -/// - `node_id` is used to emit lints -/// - `span` is used when no spans are available -/// - `lhses` and `rhses` should have the same length and represent the macro definition -pub(super) fn check_meta_variables( - sess: &ParseSess, - node_id: NodeId, - span: Span, - lhses: &[TokenTree], - rhses: &[TokenTree], -) -> bool { - if lhses.len() != rhses.len() { - sess.span_diagnostic.span_bug(span, "length mismatch between LHSes and RHSes") - } - let mut valid = true; - for (lhs, rhs) in lhses.iter().zip(rhses.iter()) { - let mut binders = Binders::default(); - check_binders(sess, node_id, lhs, &Stack::Empty, &mut binders, &Stack::Empty, &mut valid); - check_occurrences(sess, node_id, rhs, &Stack::Empty, &binders, &Stack::Empty, &mut valid); - } - valid -} - -/// Checks `lhs` as part of the LHS of a macro definition, extends `binders` with new binders, and -/// sets `valid` to false in case of errors. -/// -/// Arguments: -/// - `sess` is used to emit diagnostics and lints -/// - `node_id` is used to emit lints -/// - `lhs` is checked as part of a LHS -/// - `macros` is the stack of possible outer macros -/// - `binders` contains the binders of the LHS -/// - `ops` is the stack of Kleene operators from the LHS -/// - `valid` is set in case of errors -fn check_binders( - sess: &ParseSess, - node_id: NodeId, - lhs: &TokenTree, - macros: &Stack<'_, MacroState<'_>>, - binders: &mut Binders, - ops: &Stack<'_, KleeneToken>, - valid: &mut bool, -) { - match *lhs { - TokenTree::Token(..) => {} - // This can only happen when checking a nested macro because this LHS is then in the RHS of - // the outer macro. See ui/macros/macro-of-higher-order.rs where $y:$fragment in the - // LHS of the nested macro (and RHS of the outer macro) is parsed as MetaVar(y) Colon - // MetaVar(fragment) and not as MetaVarDecl(y, fragment). - TokenTree::MetaVar(span, name) => { - if macros.is_empty() { - sess.span_diagnostic.span_bug(span, "unexpected MetaVar in lhs"); - } - // There are 3 possibilities: - if let Some(prev_info) = binders.get(&name) { - // 1. The meta-variable is already bound in the current LHS: This is an error. - let mut span = MultiSpan::from_span(span); - span.push_span_label(prev_info.span, "previous declaration".into()); - buffer_lint(sess, span, node_id, "duplicate matcher binding"); - } else if get_binder_info(macros, binders, name).is_none() { - // 2. The meta-variable is free: This is a binder. - binders.insert(name, BinderInfo { span, ops: ops.into() }); - } else { - // 3. The meta-variable is bound: This is an occurrence. - check_occurrences(sess, node_id, lhs, macros, binders, ops, valid); - } - } - // Similarly, this can only happen when checking a toplevel macro. - TokenTree::MetaVarDecl(span, name, _kind) => { - if !macros.is_empty() { - sess.span_diagnostic.span_bug(span, "unexpected MetaVarDecl in nested lhs"); - } - if let Some(prev_info) = get_binder_info(macros, binders, name) { - // Duplicate binders at the top-level macro definition are errors. The lint is only - // for nested macro definitions. - sess.span_diagnostic - .struct_span_err(span, "duplicate matcher binding") - .span_note(prev_info.span, "previous declaration was here") - .emit(); - *valid = false; - } else { - binders.insert(name, BinderInfo { span, ops: ops.into() }); - } - } - TokenTree::Delimited(_, ref del) => { - for tt in &del.tts { - check_binders(sess, node_id, tt, macros, binders, ops, valid); - } - } - TokenTree::Sequence(_, ref seq) => { - let ops = ops.push(seq.kleene); - for tt in &seq.tts { - check_binders(sess, node_id, tt, macros, binders, &ops, valid); - } - } - } -} - -/// Returns the binder information of a meta-variable. -/// -/// Arguments: -/// - `macros` is the stack of possible outer macros -/// - `binders` contains the current binders -/// - `name` is the name of the meta-variable we are looking for -fn get_binder_info<'a>( - mut macros: &'a Stack<'a, MacroState<'a>>, - binders: &'a Binders, - name: Ident, -) -> Option<&'a BinderInfo> { - binders.get(&name).or_else(|| macros.find_map(|state| state.binders.get(&name))) -} - -/// Checks `rhs` as part of the RHS of a macro definition and sets `valid` to false in case of -/// errors. -/// -/// Arguments: -/// - `sess` is used to emit diagnostics and lints -/// - `node_id` is used to emit lints -/// - `rhs` is checked as part of a RHS -/// - `macros` is the stack of possible outer macros -/// - `binders` contains the binders of the associated LHS -/// - `ops` is the stack of Kleene operators from the RHS -/// - `valid` is set in case of errors -fn check_occurrences( - sess: &ParseSess, - node_id: NodeId, - rhs: &TokenTree, - macros: &Stack<'_, MacroState<'_>>, - binders: &Binders, - ops: &Stack<'_, KleeneToken>, - valid: &mut bool, -) { - match *rhs { - TokenTree::Token(..) => {} - TokenTree::MetaVarDecl(span, _name, _kind) => { - sess.span_diagnostic.span_bug(span, "unexpected MetaVarDecl in rhs") - } - TokenTree::MetaVar(span, name) => { - check_ops_is_prefix(sess, node_id, macros, binders, ops, span, name); - } - TokenTree::Delimited(_, ref del) => { - check_nested_occurrences(sess, node_id, &del.tts, macros, binders, ops, valid); - } - TokenTree::Sequence(_, ref seq) => { - let ops = ops.push(seq.kleene); - check_nested_occurrences(sess, node_id, &seq.tts, macros, binders, &ops, valid); - } - } -} - -/// Represents the processed prefix of a nested macro. -#[derive(Clone, Copy, PartialEq, Eq)] -enum NestedMacroState { - /// Nothing that matches a nested macro definition was processed yet. - Empty, - /// The token `macro_rules` was processed. - MacroRules, - /// The tokens `macro_rules!` were processed. - MacroRulesNot, - /// The tokens `macro_rules!` followed by a name were processed. The name may be either directly - /// an identifier or a meta-variable (that hopefully would be instantiated by an identifier). - MacroRulesNotName, - /// The keyword `macro` was processed. - Macro, - /// The keyword `macro` followed by a name was processed. - MacroName, - /// The keyword `macro` followed by a name and a token delimited by parentheses was processed. - MacroNameParen, -} - -/// Checks `tts` as part of the RHS of a macro definition, tries to recognize nested macro -/// definitions, and sets `valid` to false in case of errors. -/// -/// Arguments: -/// - `sess` is used to emit diagnostics and lints -/// - `node_id` is used to emit lints -/// - `tts` is checked as part of a RHS and may contain macro definitions -/// - `macros` is the stack of possible outer macros -/// - `binders` contains the binders of the associated LHS -/// - `ops` is the stack of Kleene operators from the RHS -/// - `valid` is set in case of errors -fn check_nested_occurrences( - sess: &ParseSess, - node_id: NodeId, - tts: &[TokenTree], - macros: &Stack<'_, MacroState<'_>>, - binders: &Binders, - ops: &Stack<'_, KleeneToken>, - valid: &mut bool, -) { - let mut state = NestedMacroState::Empty; - let nested_macros = macros.push(MacroState { binders, ops: ops.into() }); - let mut nested_binders = Binders::default(); - for tt in tts { - match (state, tt) { - ( - NestedMacroState::Empty, - &TokenTree::Token(Token { kind: TokenKind::Ident(name, false), .. }), - ) => { - if name == sym::macro_rules { - state = NestedMacroState::MacroRules; - } else if name == kw::Macro { - state = NestedMacroState::Macro; - } - } - ( - NestedMacroState::MacroRules, - &TokenTree::Token(Token { kind: TokenKind::Not, .. }), - ) => { - state = NestedMacroState::MacroRulesNot; - } - ( - NestedMacroState::MacroRulesNot, - &TokenTree::Token(Token { kind: TokenKind::Ident(..), .. }), - ) => { - state = NestedMacroState::MacroRulesNotName; - } - (NestedMacroState::MacroRulesNot, &TokenTree::MetaVar(..)) => { - state = NestedMacroState::MacroRulesNotName; - // We check that the meta-variable is correctly used. - check_occurrences(sess, node_id, tt, macros, binders, ops, valid); - } - (NestedMacroState::MacroRulesNotName, &TokenTree::Delimited(_, ref del)) - | (NestedMacroState::MacroName, &TokenTree::Delimited(_, ref del)) - if del.delim == DelimToken::Brace => - { - let legacy = state == NestedMacroState::MacroRulesNotName; - state = NestedMacroState::Empty; - let rest = - check_nested_macro(sess, node_id, legacy, &del.tts, &nested_macros, valid); - // If we did not check the whole macro definition, then check the rest as if outside - // the macro definition. - check_nested_occurrences( - sess, - node_id, - &del.tts[rest..], - macros, - binders, - ops, - valid, - ); - } - ( - NestedMacroState::Macro, - &TokenTree::Token(Token { kind: TokenKind::Ident(..), .. }), - ) => { - state = NestedMacroState::MacroName; - } - (NestedMacroState::Macro, &TokenTree::MetaVar(..)) => { - state = NestedMacroState::MacroName; - // We check that the meta-variable is correctly used. - check_occurrences(sess, node_id, tt, macros, binders, ops, valid); - } - (NestedMacroState::MacroName, &TokenTree::Delimited(_, ref del)) - if del.delim == DelimToken::Paren => - { - state = NestedMacroState::MacroNameParen; - nested_binders = Binders::default(); - check_binders( - sess, - node_id, - tt, - &nested_macros, - &mut nested_binders, - &Stack::Empty, - valid, - ); - } - (NestedMacroState::MacroNameParen, &TokenTree::Delimited(_, ref del)) - if del.delim == DelimToken::Brace => - { - state = NestedMacroState::Empty; - check_occurrences( - sess, - node_id, - tt, - &nested_macros, - &nested_binders, - &Stack::Empty, - valid, - ); - } - (_, ref tt) => { - state = NestedMacroState::Empty; - check_occurrences(sess, node_id, tt, macros, binders, ops, valid); - } - } - } -} - -/// Checks the body of nested macro, returns where the check stopped, and sets `valid` to false in -/// case of errors. -/// -/// The token trees are checked as long as they look like a list of (LHS) => {RHS} token trees. This -/// check is a best-effort to detect a macro definition. It returns the position in `tts` where we -/// stopped checking because we detected we were not in a macro definition anymore. -/// -/// Arguments: -/// - `sess` is used to emit diagnostics and lints -/// - `node_id` is used to emit lints -/// - `legacy` specifies whether the macro is legacy -/// - `tts` is checked as a list of (LHS) => {RHS} -/// - `macros` is the stack of outer macros -/// - `valid` is set in case of errors -fn check_nested_macro( - sess: &ParseSess, - node_id: NodeId, - legacy: bool, - tts: &[TokenTree], - macros: &Stack<'_, MacroState<'_>>, - valid: &mut bool, -) -> usize { - let n = tts.len(); - let mut i = 0; - let separator = if legacy { TokenKind::Semi } else { TokenKind::Comma }; - loop { - // We expect 3 token trees: `(LHS) => {RHS}`. The separator is checked after. - if i + 2 >= n - || !tts[i].is_delimited() - || !tts[i + 1].is_token(&TokenKind::FatArrow) - || !tts[i + 2].is_delimited() - { - break; - } - let lhs = &tts[i]; - let rhs = &tts[i + 2]; - let mut binders = Binders::default(); - check_binders(sess, node_id, lhs, macros, &mut binders, &Stack::Empty, valid); - check_occurrences(sess, node_id, rhs, macros, &binders, &Stack::Empty, valid); - // Since the last semicolon is optional for legacy macros and decl_macro are not terminated, - // we increment our checked position by how many token trees we already checked (the 3 - // above) before checking for the separator. - i += 3; - if i == n || !tts[i].is_token(&separator) { - break; - } - // We increment our checked position for the semicolon. - i += 1; - } - i -} - -/// Checks that a meta-variable occurrence is valid. -/// -/// Arguments: -/// - `sess` is used to emit diagnostics and lints -/// - `node_id` is used to emit lints -/// - `macros` is the stack of possible outer macros -/// - `binders` contains the binders of the associated LHS -/// - `ops` is the stack of Kleene operators from the RHS -/// - `span` is the span of the meta-variable to check -/// - `name` is the name of the meta-variable to check -fn check_ops_is_prefix( - sess: &ParseSess, - node_id: NodeId, - macros: &Stack<'_, MacroState<'_>>, - binders: &Binders, - ops: &Stack<'_, KleeneToken>, - span: Span, - name: Ident, -) { - let macros = macros.push(MacroState { binders, ops: ops.into() }); - // Accumulates the stacks the operators of each state until (and including when) the - // meta-variable is found. The innermost stack is first. - let mut acc: SmallVec<[&SmallVec<[KleeneToken; 1]>; 1]> = SmallVec::new(); - for state in ¯os { - acc.push(&state.ops); - if let Some(binder) = state.binders.get(&name) { - // This variable concatenates the stack of operators from the RHS of the LHS where the - // meta-variable was defined to where it is used (in possibly nested macros). The - // outermost operator is first. - let mut occurrence_ops: SmallVec<[KleeneToken; 2]> = SmallVec::new(); - // We need to iterate from the end to start with outermost stack. - for ops in acc.iter().rev() { - occurrence_ops.extend_from_slice(ops); - } - ops_is_prefix(sess, node_id, span, name, &binder.ops, &occurrence_ops); - return; - } - } - buffer_lint(sess, span.into(), node_id, &format!("unknown macro variable `{}`", name)); -} - -/// Returns whether `binder_ops` is a prefix of `occurrence_ops`. -/// -/// The stack of Kleene operators of a meta-variable occurrence just needs to have the stack of -/// Kleene operators of its binder as a prefix. -/// -/// Consider $i in the following example: -/// -/// ( $( $i:ident = $($j:ident),+ );* ) => { $($( $i += $j; )+)* } -/// -/// It occurs under the Kleene stack ["*", "+"] and is bound under ["*"] only. -/// -/// Arguments: -/// - `sess` is used to emit diagnostics and lints -/// - `node_id` is used to emit lints -/// - `span` is the span of the meta-variable being check -/// - `name` is the name of the meta-variable being check -/// - `binder_ops` is the stack of Kleene operators for the binder -/// - `occurrence_ops` is the stack of Kleene operators for the occurrence -fn ops_is_prefix( - sess: &ParseSess, - node_id: NodeId, - span: Span, - name: Ident, - binder_ops: &[KleeneToken], - occurrence_ops: &[KleeneToken], -) { - for (i, binder) in binder_ops.iter().enumerate() { - if i >= occurrence_ops.len() { - let mut span = MultiSpan::from_span(span); - span.push_span_label(binder.span, "expected repetition".into()); - let message = &format!("variable '{}' is still repeating at this depth", name); - buffer_lint(sess, span, node_id, message); - return; - } - let occurrence = &occurrence_ops[i]; - if occurrence.op != binder.op { - let mut span = MultiSpan::from_span(span); - span.push_span_label(binder.span, "expected repetition".into()); - span.push_span_label(occurrence.span, "conflicting repetition".into()); - let message = "meta-variable repeats with different Kleene operator"; - buffer_lint(sess, span, node_id, message); - return; - } - } -} - -fn buffer_lint(sess: &ParseSess, span: MultiSpan, node_id: NodeId, message: &str) { - sess.buffer_lint(BufferedEarlyLintId::MetaVariableMisuse, span, node_id, message); -} diff --git a/src/libsyntax/ext/mbe/macro_parser.rs b/src/libsyntax/ext/mbe/macro_parser.rs deleted file mode 100644 index ff382c316ff..00000000000 --- a/src/libsyntax/ext/mbe/macro_parser.rs +++ /dev/null @@ -1,943 +0,0 @@ -//! This is an NFA-based parser, which calls out to the main rust parser for named non-terminals -//! (which it commits to fully when it hits one in a grammar). There's a set of current NFA threads -//! and a set of next ones. Instead of NTs, we have a special case for Kleene star. The big-O, in -//! pathological cases, is worse than traditional use of NFA or Earley parsing, but it's an easier -//! fit for Macro-by-Example-style rules. -//! -//! (In order to prevent the pathological case, we'd need to lazily construct the resulting -//! `NamedMatch`es at the very end. It'd be a pain, and require more memory to keep around old -//! items, but it would also save overhead) -//! -//! We don't say this parser uses the Earley algorithm, because it's unnecessarily inaccurate. -//! The macro parser restricts itself to the features of finite state automata. Earley parsers -//! can be described as an extension of NFAs with completion rules, prediction rules, and recursion. -//! -//! Quick intro to how the parser works: -//! -//! A 'position' is a dot in the middle of a matcher, usually represented as a -//! dot. For example `· a $( a )* a b` is a position, as is `a $( · a )* a b`. -//! -//! The parser walks through the input a character at a time, maintaining a list -//! of threads consistent with the current position in the input string: `cur_items`. -//! -//! As it processes them, it fills up `eof_items` with threads that would be valid if -//! the macro invocation is now over, `bb_items` with threads that are waiting on -//! a Rust non-terminal like `$e:expr`, and `next_items` with threads that are waiting -//! on a particular token. Most of the logic concerns moving the · through the -//! repetitions indicated by Kleene stars. The rules for moving the · without -//! consuming any input are called epsilon transitions. It only advances or calls -//! out to the real Rust parser when no `cur_items` threads remain. -//! -//! Example: -//! -//! ```text, ignore -//! Start parsing a a a a b against [· a $( a )* a b]. -//! -//! Remaining input: a a a a b -//! next: [· a $( a )* a b] -//! -//! - - - Advance over an a. - - - -//! -//! Remaining input: a a a b -//! cur: [a · $( a )* a b] -//! Descend/Skip (first item). -//! next: [a $( · a )* a b] [a $( a )* · a b]. -//! -//! - - - Advance over an a. - - - -//! -//! Remaining input: a a b -//! cur: [a $( a · )* a b] [a $( a )* a · b] -//! Follow epsilon transition: Finish/Repeat (first item) -//! next: [a $( a )* · a b] [a $( · a )* a b] [a $( a )* a · b] -//! -//! - - - Advance over an a. - - - (this looks exactly like the last step) -//! -//! Remaining input: a b -//! cur: [a $( a · )* a b] [a $( a )* a · b] -//! Follow epsilon transition: Finish/Repeat (first item) -//! next: [a $( a )* · a b] [a $( · a )* a b] [a $( a )* a · b] -//! -//! - - - Advance over an a. - - - (this looks exactly like the last step) -//! -//! Remaining input: b -//! cur: [a $( a · )* a b] [a $( a )* a · b] -//! Follow epsilon transition: Finish/Repeat (first item) -//! next: [a $( a )* · a b] [a $( · a )* a b] [a $( a )* a · b] -//! -//! - - - Advance over a b. - - - -//! -//! Remaining input: '' -//! eof: [a $( a )* a b ·] -//! ``` - -crate use NamedMatch::*; -crate use ParseResult::*; -use TokenTreeOrTokenTreeSlice::*; - -use crate::ast::{Ident, Name}; -use crate::ext::mbe::{self, TokenTree}; -use crate::parse::{Directory, PResult}; -use crate::parse::parser::{Parser, PathStyle}; -use crate::parse::token::{self, DocComment, Nonterminal, Token}; -use crate::print::pprust; -use crate::sess::ParseSess; -use crate::symbol::{kw, sym, Symbol}; -use crate::tokenstream::{DelimSpan, TokenStream}; - -use errors::FatalError; -use smallvec::{smallvec, SmallVec}; -use syntax_pos::Span; - -use rustc_data_structures::fx::FxHashMap; -use rustc_data_structures::sync::Lrc; -use std::collections::hash_map::Entry::{Occupied, Vacant}; -use std::mem; -use std::ops::{Deref, DerefMut}; - -// To avoid costly uniqueness checks, we require that `MatchSeq` always has a nonempty body. - -/// Either a sequence of token trees or a single one. This is used as the representation of the -/// sequence of tokens that make up a matcher. -#[derive(Clone)] -enum TokenTreeOrTokenTreeSlice<'tt> { - Tt(TokenTree), - TtSeq(&'tt [TokenTree]), -} - -impl<'tt> TokenTreeOrTokenTreeSlice<'tt> { - /// Returns the number of constituent top-level token trees of `self` (top-level in that it - /// will not recursively descend into subtrees). - fn len(&self) -> usize { - match *self { - TtSeq(ref v) => v.len(), - Tt(ref tt) => tt.len(), - } - } - - /// The `index`-th token tree of `self`. - fn get_tt(&self, index: usize) -> TokenTree { - match *self { - TtSeq(ref v) => v[index].clone(), - Tt(ref tt) => tt.get_tt(index), - } - } -} - -/// An unzipping of `TokenTree`s... see the `stack` field of `MatcherPos`. -/// -/// This is used by `inner_parse_loop` to keep track of delimited submatchers that we have -/// descended into. -#[derive(Clone)] -struct MatcherTtFrame<'tt> { - /// The "parent" matcher that we are descending into. - elts: TokenTreeOrTokenTreeSlice<'tt>, - /// The position of the "dot" in `elts` at the time we descended. - idx: usize, -} - -type NamedMatchVec = SmallVec<[NamedMatch; 4]>; - -/// Represents a single "position" (aka "matcher position", aka "item"), as -/// described in the module documentation. -/// -/// Here: -/// -/// - `'root` represents the lifetime of the stack slot that holds the root -/// `MatcherPos`. As described in `MatcherPosHandle`, the root `MatcherPos` -/// structure is stored on the stack, but subsequent instances are put into -/// the heap. -/// - `'tt` represents the lifetime of the token trees that this matcher -/// position refers to. -/// -/// It is important to distinguish these two lifetimes because we have a -/// `SmallVec>` below, and the destructor of -/// that is considered to possibly access the data from its elements (it lacks -/// a `#[may_dangle]` attribute). As a result, the compiler needs to know that -/// all the elements in that `SmallVec` strictly outlive the root stack slot -/// lifetime. By separating `'tt` from `'root`, we can show that. -#[derive(Clone)] -struct MatcherPos<'root, 'tt> { - /// The token or sequence of tokens that make up the matcher - top_elts: TokenTreeOrTokenTreeSlice<'tt>, - - /// The position of the "dot" in this matcher - idx: usize, - - /// The first span of source that the beginning of this matcher corresponds to. In other - /// words, the token in the source whose span is `sp_open` is matched against the first token of - /// the matcher. - sp_open: Span, - - /// For each named metavar in the matcher, we keep track of token trees matched against the - /// metavar by the black box parser. In particular, there may be more than one match per - /// metavar if we are in a repetition (each repetition matches each of the variables). - /// Moreover, matchers and repetitions can be nested; the `matches` field is shared (hence the - /// `Rc`) among all "nested" matchers. `match_lo`, `match_cur`, and `match_hi` keep track of - /// the current position of the `self` matcher position in the shared `matches` list. - /// - /// Also, note that while we are descending into a sequence, matchers are given their own - /// `matches` vector. Only once we reach the end of a full repetition of the sequence do we add - /// all bound matches from the submatcher into the shared top-level `matches` vector. If `sep` - /// and `up` are `Some`, then `matches` is _not_ the shared top-level list. Instead, if one - /// wants the shared `matches`, one should use `up.matches`. - matches: Box<[Lrc]>, - /// The position in `matches` corresponding to the first metavar in this matcher's sequence of - /// token trees. In other words, the first metavar in the first token of `top_elts` corresponds - /// to `matches[match_lo]`. - match_lo: usize, - /// The position in `matches` corresponding to the metavar we are currently trying to match - /// against the source token stream. `match_lo <= match_cur <= match_hi`. - match_cur: usize, - /// Similar to `match_lo` except `match_hi` is the position in `matches` of the _last_ metavar - /// in this matcher. - match_hi: usize, - - // The following fields are used if we are matching a repetition. If we aren't, they should be - // `None`. - - /// The KleeneOp of this sequence if we are in a repetition. - seq_op: Option, - - /// The separator if we are in a repetition. - sep: Option, - - /// The "parent" matcher position if we are in a repetition. That is, the matcher position just - /// before we enter the sequence. - up: Option>, - - /// Specifically used to "unzip" token trees. By "unzip", we mean to unwrap the delimiters from - /// a delimited token tree (e.g., something wrapped in `(` `)`) or to get the contents of a doc - /// comment... - /// - /// When matching against matchers with nested delimited submatchers (e.g., `pat ( pat ( .. ) - /// pat ) pat`), we need to keep track of the matchers we are descending into. This stack does - /// that where the bottom of the stack is the outermost matcher. - /// Also, throughout the comments, this "descent" is often referred to as "unzipping"... - stack: SmallVec<[MatcherTtFrame<'tt>; 1]>, -} - -impl<'root, 'tt> MatcherPos<'root, 'tt> { - /// Adds `m` as a named match for the `idx`-th metavar. - fn push_match(&mut self, idx: usize, m: NamedMatch) { - let matches = Lrc::make_mut(&mut self.matches[idx]); - matches.push(m); - } -} - -// Lots of MatcherPos instances are created at runtime. Allocating them on the -// heap is slow. Furthermore, using SmallVec to allocate them all -// on the stack is also slow, because MatcherPos is quite a large type and -// instances get moved around a lot between vectors, which requires lots of -// slow memcpy calls. -// -// Therefore, the initial MatcherPos is always allocated on the stack, -// subsequent ones (of which there aren't that many) are allocated on the heap, -// and this type is used to encapsulate both cases. -enum MatcherPosHandle<'root, 'tt> { - Ref(&'root mut MatcherPos<'root, 'tt>), - Box(Box>), -} - -impl<'root, 'tt> Clone for MatcherPosHandle<'root, 'tt> { - // This always produces a new Box. - fn clone(&self) -> Self { - MatcherPosHandle::Box(match *self { - MatcherPosHandle::Ref(ref r) => Box::new((**r).clone()), - MatcherPosHandle::Box(ref b) => b.clone(), - }) - } -} - -impl<'root, 'tt> Deref for MatcherPosHandle<'root, 'tt> { - type Target = MatcherPos<'root, 'tt>; - fn deref(&self) -> &Self::Target { - match *self { - MatcherPosHandle::Ref(ref r) => r, - MatcherPosHandle::Box(ref b) => b, - } - } -} - -impl<'root, 'tt> DerefMut for MatcherPosHandle<'root, 'tt> { - fn deref_mut(&mut self) -> &mut MatcherPos<'root, 'tt> { - match *self { - MatcherPosHandle::Ref(ref mut r) => r, - MatcherPosHandle::Box(ref mut b) => b, - } - } -} - -/// Represents the possible results of an attempted parse. -crate enum ParseResult { - /// Parsed successfully. - Success(T), - /// Arm failed to match. If the second parameter is `token::Eof`, it indicates an unexpected - /// end of macro invocation. Otherwise, it indicates that no rules expected the given token. - Failure(Token, &'static str), - /// Fatal error (malformed macro?). Abort compilation. - Error(syntax_pos::Span, String), -} - -/// A `ParseResult` where the `Success` variant contains a mapping of `Ident`s to `NamedMatch`es. -/// This represents the mapping of metavars to the token trees they bind to. -crate type NamedParseResult = ParseResult>; - -/// Count how many metavars are named in the given matcher `ms`. -pub(super) fn count_names(ms: &[TokenTree]) -> usize { - ms.iter().fold(0, |count, elt| { - count + match *elt { - TokenTree::Sequence(_, ref seq) => seq.num_captures, - TokenTree::Delimited(_, ref delim) => count_names(&delim.tts), - TokenTree::MetaVar(..) => 0, - TokenTree::MetaVarDecl(..) => 1, - TokenTree::Token(..) => 0, - } - }) -} - -/// `len` `Vec`s (initially shared and empty) that will store matches of metavars. -fn create_matches(len: usize) -> Box<[Lrc]> { - if len == 0 { - vec![] - } else { - let empty_matches = Lrc::new(SmallVec::new()); - vec![empty_matches; len] - }.into_boxed_slice() -} - -/// Generates the top-level matcher position in which the "dot" is before the first token of the -/// matcher `ms` and we are going to start matching at the span `open` in the source. -fn initial_matcher_pos<'root, 'tt>(ms: &'tt [TokenTree], open: Span) -> MatcherPos<'root, 'tt> { - let match_idx_hi = count_names(ms); - let matches = create_matches(match_idx_hi); - MatcherPos { - // Start with the top level matcher given to us - top_elts: TtSeq(ms), // "elts" is an abbr. for "elements" - // The "dot" is before the first token of the matcher - idx: 0, - // We start matching at the span `open` in the source code - sp_open: open, - - // Initialize `matches` to a bunch of empty `Vec`s -- one for each metavar in `top_elts`. - // `match_lo` for `top_elts` is 0 and `match_hi` is `matches.len()`. `match_cur` is 0 since - // we haven't actually matched anything yet. - matches, - match_lo: 0, - match_cur: 0, - match_hi: match_idx_hi, - - // Haven't descended into any delimiters, so empty stack - stack: smallvec![], - - // Haven't descended into any sequences, so both of these are `None`. - seq_op: None, - sep: None, - up: None, - } -} - -/// `NamedMatch` is a pattern-match result for a single `token::MATCH_NONTERMINAL`: -/// so it is associated with a single ident in a parse, and all -/// `MatchedNonterminal`s in the `NamedMatch` have the same non-terminal type -/// (expr, item, etc). Each leaf in a single `NamedMatch` corresponds to a -/// single `token::MATCH_NONTERMINAL` in the `TokenTree` that produced it. -/// -/// The in-memory structure of a particular `NamedMatch` represents the match -/// that occurred when a particular subset of a matcher was applied to a -/// particular token tree. -/// -/// The width of each `MatchedSeq` in the `NamedMatch`, and the identity of -/// the `MatchedNonterminal`s, will depend on the token tree it was applied -/// to: each `MatchedSeq` corresponds to a single `TTSeq` in the originating -/// token tree. The depth of the `NamedMatch` structure will therefore depend -/// only on the nesting depth of `ast::TTSeq`s in the originating -/// token tree it was derived from. -#[derive(Debug, Clone)] -crate enum NamedMatch { - MatchedSeq(Lrc, DelimSpan), - MatchedNonterminal(Lrc), -} - -/// Takes a sequence of token trees `ms` representing a matcher which successfully matched input -/// and an iterator of items that matched input and produces a `NamedParseResult`. -fn nameize>( - sess: &ParseSess, - ms: &[TokenTree], - mut res: I, -) -> NamedParseResult { - // Recursively descend into each type of matcher (e.g., sequences, delimited, metavars) and make - // sure that each metavar has _exactly one_ binding. If a metavar does not have exactly one - // binding, then there is an error. If it does, then we insert the binding into the - // `NamedParseResult`. - fn n_rec>( - sess: &ParseSess, - m: &TokenTree, - res: &mut I, - ret_val: &mut FxHashMap, - ) -> Result<(), (syntax_pos::Span, String)> { - match *m { - TokenTree::Sequence(_, ref seq) => for next_m in &seq.tts { - n_rec(sess, next_m, res.by_ref(), ret_val)? - }, - TokenTree::Delimited(_, ref delim) => for next_m in &delim.tts { - n_rec(sess, next_m, res.by_ref(), ret_val)?; - }, - TokenTree::MetaVarDecl(span, _, id) if id.name == kw::Invalid => { - if sess.missing_fragment_specifiers.borrow_mut().remove(&span) { - return Err((span, "missing fragment specifier".to_string())); - } - } - TokenTree::MetaVarDecl(sp, bind_name, _) => { - match ret_val.entry(bind_name) { - Vacant(spot) => { - spot.insert(res.next().unwrap()); - } - Occupied(..) => { - return Err((sp, format!("duplicated bind name: {}", bind_name))) - } - } - } - TokenTree::MetaVar(..) | TokenTree::Token(..) => (), - } - - Ok(()) - } - - let mut ret_val = FxHashMap::default(); - for m in ms { - match n_rec(sess, m, res.by_ref(), &mut ret_val) { - Ok(_) => {} - Err((sp, msg)) => return Error(sp, msg), - } - } - - Success(ret_val) -} - -/// Performs a token equality check, ignoring syntax context (that is, an unhygienic comparison) -fn token_name_eq(t1: &Token, t2: &Token) -> bool { - if let (Some((ident1, is_raw1)), Some((ident2, is_raw2))) = (t1.ident(), t2.ident()) { - ident1.name == ident2.name && is_raw1 == is_raw2 - } else if let (Some(ident1), Some(ident2)) = (t1.lifetime(), t2.lifetime()) { - ident1.name == ident2.name - } else { - t1.kind == t2.kind - } -} - -/// Process the matcher positions of `cur_items` until it is empty. In the process, this will -/// produce more items in `next_items`, `eof_items`, and `bb_items`. -/// -/// For more info about the how this happens, see the module-level doc comments and the inline -/// comments of this function. -/// -/// # Parameters -/// -/// - `sess`: the parsing session into which errors are emitted. -/// - `cur_items`: the set of current items to be processed. This should be empty by the end of a -/// successful execution of this function. -/// - `next_items`: the set of newly generated items. These are used to replenish `cur_items` in -/// the function `parse`. -/// - `eof_items`: the set of items that would be valid if this was the EOF. -/// - `bb_items`: the set of items that are waiting for the black-box parser. -/// - `token`: the current token of the parser. -/// - `span`: the `Span` in the source code corresponding to the token trees we are trying to match -/// against the matcher positions in `cur_items`. -/// -/// # Returns -/// -/// A `ParseResult`. Note that matches are kept track of through the items generated. -fn inner_parse_loop<'root, 'tt>( - sess: &ParseSess, - cur_items: &mut SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>, - next_items: &mut Vec>, - eof_items: &mut SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>, - bb_items: &mut SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>, - token: &Token, -) -> ParseResult<()> { - // Pop items from `cur_items` until it is empty. - while let Some(mut item) = cur_items.pop() { - // When unzipped trees end, remove them. This corresponds to backtracking out of a - // delimited submatcher into which we already descended. In backtracking out again, we need - // to advance the "dot" past the delimiters in the outer matcher. - while item.idx >= item.top_elts.len() { - match item.stack.pop() { - Some(MatcherTtFrame { elts, idx }) => { - item.top_elts = elts; - item.idx = idx + 1; - } - None => break, - } - } - - // Get the current position of the "dot" (`idx`) in `item` and the number of token trees in - // the matcher (`len`). - let idx = item.idx; - let len = item.top_elts.len(); - - // If `idx >= len`, then we are at or past the end of the matcher of `item`. - if idx >= len { - // We are repeating iff there is a parent. If the matcher is inside of a repetition, - // then we could be at the end of a sequence or at the beginning of the next - // repetition. - if item.up.is_some() { - // At this point, regardless of whether there is a separator, we should add all - // matches from the complete repetition of the sequence to the shared, top-level - // `matches` list (actually, `up.matches`, which could itself not be the top-level, - // but anyway...). Moreover, we add another item to `cur_items` in which the "dot" - // is at the end of the `up` matcher. This ensures that the "dot" in the `up` - // matcher is also advanced sufficiently. - // - // NOTE: removing the condition `idx == len` allows trailing separators. - if idx == len { - // Get the `up` matcher - let mut new_pos = item.up.clone().unwrap(); - - // Add matches from this repetition to the `matches` of `up` - for idx in item.match_lo..item.match_hi { - let sub = item.matches[idx].clone(); - let span = DelimSpan::from_pair(item.sp_open, token.span); - new_pos.push_match(idx, MatchedSeq(sub, span)); - } - - // Move the "dot" past the repetition in `up` - new_pos.match_cur = item.match_hi; - new_pos.idx += 1; - cur_items.push(new_pos); - } - - // Check if we need a separator. - if idx == len && item.sep.is_some() { - // We have a separator, and it is the current token. We can advance past the - // separator token. - if item.sep - .as_ref() - .map(|sep| token_name_eq(token, sep)) - .unwrap_or(false) - { - item.idx += 1; - next_items.push(item); - } - } - // We don't need a separator. Move the "dot" back to the beginning of the matcher - // and try to match again UNLESS we are only allowed to have _one_ repetition. - else if item.seq_op != Some(mbe::KleeneOp::ZeroOrOne) { - item.match_cur = item.match_lo; - item.idx = 0; - cur_items.push(item); - } - } - // If we are not in a repetition, then being at the end of a matcher means that we have - // reached the potential end of the input. - else { - eof_items.push(item); - } - } - // We are in the middle of a matcher. - else { - // Look at what token in the matcher we are trying to match the current token (`token`) - // against. Depending on that, we may generate new items. - match item.top_elts.get_tt(idx) { - // Need to descend into a sequence - TokenTree::Sequence(sp, seq) => { - // Examine the case where there are 0 matches of this sequence. We are - // implicitly disallowing OneOrMore from having 0 matches here. Thus, that will - // result in a "no rules expected token" error by virtue of this matcher not - // working. - if seq.kleene.op == mbe::KleeneOp::ZeroOrMore - || seq.kleene.op == mbe::KleeneOp::ZeroOrOne - { - let mut new_item = item.clone(); - new_item.match_cur += seq.num_captures; - new_item.idx += 1; - for idx in item.match_cur..item.match_cur + seq.num_captures { - new_item.push_match(idx, MatchedSeq(Lrc::new(smallvec![]), sp)); - } - cur_items.push(new_item); - } - - let matches = create_matches(item.matches.len()); - cur_items.push(MatcherPosHandle::Box(Box::new(MatcherPos { - stack: smallvec![], - sep: seq.separator.clone(), - seq_op: Some(seq.kleene.op), - idx: 0, - matches, - match_lo: item.match_cur, - match_cur: item.match_cur, - match_hi: item.match_cur + seq.num_captures, - up: Some(item), - sp_open: sp.open, - top_elts: Tt(TokenTree::Sequence(sp, seq)), - }))); - } - - // We need to match a metavar (but the identifier is invalid)... this is an error - TokenTree::MetaVarDecl(span, _, id) if id.name == kw::Invalid => { - if sess.missing_fragment_specifiers.borrow_mut().remove(&span) { - return Error(span, "missing fragment specifier".to_string()); - } - } - - // We need to match a metavar with a valid ident... call out to the black-box - // parser by adding an item to `bb_items`. - TokenTree::MetaVarDecl(_, _, id) => { - // Built-in nonterminals never start with these tokens, - // so we can eliminate them from consideration. - if may_begin_with(token, id.name) { - bb_items.push(item); - } - } - - // We need to descend into a delimited submatcher or a doc comment. To do this, we - // push the current matcher onto a stack and push a new item containing the - // submatcher onto `cur_items`. - // - // At the beginning of the loop, if we reach the end of the delimited submatcher, - // we pop the stack to backtrack out of the descent. - seq @ TokenTree::Delimited(..) | - seq @ TokenTree::Token(Token { kind: DocComment(..), .. }) => { - let lower_elts = mem::replace(&mut item.top_elts, Tt(seq)); - let idx = item.idx; - item.stack.push(MatcherTtFrame { - elts: lower_elts, - idx, - }); - item.idx = 0; - cur_items.push(item); - } - - // We just matched a normal token. We can just advance the parser. - TokenTree::Token(t) if token_name_eq(&t, token) => { - item.idx += 1; - next_items.push(item); - } - - // There was another token that was not `token`... This means we can't add any - // rules. NOTE that this is not necessarily an error unless _all_ items in - // `cur_items` end up doing this. There may still be some other matchers that do - // end up working out. - TokenTree::Token(..) | TokenTree::MetaVar(..) => {} - } - } - } - - // Yay a successful parse (so far)! - Success(()) -} - -/// Use the given sequence of token trees (`ms`) as a matcher. Match the given token stream `tts` -/// against it and return the match. -/// -/// # Parameters -/// -/// - `sess`: The session into which errors are emitted -/// - `tts`: The tokenstream we are matching against the pattern `ms` -/// - `ms`: A sequence of token trees representing a pattern against which we are matching -/// - `directory`: Information about the file locations (needed for the black-box parser) -/// - `recurse_into_modules`: Whether or not to recurse into modules (needed for the black-box -/// parser) -pub(super) fn parse( - sess: &ParseSess, - tts: TokenStream, - ms: &[TokenTree], - directory: Option>, - recurse_into_modules: bool, -) -> NamedParseResult { - // Create a parser that can be used for the "black box" parts. - let mut parser = Parser::new( - sess, - tts, - directory, - recurse_into_modules, - true, - crate::MACRO_ARGUMENTS, - ); - - // A queue of possible matcher positions. We initialize it with the matcher position in which - // the "dot" is before the first token of the first token tree in `ms`. `inner_parse_loop` then - // processes all of these possible matcher positions and produces possible next positions into - // `next_items`. After some post-processing, the contents of `next_items` replenish `cur_items` - // and we start over again. - // - // This MatcherPos instance is allocated on the stack. All others -- and - // there are frequently *no* others! -- are allocated on the heap. - let mut initial = initial_matcher_pos(ms, parser.token.span); - let mut cur_items = smallvec![MatcherPosHandle::Ref(&mut initial)]; - let mut next_items = Vec::new(); - - loop { - // Matcher positions black-box parsed by parser.rs (`parser`) - let mut bb_items = SmallVec::new(); - - // Matcher positions that would be valid if the macro invocation was over now - let mut eof_items = SmallVec::new(); - assert!(next_items.is_empty()); - - // Process `cur_items` until either we have finished the input or we need to get some - // parsing from the black-box parser done. The result is that `next_items` will contain a - // bunch of possible next matcher positions in `next_items`. - match inner_parse_loop( - sess, - &mut cur_items, - &mut next_items, - &mut eof_items, - &mut bb_items, - &parser.token, - ) { - Success(_) => {} - Failure(token, msg) => return Failure(token, msg), - Error(sp, msg) => return Error(sp, msg), - } - - // inner parse loop handled all cur_items, so it's empty - assert!(cur_items.is_empty()); - - // We need to do some post processing after the `inner_parser_loop`. - // - // Error messages here could be improved with links to original rules. - - // If we reached the EOF, check that there is EXACTLY ONE possible matcher. Otherwise, - // either the parse is ambiguous (which should never happen) or there is a syntax error. - if parser.token == token::Eof { - if eof_items.len() == 1 { - let matches = eof_items[0] - .matches - .iter_mut() - .map(|dv| Lrc::make_mut(dv).pop().unwrap()); - return nameize(sess, ms, matches); - } else if eof_items.len() > 1 { - return Error( - parser.token.span, - "ambiguity: multiple successful parses".to_string(), - ); - } else { - return Failure( - Token::new(token::Eof, if parser.token.span.is_dummy() { - parser.token.span - } else { - sess.source_map().next_point(parser.token.span) - }), - "missing tokens in macro arguments", - ); - } - } - // Performance hack: eof_items may share matchers via Rc with other things that we want - // to modify. Dropping eof_items now may drop these refcounts to 1, preventing an - // unnecessary implicit clone later in Rc::make_mut. - drop(eof_items); - - // Another possibility is that we need to call out to parse some rust nonterminal - // (black-box) parser. However, if there is not EXACTLY ONE of these, something is wrong. - if (!bb_items.is_empty() && !next_items.is_empty()) || bb_items.len() > 1 { - let nts = bb_items - .iter() - .map(|item| match item.top_elts.get_tt(item.idx) { - TokenTree::MetaVarDecl(_, bind, name) => format!("{} ('{}')", name, bind), - _ => panic!(), - }) - .collect::>() - .join(" or "); - - return Error( - parser.token.span, - format!( - "local ambiguity: multiple parsing options: {}", - match next_items.len() { - 0 => format!("built-in NTs {}.", nts), - 1 => format!("built-in NTs {} or 1 other option.", nts), - n => format!("built-in NTs {} or {} other options.", nts, n), - } - ), - ); - } - // If there are no possible next positions AND we aren't waiting for the black-box parser, - // then there is a syntax error. - else if bb_items.is_empty() && next_items.is_empty() { - return Failure( - parser.token.take(), - "no rules expected this token in macro call", - ); - } - // Dump all possible `next_items` into `cur_items` for the next iteration. - else if !next_items.is_empty() { - // Now process the next token - cur_items.extend(next_items.drain(..)); - parser.bump(); - } - // Finally, we have the case where we need to call the black-box parser to get some - // nonterminal. - else { - assert_eq!(bb_items.len(), 1); - - let mut item = bb_items.pop().unwrap(); - if let TokenTree::MetaVarDecl(span, _, ident) = item.top_elts.get_tt(item.idx) { - let match_cur = item.match_cur; - item.push_match( - match_cur, - MatchedNonterminal(Lrc::new(parse_nt(&mut parser, span, ident.name))), - ); - item.idx += 1; - item.match_cur += 1; - } else { - unreachable!() - } - cur_items.push(item); - } - - assert!(!cur_items.is_empty()); - } -} - -/// The token is an identifier, but not `_`. -/// We prohibit passing `_` to macros expecting `ident` for now. -fn get_macro_name(token: &Token) -> Option<(Name, bool)> { - match token.kind { - token::Ident(name, is_raw) if name != kw::Underscore => Some((name, is_raw)), - _ => None, - } -} - -/// Checks whether a non-terminal may begin with a particular token. -/// -/// Returning `false` is a *stability guarantee* that such a matcher will *never* begin with that -/// token. Be conservative (return true) if not sure. -fn may_begin_with(token: &Token, name: Name) -> bool { - /// Checks whether the non-terminal may contain a single (non-keyword) identifier. - fn may_be_ident(nt: &token::Nonterminal) -> bool { - match *nt { - token::NtItem(_) | token::NtBlock(_) | token::NtVis(_) => false, - _ => true, - } - } - - match name { - sym::expr => token.can_begin_expr() - // This exception is here for backwards compatibility. - && !token.is_keyword(kw::Let), - sym::ty => token.can_begin_type(), - sym::ident => get_macro_name(token).is_some(), - sym::literal => token.can_begin_literal_or_bool(), - sym::vis => match token.kind { - // The follow-set of :vis + "priv" keyword + interpolated - token::Comma | token::Ident(..) | token::Interpolated(_) => true, - _ => token.can_begin_type(), - }, - sym::block => match token.kind { - token::OpenDelim(token::Brace) => true, - token::Interpolated(ref nt) => match **nt { - token::NtItem(_) - | token::NtPat(_) - | token::NtTy(_) - | token::NtIdent(..) - | token::NtMeta(_) - | token::NtPath(_) - | token::NtVis(_) => false, // none of these may start with '{'. - _ => true, - }, - _ => false, - }, - sym::path | sym::meta => match token.kind { - token::ModSep | token::Ident(..) => true, - token::Interpolated(ref nt) => match **nt { - token::NtPath(_) | token::NtMeta(_) => true, - _ => may_be_ident(&nt), - }, - _ => false, - }, - sym::pat => match token.kind { - token::Ident(..) | // box, ref, mut, and other identifiers (can stricten) - token::OpenDelim(token::Paren) | // tuple pattern - token::OpenDelim(token::Bracket) | // slice pattern - token::BinOp(token::And) | // reference - token::BinOp(token::Minus) | // negative literal - token::AndAnd | // double reference - token::Literal(..) | // literal - token::DotDot | // range pattern (future compat) - token::DotDotDot | // range pattern (future compat) - token::ModSep | // path - token::Lt | // path (UFCS constant) - token::BinOp(token::Shl) => true, // path (double UFCS) - token::Interpolated(ref nt) => may_be_ident(nt), - _ => false, - }, - sym::lifetime => match token.kind { - token::Lifetime(_) => true, - token::Interpolated(ref nt) => match **nt { - token::NtLifetime(_) | token::NtTT(_) => true, - _ => false, - }, - _ => false, - }, - _ => match token.kind { - token::CloseDelim(_) => false, - _ => true, - }, - } -} - -/// A call to the "black-box" parser to parse some Rust non-terminal. -/// -/// # Parameters -/// -/// - `p`: the "black-box" parser to use -/// - `sp`: the `Span` we want to parse -/// - `name`: the name of the metavar _matcher_ we want to match (e.g., `tt`, `ident`, `block`, -/// etc...) -/// -/// # Returns -/// -/// The parsed non-terminal. -fn parse_nt(p: &mut Parser<'_>, sp: Span, name: Symbol) -> Nonterminal { - if name == sym::tt { - return token::NtTT(p.parse_token_tree()); - } - // check at the beginning and the parser checks after each bump - p.process_potential_macro_variable(); - match parse_nt_inner(p, sp, name) { - Ok(nt) => nt, - Err(mut err) => { - err.emit(); - FatalError.raise(); - } - } -} - -fn parse_nt_inner<'a>(p: &mut Parser<'a>, sp: Span, name: Symbol) -> PResult<'a, Nonterminal> { - Ok(match name { - sym::item => match p.parse_item()? { - Some(i) => token::NtItem(i), - None => return Err(p.fatal("expected an item keyword")), - }, - sym::block => token::NtBlock(p.parse_block()?), - sym::stmt => match p.parse_stmt()? { - Some(s) => token::NtStmt(s), - None => return Err(p.fatal("expected a statement")), - }, - sym::pat => token::NtPat(p.parse_pat(None)?), - sym::expr => token::NtExpr(p.parse_expr()?), - sym::literal => token::NtLiteral(p.parse_literal_maybe_minus()?), - sym::ty => token::NtTy(p.parse_ty()?), - // this could be handled like a token, since it is one - sym::ident => if let Some((name, is_raw)) = get_macro_name(&p.token) { - let span = p.token.span; - p.bump(); - token::NtIdent(Ident::new(name, span), is_raw) - } else { - let token_str = pprust::token_to_string(&p.token); - return Err(p.fatal(&format!("expected ident, found {}", &token_str))); - } - sym::path => token::NtPath(p.parse_path(PathStyle::Type)?), - sym::meta => token::NtMeta(p.parse_attr_item()?), - sym::vis => token::NtVis(p.parse_visibility(true)?), - sym::lifetime => if p.check_lifetime() { - token::NtLifetime(p.expect_lifetime().ident) - } else { - let token_str = pprust::token_to_string(&p.token); - return Err(p.fatal(&format!("expected a lifetime, found `{}`", &token_str))); - } - // this is not supposed to happen, since it has been checked - // when compiling the macro. - _ => p.span_bug(sp, "invalid fragment specifier"), - }) -} diff --git a/src/libsyntax/ext/mbe/macro_rules.rs b/src/libsyntax/ext/mbe/macro_rules.rs deleted file mode 100644 index b4223298422..00000000000 --- a/src/libsyntax/ext/mbe/macro_rules.rs +++ /dev/null @@ -1,1194 +0,0 @@ -use crate::ast; -use crate::attr::{self, TransparencyError}; -use crate::edition::Edition; -use crate::ext::base::{DummyResult, ExtCtxt, MacResult, TTMacroExpander}; -use crate::ext::base::{SyntaxExtension, SyntaxExtensionKind}; -use crate::ext::expand::{AstFragment, AstFragmentKind}; -use crate::ext::mbe; -use crate::ext::mbe::macro_check; -use crate::ext::mbe::macro_parser::parse; -use crate::ext::mbe::macro_parser::{Error, Failure, Success}; -use crate::ext::mbe::macro_parser::{MatchedNonterminal, MatchedSeq, NamedParseResult}; -use crate::ext::mbe::transcribe::transcribe; -use crate::feature_gate::Features; -use crate::parse::parser::Parser; -use crate::parse::token::TokenKind::*; -use crate::parse::token::{self, NtTT, Token}; -use crate::parse::Directory; -use crate::print::pprust; -use crate::sess::ParseSess; -use crate::symbol::{kw, sym, Symbol}; -use crate::tokenstream::{DelimSpan, TokenStream, TokenTree}; - -use errors::{DiagnosticBuilder, FatalError}; -use log::debug; -use syntax_pos::hygiene::Transparency; -use syntax_pos::Span; - -use rustc_data_structures::fx::FxHashMap; -use std::borrow::Cow; -use std::collections::hash_map::Entry; -use std::slice; - -use errors::Applicability; -use rustc_data_structures::sync::Lrc; - -const VALID_FRAGMENT_NAMES_MSG: &str = "valid fragment specifiers are \ - `ident`, `block`, `stmt`, `expr`, `pat`, `ty`, `lifetime`, \ - `literal`, `path`, `meta`, `tt`, `item` and `vis`"; - -crate struct ParserAnyMacro<'a> { - parser: Parser<'a>, - - /// Span of the expansion site of the macro this parser is for - site_span: Span, - /// The ident of the macro we're parsing - macro_ident: ast::Ident, - arm_span: Span, -} - -crate fn annotate_err_with_kind( - err: &mut DiagnosticBuilder<'_>, - kind: AstFragmentKind, - span: Span, -) { - match kind { - AstFragmentKind::Ty => { - err.span_label(span, "this macro call doesn't expand to a type"); - } - AstFragmentKind::Pat => { - err.span_label(span, "this macro call doesn't expand to a pattern"); - } - _ => {} - }; -} - -impl<'a> ParserAnyMacro<'a> { - crate fn make(mut self: Box>, kind: AstFragmentKind) -> AstFragment { - let ParserAnyMacro { site_span, macro_ident, ref mut parser, arm_span } = *self; - let fragment = panictry!(parser.parse_ast_fragment(kind, true).map_err(|mut e| { - if parser.token == token::Eof && e.message().ends_with(", found ``") { - if !e.span.is_dummy() { - // early end of macro arm (#52866) - e.replace_span_with(parser.sess.source_map().next_point(parser.token.span)); - } - let msg = &e.message[0]; - e.message[0] = ( - format!( - "macro expansion ends with an incomplete expression: {}", - msg.0.replace(", found ``", ""), - ), - msg.1, - ); - } - if e.span.is_dummy() { - // Get around lack of span in error (#30128) - e.replace_span_with(site_span); - if parser.sess.source_map().span_to_filename(arm_span).is_real() { - e.span_label(arm_span, "in this macro arm"); - } - } else if !parser.sess.source_map().span_to_filename(parser.token.span).is_real() { - e.span_label(site_span, "in this macro invocation"); - } - match kind { - AstFragmentKind::Pat if macro_ident.name == sym::vec => { - let mut suggestion = None; - if let Ok(code) = parser.sess.source_map().span_to_snippet(site_span) { - if let Some(bang) = code.find('!') { - suggestion = Some(code[bang + 1..].to_string()); - } - } - if let Some(suggestion) = suggestion { - e.span_suggestion( - site_span, - "use a slice pattern here instead", - suggestion, - Applicability::MachineApplicable, - ); - } else { - e.span_label( - site_span, - "use a slice pattern here instead", - ); - } - e.help("for more information, see https://doc.rust-lang.org/edition-guide/\ - rust-2018/slice-patterns.html"); - } - _ => annotate_err_with_kind(&mut e, kind, site_span), - }; - e - })); - - // We allow semicolons at the end of expressions -- e.g., the semicolon in - // `macro_rules! m { () => { panic!(); } }` isn't parsed by `.parse_expr()`, - // but `m!()` is allowed in expression positions (cf. issue #34706). - if kind == AstFragmentKind::Expr && parser.token == token::Semi { - parser.bump(); - } - - // Make sure we don't have any tokens left to parse so we don't silently drop anything. - let path = ast::Path::from_ident(macro_ident.with_span_pos(site_span)); - parser.ensure_complete_parse(&path, kind.name(), site_span); - fragment - } -} - -struct MacroRulesMacroExpander { - name: ast::Ident, - span: Span, - transparency: Transparency, - lhses: Vec, - rhses: Vec, - valid: bool, -} - -impl TTMacroExpander for MacroRulesMacroExpander { - fn expand<'cx>( - &self, - cx: &'cx mut ExtCtxt<'_>, - sp: Span, - input: TokenStream, - ) -> Box { - if !self.valid { - return DummyResult::any(sp); - } - generic_extension( - cx, sp, self.span, self.name, self.transparency, input, &self.lhses, &self.rhses - ) - } -} - -fn trace_macros_note(cx: &mut ExtCtxt<'_>, sp: Span, message: String) { - let sp = sp.macro_backtrace().last().map(|trace| trace.call_site).unwrap_or(sp); - cx.expansions.entry(sp).or_default().push(message); -} - -/// Given `lhses` and `rhses`, this is the new macro we create -fn generic_extension<'cx>( - cx: &'cx mut ExtCtxt<'_>, - sp: Span, - def_span: Span, - name: ast::Ident, - transparency: Transparency, - arg: TokenStream, - lhses: &[mbe::TokenTree], - rhses: &[mbe::TokenTree], -) -> Box { - if cx.trace_macros() { - let msg = format!("expanding `{}! {{ {} }}`", name, pprust::tts_to_string(arg.clone())); - trace_macros_note(cx, sp, msg); - } - - // Which arm's failure should we report? (the one furthest along) - let mut best_failure: Option<(Token, &str)> = None; - - for (i, lhs) in lhses.iter().enumerate() { - // try each arm's matchers - let lhs_tt = match *lhs { - mbe::TokenTree::Delimited(_, ref delim) => &delim.tts[..], - _ => cx.span_bug(sp, "malformed macro lhs"), - }; - - match TokenTree::parse(cx, lhs_tt, arg.clone()) { - Success(named_matches) => { - let rhs = match rhses[i] { - // ignore delimiters - mbe::TokenTree::Delimited(_, ref delimed) => delimed.tts.clone(), - _ => cx.span_bug(sp, "malformed macro rhs"), - }; - let arm_span = rhses[i].span(); - - let rhs_spans = rhs.iter().map(|t| t.span()).collect::>(); - // rhs has holes ( `$id` and `$(...)` that need filled) - let mut tts = transcribe(cx, &named_matches, rhs, transparency); - - // Replace all the tokens for the corresponding positions in the macro, to maintain - // proper positions in error reporting, while maintaining the macro_backtrace. - if rhs_spans.len() == tts.len() { - tts = tts.map_enumerated(|i, mut tt| { - let mut sp = rhs_spans[i]; - sp = sp.with_ctxt(tt.span().ctxt()); - tt.set_span(sp); - tt - }); - } - - if cx.trace_macros() { - let msg = format!("to `{}`", pprust::tts_to_string(tts.clone())); - trace_macros_note(cx, sp, msg); - } - - let directory = Directory { - path: Cow::from(cx.current_expansion.module.directory.as_path()), - ownership: cx.current_expansion.directory_ownership, - }; - let mut p = Parser::new(cx.parse_sess(), tts, Some(directory), true, false, None); - p.root_module_name = - cx.current_expansion.module.mod_path.last().map(|id| id.as_str().to_string()); - p.last_type_ascription = cx.current_expansion.prior_type_ascription; - - p.process_potential_macro_variable(); - // Let the context choose how to interpret the result. - // Weird, but useful for X-macros. - return Box::new(ParserAnyMacro { - parser: p, - - // Pass along the original expansion site and the name of the macro - // so we can print a useful error message if the parse of the expanded - // macro leaves unparsed tokens. - site_span: sp, - macro_ident: name, - arm_span, - }); - } - Failure(token, msg) => match best_failure { - Some((ref best_token, _)) if best_token.span.lo() >= token.span.lo() => {} - _ => best_failure = Some((token, msg)), - }, - Error(err_sp, ref msg) => cx.span_fatal(err_sp.substitute_dummy(sp), &msg[..]), - } - } - - let (token, label) = best_failure.expect("ran no matchers"); - let span = token.span.substitute_dummy(sp); - let mut err = cx.struct_span_err(span, &parse_failure_msg(&token)); - err.span_label(span, label); - if !def_span.is_dummy() && cx.source_map().span_to_filename(def_span).is_real() { - err.span_label(cx.source_map().def_span(def_span), "when calling this macro"); - } - - // Check whether there's a missing comma in this macro call, like `println!("{}" a);` - if let Some((arg, comma_span)) = arg.add_comma() { - for lhs in lhses { - // try each arm's matchers - let lhs_tt = match *lhs { - mbe::TokenTree::Delimited(_, ref delim) => &delim.tts[..], - _ => continue, - }; - match TokenTree::parse(cx, lhs_tt, arg.clone()) { - Success(_) => { - if comma_span.is_dummy() { - err.note("you might be missing a comma"); - } else { - err.span_suggestion_short( - comma_span, - "missing comma here", - ", ".to_string(), - Applicability::MachineApplicable, - ); - } - } - _ => {} - } - } - } - err.emit(); - cx.trace_macros_diag(); - DummyResult::any(sp) -} - -// Note that macro-by-example's input is also matched against a token tree: -// $( $lhs:tt => $rhs:tt );+ -// -// Holy self-referential! - -/// Converts a macro item into a syntax extension. -pub fn compile_declarative_macro( - sess: &ParseSess, - features: &Features, - def: &ast::Item, - edition: Edition, -) -> SyntaxExtension { - let diag = &sess.span_diagnostic; - let lhs_nm = ast::Ident::new(sym::lhs, def.span); - let rhs_nm = ast::Ident::new(sym::rhs, def.span); - let tt_spec = ast::Ident::new(sym::tt, def.span); - - // Parse the macro_rules! invocation - let body = match def.kind { - ast::ItemKind::MacroDef(ref body) => body, - _ => unreachable!(), - }; - - // The pattern that macro_rules matches. - // The grammar for macro_rules! is: - // $( $lhs:tt => $rhs:tt );+ - // ...quasiquoting this would be nice. - // These spans won't matter, anyways - let argument_gram = vec![ - mbe::TokenTree::Sequence( - DelimSpan::dummy(), - Lrc::new(mbe::SequenceRepetition { - tts: vec![ - mbe::TokenTree::MetaVarDecl(def.span, lhs_nm, tt_spec), - mbe::TokenTree::token(token::FatArrow, def.span), - mbe::TokenTree::MetaVarDecl(def.span, rhs_nm, tt_spec), - ], - separator: Some(Token::new( - if body.legacy { token::Semi } else { token::Comma }, - def.span, - )), - kleene: mbe::KleeneToken::new(mbe::KleeneOp::OneOrMore, def.span), - num_captures: 2, - }), - ), - // to phase into semicolon-termination instead of semicolon-separation - mbe::TokenTree::Sequence( - DelimSpan::dummy(), - Lrc::new(mbe::SequenceRepetition { - tts: vec![mbe::TokenTree::token( - if body.legacy { token::Semi } else { token::Comma }, - def.span, - )], - separator: None, - kleene: mbe::KleeneToken::new(mbe::KleeneOp::ZeroOrMore, def.span), - num_captures: 0, - }), - ), - ]; - - let argument_map = match parse(sess, body.stream(), &argument_gram, None, true) { - Success(m) => m, - Failure(token, msg) => { - let s = parse_failure_msg(&token); - let sp = token.span.substitute_dummy(def.span); - let mut err = sess.span_diagnostic.struct_span_fatal(sp, &s); - err.span_label(sp, msg); - err.emit(); - FatalError.raise(); - } - Error(sp, s) => { - sess.span_diagnostic.span_fatal(sp.substitute_dummy(def.span), &s).raise(); - } - }; - - let mut valid = true; - - // Extract the arguments: - let lhses = match argument_map[&lhs_nm] { - MatchedSeq(ref s, _) => s - .iter() - .map(|m| { - if let MatchedNonterminal(ref nt) = *m { - if let NtTT(ref tt) = **nt { - let tt = mbe::quoted::parse( - tt.clone().into(), - true, - sess, - ) - .pop() - .unwrap(); - valid &= check_lhs_nt_follows(sess, features, &def.attrs, &tt); - return tt; - } - } - sess.span_diagnostic.span_bug(def.span, "wrong-structured lhs") - }) - .collect::>(), - _ => sess.span_diagnostic.span_bug(def.span, "wrong-structured lhs"), - }; - - let rhses = match argument_map[&rhs_nm] { - MatchedSeq(ref s, _) => s - .iter() - .map(|m| { - if let MatchedNonterminal(ref nt) = *m { - if let NtTT(ref tt) = **nt { - return mbe::quoted::parse( - tt.clone().into(), - false, - sess, - ) - .pop() - .unwrap(); - } - } - sess.span_diagnostic.span_bug(def.span, "wrong-structured lhs") - }) - .collect::>(), - _ => sess.span_diagnostic.span_bug(def.span, "wrong-structured rhs"), - }; - - for rhs in &rhses { - valid &= check_rhs(sess, rhs); - } - - // don't abort iteration early, so that errors for multiple lhses can be reported - for lhs in &lhses { - valid &= check_lhs_no_empty_seq(sess, slice::from_ref(lhs)); - } - - // We use CRATE_NODE_ID instead of `def.id` otherwise we may emit buffered lints for a node id - // that is not lint-checked and trigger the "failed to process buffered lint here" bug. - valid &= macro_check::check_meta_variables(sess, ast::CRATE_NODE_ID, def.span, &lhses, &rhses); - - let (transparency, transparency_error) = attr::find_transparency(&def.attrs, body.legacy); - match transparency_error { - Some(TransparencyError::UnknownTransparency(value, span)) => - diag.span_err(span, &format!("unknown macro transparency: `{}`", value)), - Some(TransparencyError::MultipleTransparencyAttrs(old_span, new_span)) => - diag.span_err(vec![old_span, new_span], "multiple macro transparency attributes"), - None => {} - } - - let expander: Box<_> = Box::new(MacroRulesMacroExpander { - name: def.ident, span: def.span, transparency, lhses, rhses, valid - }); - - SyntaxExtension::new( - sess, - SyntaxExtensionKind::LegacyBang(expander), - def.span, - Vec::new(), - edition, - def.ident.name, - &def.attrs, - ) -} - -fn check_lhs_nt_follows( - sess: &ParseSess, - features: &Features, - attrs: &[ast::Attribute], - lhs: &mbe::TokenTree, -) -> bool { - // lhs is going to be like TokenTree::Delimited(...), where the - // entire lhs is those tts. Or, it can be a "bare sequence", not wrapped in parens. - if let mbe::TokenTree::Delimited(_, ref tts) = *lhs { - check_matcher(sess, features, attrs, &tts.tts) - } else { - let msg = "invalid macro matcher; matchers must be contained in balanced delimiters"; - sess.span_diagnostic.span_err(lhs.span(), msg); - false - } - // we don't abort on errors on rejection, the driver will do that for us - // after parsing/expansion. we can report every error in every macro this way. -} - -/// Checks that the lhs contains no repetition which could match an empty token -/// tree, because then the matcher would hang indefinitely. -fn check_lhs_no_empty_seq(sess: &ParseSess, tts: &[mbe::TokenTree]) -> bool { - use mbe::TokenTree; - for tt in tts { - match *tt { - TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => (), - TokenTree::Delimited(_, ref del) => { - if !check_lhs_no_empty_seq(sess, &del.tts) { - return false; - } - } - TokenTree::Sequence(span, ref seq) => { - if seq.separator.is_none() - && seq.tts.iter().all(|seq_tt| match *seq_tt { - TokenTree::MetaVarDecl(_, _, id) => id.name == sym::vis, - TokenTree::Sequence(_, ref sub_seq) => { - sub_seq.kleene.op == mbe::KleeneOp::ZeroOrMore - || sub_seq.kleene.op == mbe::KleeneOp::ZeroOrOne - } - _ => false, - }) - { - let sp = span.entire(); - sess.span_diagnostic.span_err(sp, "repetition matches empty token tree"); - return false; - } - if !check_lhs_no_empty_seq(sess, &seq.tts) { - return false; - } - } - } - } - - true -} - -fn check_rhs(sess: &ParseSess, rhs: &mbe::TokenTree) -> bool { - match *rhs { - mbe::TokenTree::Delimited(..) => return true, - _ => sess.span_diagnostic.span_err(rhs.span(), "macro rhs must be delimited"), - } - false -} - -fn check_matcher( - sess: &ParseSess, - features: &Features, - attrs: &[ast::Attribute], - matcher: &[mbe::TokenTree], -) -> bool { - let first_sets = FirstSets::new(matcher); - let empty_suffix = TokenSet::empty(); - let err = sess.span_diagnostic.err_count(); - check_matcher_core(sess, features, attrs, &first_sets, matcher, &empty_suffix); - err == sess.span_diagnostic.err_count() -} - -// `The FirstSets` for a matcher is a mapping from subsequences in the -// matcher to the FIRST set for that subsequence. -// -// This mapping is partially precomputed via a backwards scan over the -// token trees of the matcher, which provides a mapping from each -// repetition sequence to its *first* set. -// -// (Hypothetically, sequences should be uniquely identifiable via their -// spans, though perhaps that is false, e.g., for macro-generated macros -// that do not try to inject artificial span information. My plan is -// to try to catch such cases ahead of time and not include them in -// the precomputed mapping.) -struct FirstSets { - // this maps each TokenTree::Sequence `$(tt ...) SEP OP` that is uniquely identified by its - // span in the original matcher to the First set for the inner sequence `tt ...`. - // - // If two sequences have the same span in a matcher, then map that - // span to None (invalidating the mapping here and forcing the code to - // use a slow path). - first: FxHashMap>, -} - -impl FirstSets { - fn new(tts: &[mbe::TokenTree]) -> FirstSets { - use mbe::TokenTree; - - let mut sets = FirstSets { first: FxHashMap::default() }; - build_recur(&mut sets, tts); - return sets; - - // walks backward over `tts`, returning the FIRST for `tts` - // and updating `sets` at the same time for all sequence - // substructure we find within `tts`. - fn build_recur(sets: &mut FirstSets, tts: &[TokenTree]) -> TokenSet { - let mut first = TokenSet::empty(); - for tt in tts.iter().rev() { - match *tt { - TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => { - first.replace_with(tt.clone()); - } - TokenTree::Delimited(span, ref delimited) => { - build_recur(sets, &delimited.tts[..]); - first.replace_with(delimited.open_tt(span.open)); - } - TokenTree::Sequence(sp, ref seq_rep) => { - let subfirst = build_recur(sets, &seq_rep.tts[..]); - - match sets.first.entry(sp.entire()) { - Entry::Vacant(vac) => { - vac.insert(Some(subfirst.clone())); - } - Entry::Occupied(mut occ) => { - // if there is already an entry, then a span must have collided. - // This should not happen with typical macro_rules macros, - // but syntax extensions need not maintain distinct spans, - // so distinct syntax trees can be assigned the same span. - // In such a case, the map cannot be trusted; so mark this - // entry as unusable. - occ.insert(None); - } - } - - // If the sequence contents can be empty, then the first - // token could be the separator token itself. - - if let (Some(sep), true) = (&seq_rep.separator, subfirst.maybe_empty) { - first.add_one_maybe(TokenTree::Token(sep.clone())); - } - - // Reverse scan: Sequence comes before `first`. - if subfirst.maybe_empty - || seq_rep.kleene.op == mbe::KleeneOp::ZeroOrMore - || seq_rep.kleene.op == mbe::KleeneOp::ZeroOrOne - { - // If sequence is potentially empty, then - // union them (preserving first emptiness). - first.add_all(&TokenSet { maybe_empty: true, ..subfirst }); - } else { - // Otherwise, sequence guaranteed - // non-empty; replace first. - first = subfirst; - } - } - } - } - - first - } - } - - // walks forward over `tts` until all potential FIRST tokens are - // identified. - fn first(&self, tts: &[mbe::TokenTree]) -> TokenSet { - use mbe::TokenTree; - - let mut first = TokenSet::empty(); - for tt in tts.iter() { - assert!(first.maybe_empty); - match *tt { - TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => { - first.add_one(tt.clone()); - return first; - } - TokenTree::Delimited(span, ref delimited) => { - first.add_one(delimited.open_tt(span.open)); - return first; - } - TokenTree::Sequence(sp, ref seq_rep) => { - let subfirst_owned; - let subfirst = match self.first.get(&sp.entire()) { - Some(&Some(ref subfirst)) => subfirst, - Some(&None) => { - subfirst_owned = self.first(&seq_rep.tts[..]); - &subfirst_owned - } - None => { - panic!("We missed a sequence during FirstSets construction"); - } - }; - - // If the sequence contents can be empty, then the first - // token could be the separator token itself. - if let (Some(sep), true) = (&seq_rep.separator, subfirst.maybe_empty) { - first.add_one_maybe(TokenTree::Token(sep.clone())); - } - - assert!(first.maybe_empty); - first.add_all(subfirst); - if subfirst.maybe_empty - || seq_rep.kleene.op == mbe::KleeneOp::ZeroOrMore - || seq_rep.kleene.op == mbe::KleeneOp::ZeroOrOne - { - // Continue scanning for more first - // tokens, but also make sure we - // restore empty-tracking state. - first.maybe_empty = true; - continue; - } else { - return first; - } - } - } - } - - // we only exit the loop if `tts` was empty or if every - // element of `tts` matches the empty sequence. - assert!(first.maybe_empty); - first - } -} - -// A set of `mbe::TokenTree`s, which may include `TokenTree::Match`s -// (for macro-by-example syntactic variables). It also carries the -// `maybe_empty` flag; that is true if and only if the matcher can -// match an empty token sequence. -// -// The First set is computed on submatchers like `$($a:expr b),* $(c)* d`, -// which has corresponding FIRST = {$a:expr, c, d}. -// Likewise, `$($a:expr b),* $(c)+ d` has FIRST = {$a:expr, c}. -// -// (Notably, we must allow for *-op to occur zero times.) -#[derive(Clone, Debug)] -struct TokenSet { - tokens: Vec, - maybe_empty: bool, -} - -impl TokenSet { - // Returns a set for the empty sequence. - fn empty() -> Self { - TokenSet { tokens: Vec::new(), maybe_empty: true } - } - - // Returns the set `{ tok }` for the single-token (and thus - // non-empty) sequence [tok]. - fn singleton(tok: mbe::TokenTree) -> Self { - TokenSet { tokens: vec![tok], maybe_empty: false } - } - - // Changes self to be the set `{ tok }`. - // Since `tok` is always present, marks self as non-empty. - fn replace_with(&mut self, tok: mbe::TokenTree) { - self.tokens.clear(); - self.tokens.push(tok); - self.maybe_empty = false; - } - - // Changes self to be the empty set `{}`; meant for use when - // the particular token does not matter, but we want to - // record that it occurs. - fn replace_with_irrelevant(&mut self) { - self.tokens.clear(); - self.maybe_empty = false; - } - - // Adds `tok` to the set for `self`, marking sequence as non-empy. - fn add_one(&mut self, tok: mbe::TokenTree) { - if !self.tokens.contains(&tok) { - self.tokens.push(tok); - } - self.maybe_empty = false; - } - - // Adds `tok` to the set for `self`. (Leaves `maybe_empty` flag alone.) - fn add_one_maybe(&mut self, tok: mbe::TokenTree) { - if !self.tokens.contains(&tok) { - self.tokens.push(tok); - } - } - - // Adds all elements of `other` to this. - // - // (Since this is a set, we filter out duplicates.) - // - // If `other` is potentially empty, then preserves the previous - // setting of the empty flag of `self`. If `other` is guaranteed - // non-empty, then `self` is marked non-empty. - fn add_all(&mut self, other: &Self) { - for tok in &other.tokens { - if !self.tokens.contains(tok) { - self.tokens.push(tok.clone()); - } - } - if !other.maybe_empty { - self.maybe_empty = false; - } - } -} - -// Checks that `matcher` is internally consistent and that it -// can legally be followed by a token `N`, for all `N` in `follow`. -// (If `follow` is empty, then it imposes no constraint on -// the `matcher`.) -// -// Returns the set of NT tokens that could possibly come last in -// `matcher`. (If `matcher` matches the empty sequence, then -// `maybe_empty` will be set to true.) -// -// Requires that `first_sets` is pre-computed for `matcher`; -// see `FirstSets::new`. -fn check_matcher_core( - sess: &ParseSess, - features: &Features, - attrs: &[ast::Attribute], - first_sets: &FirstSets, - matcher: &[mbe::TokenTree], - follow: &TokenSet, -) -> TokenSet { - use mbe::TokenTree; - - let mut last = TokenSet::empty(); - - // 2. For each token and suffix [T, SUFFIX] in M: - // ensure that T can be followed by SUFFIX, and if SUFFIX may be empty, - // then ensure T can also be followed by any element of FOLLOW. - 'each_token: for i in 0..matcher.len() { - let token = &matcher[i]; - let suffix = &matcher[i + 1..]; - - let build_suffix_first = || { - let mut s = first_sets.first(suffix); - if s.maybe_empty { - s.add_all(follow); - } - s - }; - - // (we build `suffix_first` on demand below; you can tell - // which cases are supposed to fall through by looking for the - // initialization of this variable.) - let suffix_first; - - // First, update `last` so that it corresponds to the set - // of NT tokens that might end the sequence `... token`. - match *token { - TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => { - let can_be_followed_by_any; - if let Err(bad_frag) = has_legal_fragment_specifier(sess, features, attrs, token) { - let msg = format!("invalid fragment specifier `{}`", bad_frag); - sess.span_diagnostic - .struct_span_err(token.span(), &msg) - .help(VALID_FRAGMENT_NAMES_MSG) - .emit(); - // (This eliminates false positives and duplicates - // from error messages.) - can_be_followed_by_any = true; - } else { - can_be_followed_by_any = token_can_be_followed_by_any(token); - } - - if can_be_followed_by_any { - // don't need to track tokens that work with any, - last.replace_with_irrelevant(); - // ... and don't need to check tokens that can be - // followed by anything against SUFFIX. - continue 'each_token; - } else { - last.replace_with(token.clone()); - suffix_first = build_suffix_first(); - } - } - TokenTree::Delimited(span, ref d) => { - let my_suffix = TokenSet::singleton(d.close_tt(span.close)); - check_matcher_core(sess, features, attrs, first_sets, &d.tts, &my_suffix); - // don't track non NT tokens - last.replace_with_irrelevant(); - - // also, we don't need to check delimited sequences - // against SUFFIX - continue 'each_token; - } - TokenTree::Sequence(_, ref seq_rep) => { - suffix_first = build_suffix_first(); - // The trick here: when we check the interior, we want - // to include the separator (if any) as a potential - // (but not guaranteed) element of FOLLOW. So in that - // case, we make a temp copy of suffix and stuff - // delimiter in there. - // - // FIXME: Should I first scan suffix_first to see if - // delimiter is already in it before I go through the - // work of cloning it? But then again, this way I may - // get a "tighter" span? - let mut new; - let my_suffix = if let Some(sep) = &seq_rep.separator { - new = suffix_first.clone(); - new.add_one_maybe(TokenTree::Token(sep.clone())); - &new - } else { - &suffix_first - }; - - // At this point, `suffix_first` is built, and - // `my_suffix` is some TokenSet that we can use - // for checking the interior of `seq_rep`. - let next = - check_matcher_core(sess, features, attrs, first_sets, &seq_rep.tts, my_suffix); - if next.maybe_empty { - last.add_all(&next); - } else { - last = next; - } - - // the recursive call to check_matcher_core already ran the 'each_last - // check below, so we can just keep going forward here. - continue 'each_token; - } - } - - // (`suffix_first` guaranteed initialized once reaching here.) - - // Now `last` holds the complete set of NT tokens that could - // end the sequence before SUFFIX. Check that every one works with `suffix`. - 'each_last: for token in &last.tokens { - if let TokenTree::MetaVarDecl(_, name, frag_spec) = *token { - for next_token in &suffix_first.tokens { - match is_in_follow(next_token, frag_spec.name) { - IsInFollow::Invalid(msg, help) => { - sess.span_diagnostic - .struct_span_err(next_token.span(), &msg) - .help(help) - .emit(); - // don't bother reporting every source of - // conflict for a particular element of `last`. - continue 'each_last; - } - IsInFollow::Yes => {} - IsInFollow::No(possible) => { - let may_be = if last.tokens.len() == 1 && suffix_first.tokens.len() == 1 - { - "is" - } else { - "may be" - }; - - let sp = next_token.span(); - let mut err = sess.span_diagnostic.struct_span_err( - sp, - &format!( - "`${name}:{frag}` {may_be} followed by `{next}`, which \ - is not allowed for `{frag}` fragments", - name = name, - frag = frag_spec, - next = quoted_tt_to_string(next_token), - may_be = may_be - ), - ); - err.span_label( - sp, - format!("not allowed after `{}` fragments", frag_spec), - ); - let msg = "allowed there are: "; - match possible { - &[] => {} - &[t] => { - err.note(&format!( - "only {} is allowed after `{}` fragments", - t, frag_spec, - )); - } - ts => { - err.note(&format!( - "{}{} or {}", - msg, - ts[..ts.len() - 1] - .iter() - .map(|s| *s) - .collect::>() - .join(", "), - ts[ts.len() - 1], - )); - } - } - err.emit(); - } - } - } - } - } - } - last -} - -fn token_can_be_followed_by_any(tok: &mbe::TokenTree) -> bool { - if let mbe::TokenTree::MetaVarDecl(_, _, frag_spec) = *tok { - frag_can_be_followed_by_any(frag_spec.name) - } else { - // (Non NT's can always be followed by anthing in matchers.) - true - } -} - -/// Returns `true` if a fragment of type `frag` can be followed by any sort of -/// token. We use this (among other things) as a useful approximation -/// for when `frag` can be followed by a repetition like `$(...)*` or -/// `$(...)+`. In general, these can be a bit tricky to reason about, -/// so we adopt a conservative position that says that any fragment -/// specifier which consumes at most one token tree can be followed by -/// a fragment specifier (indeed, these fragments can be followed by -/// ANYTHING without fear of future compatibility hazards). -fn frag_can_be_followed_by_any(frag: Symbol) -> bool { - match frag { - sym::item | // always terminated by `}` or `;` - sym::block | // exactly one token tree - sym::ident | // exactly one token tree - sym::literal | // exactly one token tree - sym::meta | // exactly one token tree - sym::lifetime | // exactly one token tree - sym::tt => // exactly one token tree - true, - - _ => - false, - } -} - -enum IsInFollow { - Yes, - No(&'static [&'static str]), - Invalid(String, &'static str), -} - -/// Returns `true` if `frag` can legally be followed by the token `tok`. For -/// fragments that can consume an unbounded number of tokens, `tok` -/// must be within a well-defined follow set. This is intended to -/// guarantee future compatibility: for example, without this rule, if -/// we expanded `expr` to include a new binary operator, we might -/// break macros that were relying on that binary operator as a -/// separator. -// when changing this do not forget to update doc/book/macros.md! -fn is_in_follow(tok: &mbe::TokenTree, frag: Symbol) -> IsInFollow { - use mbe::TokenTree; - - if let TokenTree::Token(Token { kind: token::CloseDelim(_), .. }) = *tok { - // closing a token tree can never be matched by any fragment; - // iow, we always require that `(` and `)` match, etc. - IsInFollow::Yes - } else { - match frag { - sym::item => { - // since items *must* be followed by either a `;` or a `}`, we can - // accept anything after them - IsInFollow::Yes - } - sym::block => { - // anything can follow block, the braces provide an easy boundary to - // maintain - IsInFollow::Yes - } - sym::stmt | sym::expr => { - const TOKENS: &[&str] = &["`=>`", "`,`", "`;`"]; - match tok { - TokenTree::Token(token) => match token.kind { - FatArrow | Comma | Semi => IsInFollow::Yes, - _ => IsInFollow::No(TOKENS), - }, - _ => IsInFollow::No(TOKENS), - } - } - sym::pat => { - const TOKENS: &[&str] = &["`=>`", "`,`", "`=`", "`|`", "`if`", "`in`"]; - match tok { - TokenTree::Token(token) => match token.kind { - FatArrow | Comma | Eq | BinOp(token::Or) => IsInFollow::Yes, - Ident(name, false) if name == kw::If || name == kw::In => IsInFollow::Yes, - _ => IsInFollow::No(TOKENS), - }, - _ => IsInFollow::No(TOKENS), - } - } - sym::path | sym::ty => { - const TOKENS: &[&str] = &[ - "`{`", "`[`", "`=>`", "`,`", "`>`", "`=`", "`:`", "`;`", "`|`", "`as`", - "`where`", - ]; - match tok { - TokenTree::Token(token) => match token.kind { - OpenDelim(token::DelimToken::Brace) - | OpenDelim(token::DelimToken::Bracket) - | Comma - | FatArrow - | Colon - | Eq - | Gt - | BinOp(token::Shr) - | Semi - | BinOp(token::Or) => IsInFollow::Yes, - Ident(name, false) if name == kw::As || name == kw::Where => { - IsInFollow::Yes - } - _ => IsInFollow::No(TOKENS), - }, - TokenTree::MetaVarDecl(_, _, frag) if frag.name == sym::block => { - IsInFollow::Yes - } - _ => IsInFollow::No(TOKENS), - } - } - sym::ident | sym::lifetime => { - // being a single token, idents and lifetimes are harmless - IsInFollow::Yes - } - sym::literal => { - // literals may be of a single token, or two tokens (negative numbers) - IsInFollow::Yes - } - sym::meta | sym::tt => { - // being either a single token or a delimited sequence, tt is - // harmless - IsInFollow::Yes - } - sym::vis => { - // Explicitly disallow `priv`, on the off chance it comes back. - const TOKENS: &[&str] = &["`,`", "an ident", "a type"]; - match tok { - TokenTree::Token(token) => match token.kind { - Comma => IsInFollow::Yes, - Ident(name, is_raw) if is_raw || name != kw::Priv => IsInFollow::Yes, - _ => { - if token.can_begin_type() { - IsInFollow::Yes - } else { - IsInFollow::No(TOKENS) - } - } - }, - TokenTree::MetaVarDecl(_, _, frag) - if frag.name == sym::ident - || frag.name == sym::ty - || frag.name == sym::path => - { - IsInFollow::Yes - } - _ => IsInFollow::No(TOKENS), - } - } - kw::Invalid => IsInFollow::Yes, - _ => IsInFollow::Invalid( - format!("invalid fragment specifier `{}`", frag), - VALID_FRAGMENT_NAMES_MSG, - ), - } - } -} - -fn has_legal_fragment_specifier( - sess: &ParseSess, - features: &Features, - attrs: &[ast::Attribute], - tok: &mbe::TokenTree, -) -> Result<(), String> { - debug!("has_legal_fragment_specifier({:?})", tok); - if let mbe::TokenTree::MetaVarDecl(_, _, ref frag_spec) = *tok { - let frag_span = tok.span(); - if !is_legal_fragment_specifier(sess, features, attrs, frag_spec.name, frag_span) { - return Err(frag_spec.to_string()); - } - } - Ok(()) -} - -fn is_legal_fragment_specifier( - _sess: &ParseSess, - _features: &Features, - _attrs: &[ast::Attribute], - frag_name: Symbol, - _frag_span: Span, -) -> bool { - /* - * If new fragment specifiers are invented in nightly, `_sess`, - * `_features`, `_attrs`, and `_frag_span` will be useful here - * for checking against feature gates. See past versions of - * this function. - */ - match frag_name { - sym::item - | sym::block - | sym::stmt - | sym::expr - | sym::pat - | sym::lifetime - | sym::path - | sym::ty - | sym::ident - | sym::meta - | sym::tt - | sym::vis - | sym::literal - | kw::Invalid => true, - _ => false, - } -} - -fn quoted_tt_to_string(tt: &mbe::TokenTree) -> String { - match *tt { - mbe::TokenTree::Token(ref token) => crate::print::pprust::token_to_string(&token), - mbe::TokenTree::MetaVar(_, name) => format!("${}", name), - mbe::TokenTree::MetaVarDecl(_, name, kind) => format!("${}:{}", name, kind), - _ => panic!( - "unexpected mbe::TokenTree::{{Sequence or Delimited}} \ - in follow set checker" - ), - } -} - -impl TokenTree { - /// Use this token tree as a matcher to parse given tts. - fn parse(cx: &ExtCtxt<'_>, mtch: &[mbe::TokenTree], tts: TokenStream) - -> NamedParseResult { - // `None` is because we're not interpolating - let directory = Directory { - path: Cow::from(cx.current_expansion.module.directory.as_path()), - ownership: cx.current_expansion.directory_ownership, - }; - parse(cx.parse_sess(), tts, mtch, Some(directory), true) - } -} - -/// Generates an appropriate parsing failure message. For EOF, this is "unexpected end...". For -/// other tokens, this is "unexpected token...". -fn parse_failure_msg(tok: &Token) -> String { - match tok.kind { - token::Eof => "unexpected end of macro invocation".to_string(), - _ => format!( - "no rules expected the token `{}`", - pprust::token_to_string(tok), - ), - } -} diff --git a/src/libsyntax/ext/mbe/quoted.rs b/src/libsyntax/ext/mbe/quoted.rs deleted file mode 100644 index 3cec4bc60e7..00000000000 --- a/src/libsyntax/ext/mbe/quoted.rs +++ /dev/null @@ -1,263 +0,0 @@ -use crate::ast; -use crate::ext::mbe::macro_parser; -use crate::ext::mbe::{TokenTree, KleeneOp, KleeneToken, SequenceRepetition, Delimited}; -use crate::parse::token::{self, Token}; -use crate::print::pprust; -use crate::sess::ParseSess; -use crate::symbol::kw; -use crate::tokenstream; - -use syntax_pos::Span; - -use rustc_data_structures::sync::Lrc; - -/// Takes a `tokenstream::TokenStream` and returns a `Vec`. Specifically, this -/// takes a generic `TokenStream`, such as is used in the rest of the compiler, and returns a -/// collection of `TokenTree` for use in parsing a macro. -/// -/// # Parameters -/// -/// - `input`: a token stream to read from, the contents of which we are parsing. -/// - `expect_matchers`: `parse` can be used to parse either the "patterns" or the "body" of a -/// macro. Both take roughly the same form _except_ that in a pattern, metavars are declared with -/// their "matcher" type. For example `$var:expr` or `$id:ident`. In this example, `expr` and -/// `ident` are "matchers". They are not present in the body of a macro rule -- just in the -/// pattern, so we pass a parameter to indicate whether to expect them or not. -/// - `sess`: the parsing session. Any errors will be emitted to this session. -/// - `features`, `attrs`: language feature flags and attributes so that we know whether to use -/// unstable features or not. -/// - `edition`: which edition are we in. -/// - `macro_node_id`: the NodeId of the macro we are parsing. -/// -/// # Returns -/// -/// A collection of `self::TokenTree`. There may also be some errors emitted to `sess`. -pub(super) fn parse( - input: tokenstream::TokenStream, - expect_matchers: bool, - sess: &ParseSess, -) -> Vec { - // Will contain the final collection of `self::TokenTree` - let mut result = Vec::new(); - - // For each token tree in `input`, parse the token into a `self::TokenTree`, consuming - // additional trees if need be. - let mut trees = input.trees(); - while let Some(tree) = trees.next() { - // Given the parsed tree, if there is a metavar and we are expecting matchers, actually - // parse out the matcher (i.e., in `$id:ident` this would parse the `:` and `ident`). - let tree = parse_tree( - tree, - &mut trees, - expect_matchers, - sess, - ); - match tree { - TokenTree::MetaVar(start_sp, ident) if expect_matchers => { - let span = match trees.next() { - Some(tokenstream::TokenTree::Token(Token { kind: token::Colon, span })) => { - match trees.next() { - Some(tokenstream::TokenTree::Token(token)) => match token.ident() { - Some((kind, _)) => { - let span = token.span.with_lo(start_sp.lo()); - result.push(TokenTree::MetaVarDecl(span, ident, kind)); - continue; - } - _ => token.span, - }, - tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span), - } - } - tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(start_sp), - }; - sess.missing_fragment_specifiers.borrow_mut().insert(span); - result.push(TokenTree::MetaVarDecl(span, ident, ast::Ident::invalid())); - } - - // Not a metavar or no matchers allowed, so just return the tree - _ => result.push(tree), - } - } - result -} - -/// Takes a `tokenstream::TokenTree` and returns a `self::TokenTree`. Specifically, this takes a -/// generic `TokenTree`, such as is used in the rest of the compiler, and returns a `TokenTree` -/// for use in parsing a macro. -/// -/// Converting the given tree may involve reading more tokens. -/// -/// # Parameters -/// -/// - `tree`: the tree we wish to convert. -/// - `trees`: an iterator over trees. We may need to read more tokens from it in order to finish -/// converting `tree` -/// - `expect_matchers`: same as for `parse` (see above). -/// - `sess`: the parsing session. Any errors will be emitted to this session. -/// - `features`, `attrs`: language feature flags and attributes so that we know whether to use -/// unstable features or not. -fn parse_tree( - tree: tokenstream::TokenTree, - trees: &mut impl Iterator, - expect_matchers: bool, - sess: &ParseSess, -) -> TokenTree { - // Depending on what `tree` is, we could be parsing different parts of a macro - match tree { - // `tree` is a `$` token. Look at the next token in `trees` - tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }) => match trees.next() { - // `tree` is followed by a delimited set of token trees. This indicates the beginning - // of a repetition sequence in the macro (e.g. `$(pat)*`). - Some(tokenstream::TokenTree::Delimited(span, delim, tts)) => { - // Must have `(` not `{` or `[` - if delim != token::Paren { - let tok = pprust::token_kind_to_string(&token::OpenDelim(delim)); - let msg = format!("expected `(`, found `{}`", tok); - sess.span_diagnostic.span_err(span.entire(), &msg); - } - // Parse the contents of the sequence itself - let sequence = parse( - tts.into(), - expect_matchers, - sess, - ); - // Get the Kleene operator and optional separator - let (separator, kleene) = parse_sep_and_kleene_op(trees, span.entire(), sess); - // Count the number of captured "names" (i.e., named metavars) - let name_captures = macro_parser::count_names(&sequence); - TokenTree::Sequence( - span, - Lrc::new(SequenceRepetition { - tts: sequence, - separator, - kleene, - num_captures: name_captures, - }), - ) - } - - // `tree` is followed by an `ident`. This could be `$meta_var` or the `$crate` special - // metavariable that names the crate of the invocation. - Some(tokenstream::TokenTree::Token(token)) if token.is_ident() => { - let (ident, is_raw) = token.ident().unwrap(); - let span = ident.span.with_lo(span.lo()); - if ident.name == kw::Crate && !is_raw { - TokenTree::token(token::Ident(kw::DollarCrate, is_raw), span) - } else { - TokenTree::MetaVar(span, ident) - } - } - - // `tree` is followed by a random token. This is an error. - Some(tokenstream::TokenTree::Token(token)) => { - let msg = - format!("expected identifier, found `{}`", pprust::token_to_string(&token),); - sess.span_diagnostic.span_err(token.span, &msg); - TokenTree::MetaVar(token.span, ast::Ident::invalid()) - } - - // There are no more tokens. Just return the `$` we already have. - None => TokenTree::token(token::Dollar, span), - }, - - // `tree` is an arbitrary token. Keep it. - tokenstream::TokenTree::Token(token) => TokenTree::Token(token), - - // `tree` is the beginning of a delimited set of tokens (e.g., `(` or `{`). We need to - // descend into the delimited set and further parse it. - tokenstream::TokenTree::Delimited(span, delim, tts) => TokenTree::Delimited( - span, - Lrc::new(Delimited { - delim, - tts: parse( - tts.into(), - expect_matchers, - sess, - ), - }), - ), - } -} - -/// Takes a token and returns `Some(KleeneOp)` if the token is `+` `*` or `?`. Otherwise, return -/// `None`. -fn kleene_op(token: &Token) -> Option { - match token.kind { - token::BinOp(token::Star) => Some(KleeneOp::ZeroOrMore), - token::BinOp(token::Plus) => Some(KleeneOp::OneOrMore), - token::Question => Some(KleeneOp::ZeroOrOne), - _ => None, - } -} - -/// Parse the next token tree of the input looking for a KleeneOp. Returns -/// -/// - Ok(Ok((op, span))) if the next token tree is a KleeneOp -/// - Ok(Err(tok, span)) if the next token tree is a token but not a KleeneOp -/// - Err(span) if the next token tree is not a token -fn parse_kleene_op( - input: &mut impl Iterator, - span: Span, -) -> Result, Span> { - match input.next() { - Some(tokenstream::TokenTree::Token(token)) => match kleene_op(&token) { - Some(op) => Ok(Ok((op, token.span))), - None => Ok(Err(token)), - }, - tree => Err(tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span)), - } -} - -/// Attempt to parse a single Kleene star, possibly with a separator. -/// -/// For example, in a pattern such as `$(a),*`, `a` is the pattern to be repeated, `,` is the -/// separator, and `*` is the Kleene operator. This function is specifically concerned with parsing -/// the last two tokens of such a pattern: namely, the optional separator and the Kleene operator -/// itself. Note that here we are parsing the _macro_ itself, rather than trying to match some -/// stream of tokens in an invocation of a macro. -/// -/// This function will take some input iterator `input` corresponding to `span` and a parsing -/// session `sess`. If the next one (or possibly two) tokens in `input` correspond to a Kleene -/// operator and separator, then a tuple with `(separator, KleeneOp)` is returned. Otherwise, an -/// error with the appropriate span is emitted to `sess` and a dummy value is returned. -fn parse_sep_and_kleene_op( - input: &mut impl Iterator, - span: Span, - sess: &ParseSess, -) -> (Option, KleeneToken) { - // We basically look at two token trees here, denoted as #1 and #2 below - let span = match parse_kleene_op(input, span) { - // #1 is a `?`, `+`, or `*` KleeneOp - Ok(Ok((op, span))) => return (None, KleeneToken::new(op, span)), - - // #1 is a separator followed by #2, a KleeneOp - Ok(Err(token)) => match parse_kleene_op(input, token.span) { - // #2 is the `?` Kleene op, which does not take a separator (error) - Ok(Ok((KleeneOp::ZeroOrOne, span))) => { - // Error! - sess.span_diagnostic.span_err( - token.span, - "the `?` macro repetition operator does not take a separator", - ); - - // Return a dummy - return (None, KleeneToken::new(KleeneOp::ZeroOrMore, span)); - } - - // #2 is a KleeneOp :D - Ok(Ok((op, span))) => return (Some(token), KleeneToken::new(op, span)), - - // #2 is a random token or not a token at all :( - Ok(Err(Token { span, .. })) | Err(span) => span, - }, - - // #1 is not a token - Err(span) => span, - }; - - // If we ever get to this point, we have experienced an "unexpected token" error - sess.span_diagnostic.span_err(span, "expected one of: `*`, `+`, or `?`"); - - // Return a dummy - (None, KleeneToken::new(KleeneOp::ZeroOrMore, span)) -} diff --git a/src/libsyntax/ext/mbe/transcribe.rs b/src/libsyntax/ext/mbe/transcribe.rs deleted file mode 100644 index da930436d81..00000000000 --- a/src/libsyntax/ext/mbe/transcribe.rs +++ /dev/null @@ -1,398 +0,0 @@ -use crate::ast::{Ident, Mac}; -use crate::ext::base::ExtCtxt; -use crate::ext::mbe; -use crate::ext::mbe::macro_parser::{MatchedNonterminal, MatchedSeq, NamedMatch}; -use crate::mut_visit::{self, MutVisitor}; -use crate::parse::token::{self, NtTT, Token}; -use crate::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndJoint}; - -use smallvec::{smallvec, SmallVec}; - -use errors::pluralise; -use rustc_data_structures::fx::FxHashMap; -use rustc_data_structures::sync::Lrc; -use syntax_pos::hygiene::{ExpnId, Transparency}; -use syntax_pos::Span; - -use std::mem; - -// A Marker adds the given mark to the syntax context. -struct Marker(ExpnId, Transparency); - -impl MutVisitor for Marker { - fn visit_span(&mut self, span: &mut Span) { - *span = span.apply_mark(self.0, self.1) - } - - fn visit_mac(&mut self, mac: &mut Mac) { - mut_visit::noop_visit_mac(mac, self) - } -} - -impl Marker { - fn visit_delim_span(&mut self, dspan: &mut DelimSpan) { - self.visit_span(&mut dspan.open); - self.visit_span(&mut dspan.close); - } -} - -/// An iterator over the token trees in a delimited token tree (`{ ... }`) or a sequence (`$(...)`). -enum Frame { - Delimited { forest: Lrc, idx: usize, span: DelimSpan }, - Sequence { forest: Lrc, idx: usize, sep: Option }, -} - -impl Frame { - /// Construct a new frame around the delimited set of tokens. - fn new(tts: Vec) -> Frame { - let forest = Lrc::new(mbe::Delimited { delim: token::NoDelim, tts }); - Frame::Delimited { forest, idx: 0, span: DelimSpan::dummy() } - } -} - -impl Iterator for Frame { - type Item = mbe::TokenTree; - - fn next(&mut self) -> Option { - match *self { - Frame::Delimited { ref forest, ref mut idx, .. } => { - *idx += 1; - forest.tts.get(*idx - 1).cloned() - } - Frame::Sequence { ref forest, ref mut idx, .. } => { - *idx += 1; - forest.tts.get(*idx - 1).cloned() - } - } - } -} - -/// This can do Macro-By-Example transcription. -/// - `interp` is a map of meta-variables to the tokens (non-terminals) they matched in the -/// invocation. We are assuming we already know there is a match. -/// - `src` is the RHS of the MBE, that is, the "example" we are filling in. -/// -/// For example, -/// -/// ```rust -/// macro_rules! foo { -/// ($id:ident) => { println!("{}", stringify!($id)); } -/// } -/// -/// foo!(bar); -/// ``` -/// -/// `interp` would contain `$id => bar` and `src` would contain `println!("{}", stringify!($id));`. -/// -/// `transcribe` would return a `TokenStream` containing `println!("{}", stringify!(bar));`. -/// -/// Along the way, we do some additional error checking. -pub(super) fn transcribe( - cx: &ExtCtxt<'_>, - interp: &FxHashMap, - src: Vec, - transparency: Transparency, -) -> TokenStream { - // Nothing for us to transcribe... - if src.is_empty() { - return TokenStream::default(); - } - - // We descend into the RHS (`src`), expanding things as we go. This stack contains the things - // we have yet to expand/are still expanding. We start the stack off with the whole RHS. - let mut stack: SmallVec<[Frame; 1]> = smallvec![Frame::new(src)]; - - // As we descend in the RHS, we will need to be able to match nested sequences of matchers. - // `repeats` keeps track of where we are in matching at each level, with the last element being - // the most deeply nested sequence. This is used as a stack. - let mut repeats = Vec::new(); - - // `result` contains resulting token stream from the TokenTree we just finished processing. At - // the end, this will contain the full result of transcription, but at arbitrary points during - // `transcribe`, `result` will contain subsets of the final result. - // - // Specifically, as we descend into each TokenTree, we will push the existing results onto the - // `result_stack` and clear `results`. We will then produce the results of transcribing the - // TokenTree into `results`. Then, as we unwind back out of the `TokenTree`, we will pop the - // `result_stack` and append `results` too it to produce the new `results` up to that point. - // - // Thus, if we try to pop the `result_stack` and it is empty, we have reached the top-level - // again, and we are done transcribing. - let mut result: Vec = Vec::new(); - let mut result_stack = Vec::new(); - let mut marker = Marker(cx.current_expansion.id, transparency); - - loop { - // Look at the last frame on the stack. - let tree = if let Some(tree) = stack.last_mut().unwrap().next() { - // If it still has a TokenTree we have not looked at yet, use that tree. - tree - } - // The else-case never produces a value for `tree` (it `continue`s or `return`s). - else { - // Otherwise, if we have just reached the end of a sequence and we can keep repeating, - // go back to the beginning of the sequence. - if let Frame::Sequence { idx, sep, .. } = stack.last_mut().unwrap() { - let (repeat_idx, repeat_len) = repeats.last_mut().unwrap(); - *repeat_idx += 1; - if repeat_idx < repeat_len { - *idx = 0; - if let Some(sep) = sep { - result.push(TokenTree::Token(sep.clone()).into()); - } - continue; - } - } - - // We are done with the top of the stack. Pop it. Depending on what it was, we do - // different things. Note that the outermost item must be the delimited, wrapped RHS - // that was passed in originally to `transcribe`. - match stack.pop().unwrap() { - // Done with a sequence. Pop from repeats. - Frame::Sequence { .. } => { - repeats.pop(); - } - - // We are done processing a Delimited. If this is the top-level delimited, we are - // done. Otherwise, we unwind the result_stack to append what we have produced to - // any previous results. - Frame::Delimited { forest, span, .. } => { - if result_stack.is_empty() { - // No results left to compute! We are back at the top-level. - return TokenStream::new(result); - } - - // Step back into the parent Delimited. - let tree = - TokenTree::Delimited(span, forest.delim, TokenStream::new(result).into()); - result = result_stack.pop().unwrap(); - result.push(tree.into()); - } - } - continue; - }; - - // At this point, we know we are in the middle of a TokenTree (the last one on `stack`). - // `tree` contains the next `TokenTree` to be processed. - match tree { - // We are descending into a sequence. We first make sure that the matchers in the RHS - // and the matches in `interp` have the same shape. Otherwise, either the caller or the - // macro writer has made a mistake. - seq @ mbe::TokenTree::Sequence(..) => { - match lockstep_iter_size(&seq, interp, &repeats) { - LockstepIterSize::Unconstrained => { - cx.span_fatal( - seq.span(), /* blame macro writer */ - "attempted to repeat an expression containing no syntax variables \ - matched as repeating at this depth", - ); - } - - LockstepIterSize::Contradiction(ref msg) => { - // FIXME: this really ought to be caught at macro definition time... It - // happens when two meta-variables are used in the same repetition in a - // sequence, but they come from different sequence matchers and repeat - // different amounts. - cx.span_fatal(seq.span(), &msg[..]); - } - - LockstepIterSize::Constraint(len, _) => { - // We do this to avoid an extra clone above. We know that this is a - // sequence already. - let (sp, seq) = if let mbe::TokenTree::Sequence(sp, seq) = seq { - (sp, seq) - } else { - unreachable!() - }; - - // Is the repetition empty? - if len == 0 { - if seq.kleene.op == mbe::KleeneOp::OneOrMore { - // FIXME: this really ought to be caught at macro definition - // time... It happens when the Kleene operator in the matcher and - // the body for the same meta-variable do not match. - cx.span_fatal(sp.entire(), "this must repeat at least once"); - } - } else { - // 0 is the initial counter (we have done 0 repretitions so far). `len` - // is the total number of reptitions we should generate. - repeats.push((0, len)); - - // The first time we encounter the sequence we push it to the stack. It - // then gets reused (see the beginning of the loop) until we are done - // repeating. - stack.push(Frame::Sequence { - idx: 0, - sep: seq.separator.clone(), - forest: seq, - }); - } - } - } - } - - // Replace the meta-var with the matched token tree from the invocation. - mbe::TokenTree::MetaVar(mut sp, mut ident) => { - // Find the matched nonterminal from the macro invocation, and use it to replace - // the meta-var. - if let Some(cur_matched) = lookup_cur_matched(ident, interp, &repeats) { - if let MatchedNonterminal(ref nt) = cur_matched { - // FIXME #2887: why do we apply a mark when matching a token tree meta-var - // (e.g. `$x:tt`), but not when we are matching any other type of token - // tree? - if let NtTT(ref tt) = **nt { - result.push(tt.clone().into()); - } else { - marker.visit_span(&mut sp); - let token = TokenTree::token(token::Interpolated(nt.clone()), sp); - result.push(token.into()); - } - } else { - // We were unable to descend far enough. This is an error. - cx.span_fatal( - sp, /* blame the macro writer */ - &format!("variable '{}' is still repeating at this depth", ident), - ); - } - } else { - // If we aren't able to match the meta-var, we push it back into the result but - // with modified syntax context. (I believe this supports nested macros). - marker.visit_span(&mut sp); - marker.visit_ident(&mut ident); - result.push(TokenTree::token(token::Dollar, sp).into()); - result.push(TokenTree::Token(Token::from_ast_ident(ident)).into()); - } - } - - // If we are entering a new delimiter, we push its contents to the `stack` to be - // processed, and we push all of the currently produced results to the `result_stack`. - // We will produce all of the results of the inside of the `Delimited` and then we will - // jump back out of the Delimited, pop the result_stack and add the new results back to - // the previous results (from outside the Delimited). - mbe::TokenTree::Delimited(mut span, delimited) => { - marker.visit_delim_span(&mut span); - stack.push(Frame::Delimited { forest: delimited, idx: 0, span }); - result_stack.push(mem::take(&mut result)); - } - - // Nothing much to do here. Just push the token to the result, being careful to - // preserve syntax context. - mbe::TokenTree::Token(token) => { - let mut tt = TokenTree::Token(token); - marker.visit_tt(&mut tt); - result.push(tt.into()); - } - - // There should be no meta-var declarations in the invocation of a macro. - mbe::TokenTree::MetaVarDecl(..) => panic!("unexpected `TokenTree::MetaVarDecl"), - } - } -} - -/// Lookup the meta-var named `ident` and return the matched token tree from the invocation using -/// the set of matches `interpolations`. -/// -/// See the definition of `repeats` in the `transcribe` function. `repeats` is used to descend -/// into the right place in nested matchers. If we attempt to descend too far, the macro writer has -/// made a mistake, and we return `None`. -fn lookup_cur_matched<'a>( - ident: Ident, - interpolations: &'a FxHashMap, - repeats: &[(usize, usize)], -) -> Option<&'a NamedMatch> { - interpolations.get(&ident).map(|matched| { - let mut matched = matched; - for &(idx, _) in repeats { - match matched { - MatchedNonterminal(_) => break, - MatchedSeq(ref ads, _) => matched = ads.get(idx).unwrap(), - } - } - - matched - }) -} - -/// An accumulator over a TokenTree to be used with `fold`. During transcription, we need to make -/// sure that the size of each sequence and all of its nested sequences are the same as the sizes -/// of all the matched (nested) sequences in the macro invocation. If they don't match, somebody -/// has made a mistake (either the macro writer or caller). -#[derive(Clone)] -enum LockstepIterSize { - /// No constraints on length of matcher. This is true for any TokenTree variants except a - /// `MetaVar` with an actual `MatchedSeq` (as opposed to a `MatchedNonterminal`). - Unconstrained, - - /// A `MetaVar` with an actual `MatchedSeq`. The length of the match and the name of the - /// meta-var are returned. - Constraint(usize, Ident), - - /// Two `Constraint`s on the same sequence had different lengths. This is an error. - Contradiction(String), -} - -impl LockstepIterSize { - /// Find incompatibilities in matcher/invocation sizes. - /// - `Unconstrained` is compatible with everything. - /// - `Contradiction` is incompatible with everything. - /// - `Constraint(len)` is only compatible with other constraints of the same length. - fn with(self, other: LockstepIterSize) -> LockstepIterSize { - match self { - LockstepIterSize::Unconstrained => other, - LockstepIterSize::Contradiction(_) => self, - LockstepIterSize::Constraint(l_len, ref l_id) => match other { - LockstepIterSize::Unconstrained => self, - LockstepIterSize::Contradiction(_) => other, - LockstepIterSize::Constraint(r_len, _) if l_len == r_len => self, - LockstepIterSize::Constraint(r_len, r_id) => { - let msg = format!( - "meta-variable `{}` repeats {} time{}, but `{}` repeats {} time{}", - l_id, - l_len, - pluralise!(l_len), - r_id, - r_len, - pluralise!(r_len), - ); - LockstepIterSize::Contradiction(msg) - } - }, - } - } -} - -/// Given a `tree`, make sure that all sequences have the same length as the matches for the -/// appropriate meta-vars in `interpolations`. -/// -/// Note that if `repeats` does not match the exact correct depth of a meta-var, -/// `lookup_cur_matched` will return `None`, which is why this still works even in the presnece of -/// multiple nested matcher sequences. -fn lockstep_iter_size( - tree: &mbe::TokenTree, - interpolations: &FxHashMap, - repeats: &[(usize, usize)], -) -> LockstepIterSize { - use mbe::TokenTree; - match *tree { - TokenTree::Delimited(_, ref delimed) => { - delimed.tts.iter().fold(LockstepIterSize::Unconstrained, |size, tt| { - size.with(lockstep_iter_size(tt, interpolations, repeats)) - }) - } - TokenTree::Sequence(_, ref seq) => { - seq.tts.iter().fold(LockstepIterSize::Unconstrained, |size, tt| { - size.with(lockstep_iter_size(tt, interpolations, repeats)) - }) - } - TokenTree::MetaVar(_, name) | TokenTree::MetaVarDecl(_, name, _) => { - match lookup_cur_matched(name, interpolations, repeats) { - Some(matched) => match matched { - MatchedNonterminal(_) => LockstepIterSize::Unconstrained, - MatchedSeq(ref ads, _) => LockstepIterSize::Constraint(ads.len(), name), - }, - _ => LockstepIterSize::Unconstrained, - } - } - TokenTree::Token(..) => LockstepIterSize::Unconstrained, - } -} diff --git a/src/libsyntax/ext/placeholders.rs b/src/libsyntax/ext/placeholders.rs deleted file mode 100644 index 4fae25bbde6..00000000000 --- a/src/libsyntax/ext/placeholders.rs +++ /dev/null @@ -1,349 +0,0 @@ -use crate::ast::{self, NodeId}; -use crate::source_map::{DUMMY_SP, dummy_spanned}; -use crate::ext::base::ExtCtxt; -use crate::ext::expand::{AstFragment, AstFragmentKind}; -use crate::tokenstream::TokenStream; -use crate::mut_visit::*; -use crate::ptr::P; -use crate::ThinVec; - -use smallvec::{smallvec, SmallVec}; - -use rustc_data_structures::fx::FxHashMap; - -pub fn placeholder(kind: AstFragmentKind, id: ast::NodeId) -> AstFragment { - fn mac_placeholder() -> ast::Mac { - ast::Mac { - path: ast::Path { span: DUMMY_SP, segments: Vec::new() }, - tts: TokenStream::default().into(), - delim: ast::MacDelimiter::Brace, - span: DUMMY_SP, - prior_type_ascription: None, - } - } - - let ident = ast::Ident::invalid(); - let attrs = Vec::new(); - let generics = ast::Generics::default(); - let vis = dummy_spanned(ast::VisibilityKind::Inherited); - let span = DUMMY_SP; - let expr_placeholder = || P(ast::Expr { - id, span, - attrs: ThinVec::new(), - kind: ast::ExprKind::Mac(mac_placeholder()), - }); - let ty = || P(ast::Ty { - id, - kind: ast::TyKind::Mac(mac_placeholder()), - span, - }); - let pat = || P(ast::Pat { - id, - kind: ast::PatKind::Mac(mac_placeholder()), - span, - }); - - match kind { - AstFragmentKind::Expr => AstFragment::Expr(expr_placeholder()), - AstFragmentKind::OptExpr => AstFragment::OptExpr(Some(expr_placeholder())), - AstFragmentKind::Items => AstFragment::Items(smallvec![P(ast::Item { - id, span, ident, vis, attrs, - kind: ast::ItemKind::Mac(mac_placeholder()), - tokens: None, - })]), - AstFragmentKind::TraitItems => AstFragment::TraitItems(smallvec![ast::TraitItem { - id, span, ident, attrs, generics, - kind: ast::TraitItemKind::Macro(mac_placeholder()), - tokens: None, - }]), - AstFragmentKind::ImplItems => AstFragment::ImplItems(smallvec![ast::ImplItem { - id, span, ident, vis, attrs, generics, - kind: ast::ImplItemKind::Macro(mac_placeholder()), - defaultness: ast::Defaultness::Final, - tokens: None, - }]), - AstFragmentKind::ForeignItems => - AstFragment::ForeignItems(smallvec![ast::ForeignItem { - id, span, ident, vis, attrs, - kind: ast::ForeignItemKind::Macro(mac_placeholder()), - }]), - AstFragmentKind::Pat => AstFragment::Pat(P(ast::Pat { - id, span, kind: ast::PatKind::Mac(mac_placeholder()), - })), - AstFragmentKind::Ty => AstFragment::Ty(P(ast::Ty { - id, span, kind: ast::TyKind::Mac(mac_placeholder()), - })), - AstFragmentKind::Stmts => AstFragment::Stmts(smallvec![{ - let mac = P((mac_placeholder(), ast::MacStmtStyle::Braces, ThinVec::new())); - ast::Stmt { id, span, kind: ast::StmtKind::Mac(mac) } - }]), - AstFragmentKind::Arms => AstFragment::Arms(smallvec![ - ast::Arm { - attrs: Default::default(), - body: expr_placeholder(), - guard: None, - id, - pat: pat(), - span, - is_placeholder: true, - } - ]), - AstFragmentKind::Fields => AstFragment::Fields(smallvec![ - ast::Field { - attrs: Default::default(), - expr: expr_placeholder(), - id, - ident, - is_shorthand: false, - span, - is_placeholder: true, - } - ]), - AstFragmentKind::FieldPats => AstFragment::FieldPats(smallvec![ - ast::FieldPat { - attrs: Default::default(), - id, - ident, - is_shorthand: false, - pat: pat(), - span, - is_placeholder: true, - } - ]), - AstFragmentKind::GenericParams => AstFragment::GenericParams(smallvec![{ - ast::GenericParam { - attrs: Default::default(), - bounds: Default::default(), - id, - ident, - is_placeholder: true, - kind: ast::GenericParamKind::Lifetime, - } - }]), - AstFragmentKind::Params => AstFragment::Params(smallvec![ - ast::Param { - attrs: Default::default(), - id, - pat: pat(), - span, - ty: ty(), - is_placeholder: true, - } - ]), - AstFragmentKind::StructFields => AstFragment::StructFields(smallvec![ - ast::StructField { - attrs: Default::default(), - id, - ident: None, - span, - ty: ty(), - vis, - is_placeholder: true, - } - ]), - AstFragmentKind::Variants => AstFragment::Variants(smallvec![ - ast::Variant { - attrs: Default::default(), - data: ast::VariantData::Struct(Default::default(), false), - disr_expr: None, - id, - ident, - span, - is_placeholder: true, - } - ]) - } -} - -pub struct PlaceholderExpander<'a, 'b> { - expanded_fragments: FxHashMap, - cx: &'a mut ExtCtxt<'b>, - monotonic: bool, -} - -impl<'a, 'b> PlaceholderExpander<'a, 'b> { - pub fn new(cx: &'a mut ExtCtxt<'b>, monotonic: bool) -> Self { - PlaceholderExpander { - cx, - expanded_fragments: FxHashMap::default(), - monotonic, - } - } - - pub fn add(&mut self, id: ast::NodeId, mut fragment: AstFragment, placeholders: Vec) { - fragment.mut_visit_with(self); - if let AstFragment::Items(mut items) = fragment { - for placeholder in placeholders { - match self.remove(placeholder) { - AstFragment::Items(derived_items) => items.extend(derived_items), - _ => unreachable!(), - } - } - fragment = AstFragment::Items(items); - } - self.expanded_fragments.insert(id, fragment); - } - - fn remove(&mut self, id: ast::NodeId) -> AstFragment { - self.expanded_fragments.remove(&id).unwrap() - } -} - -impl<'a, 'b> MutVisitor for PlaceholderExpander<'a, 'b> { - fn flat_map_arm(&mut self, arm: ast::Arm) -> SmallVec<[ast::Arm; 1]> { - if arm.is_placeholder { - self.remove(arm.id).make_arms() - } else { - noop_flat_map_arm(arm, self) - } - } - - fn flat_map_field(&mut self, field: ast::Field) -> SmallVec<[ast::Field; 1]> { - if field.is_placeholder { - self.remove(field.id).make_fields() - } else { - noop_flat_map_field(field, self) - } - } - - fn flat_map_field_pattern(&mut self, fp: ast::FieldPat) -> SmallVec<[ast::FieldPat; 1]> { - if fp.is_placeholder { - self.remove(fp.id).make_field_patterns() - } else { - noop_flat_map_field_pattern(fp, self) - } - } - - fn flat_map_generic_param( - &mut self, - param: ast::GenericParam - ) -> SmallVec<[ast::GenericParam; 1]> - { - if param.is_placeholder { - self.remove(param.id).make_generic_params() - } else { - noop_flat_map_generic_param(param, self) - } - } - - fn flat_map_param(&mut self, p: ast::Param) -> SmallVec<[ast::Param; 1]> { - if p.is_placeholder { - self.remove(p.id).make_params() - } else { - noop_flat_map_param(p, self) - } - } - - fn flat_map_struct_field(&mut self, sf: ast::StructField) -> SmallVec<[ast::StructField; 1]> { - if sf.is_placeholder { - self.remove(sf.id).make_struct_fields() - } else { - noop_flat_map_struct_field(sf, self) - } - } - - fn flat_map_variant(&mut self, variant: ast::Variant) -> SmallVec<[ast::Variant; 1]> { - if variant.is_placeholder { - self.remove(variant.id).make_variants() - } else { - noop_flat_map_variant(variant, self) - } - } - - fn flat_map_item(&mut self, item: P) -> SmallVec<[P; 1]> { - match item.kind { - ast::ItemKind::Mac(_) => return self.remove(item.id).make_items(), - ast::ItemKind::MacroDef(_) => return smallvec![item], - _ => {} - } - - noop_flat_map_item(item, self) - } - - fn flat_map_trait_item(&mut self, item: ast::TraitItem) -> SmallVec<[ast::TraitItem; 1]> { - match item.kind { - ast::TraitItemKind::Macro(_) => self.remove(item.id).make_trait_items(), - _ => noop_flat_map_trait_item(item, self), - } - } - - fn flat_map_impl_item(&mut self, item: ast::ImplItem) -> SmallVec<[ast::ImplItem; 1]> { - match item.kind { - ast::ImplItemKind::Macro(_) => self.remove(item.id).make_impl_items(), - _ => noop_flat_map_impl_item(item, self), - } - } - - fn flat_map_foreign_item(&mut self, item: ast::ForeignItem) -> SmallVec<[ast::ForeignItem; 1]> { - match item.kind { - ast::ForeignItemKind::Macro(_) => self.remove(item.id).make_foreign_items(), - _ => noop_flat_map_foreign_item(item, self), - } - } - - fn visit_expr(&mut self, expr: &mut P) { - match expr.kind { - ast::ExprKind::Mac(_) => *expr = self.remove(expr.id).make_expr(), - _ => noop_visit_expr(expr, self), - } - } - - fn filter_map_expr(&mut self, expr: P) -> Option> { - match expr.kind { - ast::ExprKind::Mac(_) => self.remove(expr.id).make_opt_expr(), - _ => noop_filter_map_expr(expr, self), - } - } - - fn flat_map_stmt(&mut self, stmt: ast::Stmt) -> SmallVec<[ast::Stmt; 1]> { - let (style, mut stmts) = match stmt.kind { - ast::StmtKind::Mac(mac) => (mac.1, self.remove(stmt.id).make_stmts()), - _ => return noop_flat_map_stmt(stmt, self), - }; - - if style == ast::MacStmtStyle::Semicolon { - if let Some(stmt) = stmts.pop() { - stmts.push(stmt.add_trailing_semicolon()); - } - } - - stmts - } - - fn visit_pat(&mut self, pat: &mut P) { - match pat.kind { - ast::PatKind::Mac(_) => *pat = self.remove(pat.id).make_pat(), - _ => noop_visit_pat(pat, self), - } - } - - fn visit_ty(&mut self, ty: &mut P) { - match ty.kind { - ast::TyKind::Mac(_) => *ty = self.remove(ty.id).make_ty(), - _ => noop_visit_ty(ty, self), - } - } - - fn visit_block(&mut self, block: &mut P) { - noop_visit_block(block, self); - - for stmt in block.stmts.iter_mut() { - if self.monotonic { - assert_eq!(stmt.id, ast::DUMMY_NODE_ID); - stmt.id = self.cx.resolver.next_node_id(); - } - } - } - - fn visit_mod(&mut self, module: &mut ast::Mod) { - noop_visit_mod(module, self); - module.items.retain(|item| match item.kind { - ast::ItemKind::Mac(_) if !self.cx.ecfg.keep_macs => false, // remove macro definitions - _ => true, - }); - } - - fn visit_mac(&mut self, _mac: &mut ast::Mac) { - // Do nothing. - } -} diff --git a/src/libsyntax/ext/proc_macro.rs b/src/libsyntax/ext/proc_macro.rs deleted file mode 100644 index 72062d2ffbf..00000000000 --- a/src/libsyntax/ext/proc_macro.rs +++ /dev/null @@ -1,219 +0,0 @@ -use crate::ast::{self, ItemKind, Attribute, Mac}; -use crate::attr::{mark_used, mark_known}; -use crate::errors::{Applicability, FatalError}; -use crate::ext::base::{self, *}; -use crate::ext::proc_macro_server; -use crate::parse::{self, token}; -use crate::symbol::sym; -use crate::tokenstream::{self, TokenStream}; -use crate::visit::Visitor; - -use rustc_data_structures::sync::Lrc; -use syntax_pos::{Span, DUMMY_SP}; - -const EXEC_STRATEGY: proc_macro::bridge::server::SameThread = - proc_macro::bridge::server::SameThread; - -pub struct BangProcMacro { - pub client: proc_macro::bridge::client::Client< - fn(proc_macro::TokenStream) -> proc_macro::TokenStream, - >, -} - -impl base::ProcMacro for BangProcMacro { - fn expand<'cx>(&self, - ecx: &'cx mut ExtCtxt<'_>, - span: Span, - input: TokenStream) - -> TokenStream { - let server = proc_macro_server::Rustc::new(ecx); - match self.client.run(&EXEC_STRATEGY, server, input) { - Ok(stream) => stream, - Err(e) => { - let msg = "proc macro panicked"; - let mut err = ecx.struct_span_fatal(span, msg); - if let Some(s) = e.as_str() { - err.help(&format!("message: {}", s)); - } - - err.emit(); - FatalError.raise(); - } - } - } -} - -pub struct AttrProcMacro { - pub client: proc_macro::bridge::client::Client< - fn(proc_macro::TokenStream, proc_macro::TokenStream) -> proc_macro::TokenStream, - >, -} - -impl base::AttrProcMacro for AttrProcMacro { - fn expand<'cx>(&self, - ecx: &'cx mut ExtCtxt<'_>, - span: Span, - annotation: TokenStream, - annotated: TokenStream) - -> TokenStream { - let server = proc_macro_server::Rustc::new(ecx); - match self.client.run(&EXEC_STRATEGY, server, annotation, annotated) { - Ok(stream) => stream, - Err(e) => { - let msg = "custom attribute panicked"; - let mut err = ecx.struct_span_fatal(span, msg); - if let Some(s) = e.as_str() { - err.help(&format!("message: {}", s)); - } - - err.emit(); - FatalError.raise(); - } - } - } -} - -pub struct ProcMacroDerive { - pub client: proc_macro::bridge::client::Client< - fn(proc_macro::TokenStream) -> proc_macro::TokenStream, - >, -} - -impl MultiItemModifier for ProcMacroDerive { - fn expand(&self, - ecx: &mut ExtCtxt<'_>, - span: Span, - _meta_item: &ast::MetaItem, - item: Annotatable) - -> Vec { - let item = match item { - Annotatable::Arm(..) | - Annotatable::Field(..) | - Annotatable::FieldPat(..) | - Annotatable::GenericParam(..) | - Annotatable::Param(..) | - Annotatable::StructField(..) | - Annotatable::Variant(..) - => panic!("unexpected annotatable"), - Annotatable::Item(item) => item, - Annotatable::ImplItem(_) | - Annotatable::TraitItem(_) | - Annotatable::ForeignItem(_) | - Annotatable::Stmt(_) | - Annotatable::Expr(_) => { - ecx.span_err(span, "proc-macro derives may only be \ - applied to a struct, enum, or union"); - return Vec::new() - } - }; - match item.kind { - ItemKind::Struct(..) | - ItemKind::Enum(..) | - ItemKind::Union(..) => {}, - _ => { - ecx.span_err(span, "proc-macro derives may only be \ - applied to a struct, enum, or union"); - return Vec::new() - } - } - - let token = token::Interpolated(Lrc::new(token::NtItem(item))); - let input = tokenstream::TokenTree::token(token, DUMMY_SP).into(); - - let server = proc_macro_server::Rustc::new(ecx); - let stream = match self.client.run(&EXEC_STRATEGY, server, input) { - Ok(stream) => stream, - Err(e) => { - let msg = "proc-macro derive panicked"; - let mut err = ecx.struct_span_fatal(span, msg); - if let Some(s) = e.as_str() { - err.help(&format!("message: {}", s)); - } - - err.emit(); - FatalError.raise(); - } - }; - - let error_count_before = ecx.parse_sess.span_diagnostic.err_count(); - let msg = "proc-macro derive produced unparseable tokens"; - - let mut parser = parse::stream_to_parser(ecx.parse_sess, stream, Some("proc-macro derive")); - let mut items = vec![]; - - loop { - match parser.parse_item() { - Ok(None) => break, - Ok(Some(item)) => { - items.push(Annotatable::Item(item)) - } - Err(mut err) => { - // FIXME: handle this better - err.cancel(); - ecx.struct_span_fatal(span, msg).emit(); - FatalError.raise(); - } - } - } - - - // fail if there have been errors emitted - if ecx.parse_sess.span_diagnostic.err_count() > error_count_before { - ecx.struct_span_fatal(span, msg).emit(); - FatalError.raise(); - } - - items - } -} - -crate struct MarkAttrs<'a>(crate &'a [ast::Name]); - -impl<'a> Visitor<'a> for MarkAttrs<'a> { - fn visit_attribute(&mut self, attr: &Attribute) { - if let Some(ident) = attr.ident() { - if self.0.contains(&ident.name) { - mark_used(attr); - mark_known(attr); - } - } - } - - fn visit_mac(&mut self, _mac: &Mac) {} -} - -pub fn is_proc_macro_attr(attr: &Attribute) -> bool { - [sym::proc_macro, sym::proc_macro_attribute, sym::proc_macro_derive] - .iter().any(|kind| attr.check_name(*kind)) -} - -crate fn collect_derives(cx: &mut ExtCtxt<'_>, attrs: &mut Vec) -> Vec { - let mut result = Vec::new(); - attrs.retain(|attr| { - if attr.path != sym::derive { - return true; - } - if !attr.is_meta_item_list() { - cx.struct_span_err(attr.span, "malformed `derive` attribute input") - .span_suggestion( - attr.span, - "missing traits to be derived", - "#[derive(Trait1, Trait2, ...)]".to_owned(), - Applicability::HasPlaceholders, - ).emit(); - return false; - } - - match attr.parse_derive_paths(cx.parse_sess) { - Ok(traits) => { - result.extend(traits); - true - } - Err(mut e) => { - e.emit(); - false - } - } - }); - result -} diff --git a/src/libsyntax/ext/proc_macro_server.rs b/src/libsyntax/ext/proc_macro_server.rs deleted file mode 100644 index e0e1a1c468c..00000000000 --- a/src/libsyntax/ext/proc_macro_server.rs +++ /dev/null @@ -1,712 +0,0 @@ -use crate::ast; -use crate::ext::base::ExtCtxt; -use crate::parse::{self, token}; -use crate::parse::lexer::comments; -use crate::print::pprust; -use crate::sess::ParseSess; -use crate::tokenstream::{self, DelimSpan, IsJoint::*, TokenStream, TreeAndJoint}; - -use errors::Diagnostic; -use rustc_data_structures::sync::Lrc; -use syntax_pos::{BytePos, FileName, MultiSpan, Pos, SourceFile, Span}; -use syntax_pos::symbol::{kw, sym, Symbol}; - -use proc_macro::{Delimiter, Level, LineColumn, Spacing}; -use proc_macro::bridge::{server, TokenTree}; -use std::{ascii, panic}; -use std::ops::Bound; - -trait FromInternal { - fn from_internal(x: T) -> Self; -} - -trait ToInternal { - fn to_internal(self) -> T; -} - -impl FromInternal for Delimiter { - fn from_internal(delim: token::DelimToken) -> Delimiter { - match delim { - token::Paren => Delimiter::Parenthesis, - token::Brace => Delimiter::Brace, - token::Bracket => Delimiter::Bracket, - token::NoDelim => Delimiter::None, - } - } -} - -impl ToInternal for Delimiter { - fn to_internal(self) -> token::DelimToken { - match self { - Delimiter::Parenthesis => token::Paren, - Delimiter::Brace => token::Brace, - Delimiter::Bracket => token::Bracket, - Delimiter::None => token::NoDelim, - } - } -} - -impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec)> - for TokenTree -{ - fn from_internal(((tree, is_joint), sess, stack): (TreeAndJoint, &ParseSess, &mut Vec)) - -> Self { - use crate::parse::token::*; - - let joint = is_joint == Joint; - let Token { kind, span } = match tree { - tokenstream::TokenTree::Delimited(span, delim, tts) => { - let delimiter = Delimiter::from_internal(delim); - return TokenTree::Group(Group { - delimiter, - stream: tts.into(), - span, - }); - } - tokenstream::TokenTree::Token(token) => token, - }; - - macro_rules! tt { - ($ty:ident { $($field:ident $(: $value:expr)*),+ $(,)? }) => ( - TokenTree::$ty(self::$ty { - $($field $(: $value)*,)+ - span, - }) - ); - ($ty:ident::$method:ident($($value:expr),*)) => ( - TokenTree::$ty(self::$ty::$method($($value,)* span)) - ); - } - macro_rules! op { - ($a:expr) => { - tt!(Punct::new($a, joint)) - }; - ($a:expr, $b:expr) => {{ - stack.push(tt!(Punct::new($b, joint))); - tt!(Punct::new($a, true)) - }}; - ($a:expr, $b:expr, $c:expr) => {{ - stack.push(tt!(Punct::new($c, joint))); - stack.push(tt!(Punct::new($b, true))); - tt!(Punct::new($a, true)) - }}; - } - - match kind { - Eq => op!('='), - Lt => op!('<'), - Le => op!('<', '='), - EqEq => op!('=', '='), - Ne => op!('!', '='), - Ge => op!('>', '='), - Gt => op!('>'), - AndAnd => op!('&', '&'), - OrOr => op!('|', '|'), - Not => op!('!'), - Tilde => op!('~'), - BinOp(Plus) => op!('+'), - BinOp(Minus) => op!('-'), - BinOp(Star) => op!('*'), - BinOp(Slash) => op!('/'), - BinOp(Percent) => op!('%'), - BinOp(Caret) => op!('^'), - BinOp(And) => op!('&'), - BinOp(Or) => op!('|'), - BinOp(Shl) => op!('<', '<'), - BinOp(Shr) => op!('>', '>'), - BinOpEq(Plus) => op!('+', '='), - BinOpEq(Minus) => op!('-', '='), - BinOpEq(Star) => op!('*', '='), - BinOpEq(Slash) => op!('/', '='), - BinOpEq(Percent) => op!('%', '='), - BinOpEq(Caret) => op!('^', '='), - BinOpEq(And) => op!('&', '='), - BinOpEq(Or) => op!('|', '='), - BinOpEq(Shl) => op!('<', '<', '='), - BinOpEq(Shr) => op!('>', '>', '='), - At => op!('@'), - Dot => op!('.'), - DotDot => op!('.', '.'), - DotDotDot => op!('.', '.', '.'), - DotDotEq => op!('.', '.', '='), - Comma => op!(','), - Semi => op!(';'), - Colon => op!(':'), - ModSep => op!(':', ':'), - RArrow => op!('-', '>'), - LArrow => op!('<', '-'), - FatArrow => op!('=', '>'), - Pound => op!('#'), - Dollar => op!('$'), - Question => op!('?'), - SingleQuote => op!('\''), - - Ident(name, false) if name == kw::DollarCrate => tt!(Ident::dollar_crate()), - Ident(name, is_raw) => tt!(Ident::new(name, is_raw)), - Lifetime(name) => { - let ident = ast::Ident::new(name, span).without_first_quote(); - stack.push(tt!(Ident::new(ident.name, false))); - tt!(Punct::new('\'', true)) - } - Literal(lit) => tt!(Literal { lit }), - DocComment(c) => { - let style = comments::doc_comment_style(&c.as_str()); - let stripped = comments::strip_doc_comment_decoration(&c.as_str()); - let mut escaped = String::new(); - for ch in stripped.chars() { - escaped.extend(ch.escape_debug()); - } - let stream = vec![ - Ident(sym::doc, false), - Eq, - TokenKind::lit(token::Str, Symbol::intern(&escaped), None), - ] - .into_iter() - .map(|kind| tokenstream::TokenTree::token(kind, span)) - .collect(); - stack.push(TokenTree::Group(Group { - delimiter: Delimiter::Bracket, - stream, - span: DelimSpan::from_single(span), - })); - if style == ast::AttrStyle::Inner { - stack.push(tt!(Punct::new('!', false))); - } - tt!(Punct::new('#', false)) - } - - Interpolated(nt) => { - let stream = parse::nt_to_tokenstream(&nt, sess, span); - TokenTree::Group(Group { - delimiter: Delimiter::None, - stream, - span: DelimSpan::from_single(span), - }) - } - - OpenDelim(..) | CloseDelim(..) => unreachable!(), - Whitespace | Comment | Shebang(..) | Unknown(..) | Eof => unreachable!(), - } - } -} - -impl ToInternal for TokenTree { - fn to_internal(self) -> TokenStream { - use crate::parse::token::*; - - let (ch, joint, span) = match self { - TokenTree::Punct(Punct { ch, joint, span }) => (ch, joint, span), - TokenTree::Group(Group { - delimiter, - stream, - span, - }) => { - return tokenstream::TokenTree::Delimited( - span, - delimiter.to_internal(), - stream.into(), - ) - .into(); - } - TokenTree::Ident(self::Ident { sym, is_raw, span }) => { - return tokenstream::TokenTree::token(Ident(sym, is_raw), span).into(); - } - TokenTree::Literal(self::Literal { - lit: token::Lit { kind: token::Integer, symbol, suffix }, - span, - }) if symbol.as_str().starts_with("-") => { - let minus = BinOp(BinOpToken::Minus); - let symbol = Symbol::intern(&symbol.as_str()[1..]); - let integer = TokenKind::lit(token::Integer, symbol, suffix); - let a = tokenstream::TokenTree::token(minus, span); - let b = tokenstream::TokenTree::token(integer, span); - return vec![a, b].into_iter().collect(); - } - TokenTree::Literal(self::Literal { - lit: token::Lit { kind: token::Float, symbol, suffix }, - span, - }) if symbol.as_str().starts_with("-") => { - let minus = BinOp(BinOpToken::Minus); - let symbol = Symbol::intern(&symbol.as_str()[1..]); - let float = TokenKind::lit(token::Float, symbol, suffix); - let a = tokenstream::TokenTree::token(minus, span); - let b = tokenstream::TokenTree::token(float, span); - return vec![a, b].into_iter().collect(); - } - TokenTree::Literal(self::Literal { lit, span }) => { - return tokenstream::TokenTree::token(Literal(lit), span).into() - } - }; - - let kind = match ch { - '=' => Eq, - '<' => Lt, - '>' => Gt, - '!' => Not, - '~' => Tilde, - '+' => BinOp(Plus), - '-' => BinOp(Minus), - '*' => BinOp(Star), - '/' => BinOp(Slash), - '%' => BinOp(Percent), - '^' => BinOp(Caret), - '&' => BinOp(And), - '|' => BinOp(Or), - '@' => At, - '.' => Dot, - ',' => Comma, - ';' => Semi, - ':' => Colon, - '#' => Pound, - '$' => Dollar, - '?' => Question, - '\'' => SingleQuote, - _ => unreachable!(), - }; - - let tree = tokenstream::TokenTree::token(kind, span); - TokenStream::new(vec![(tree, if joint { Joint } else { NonJoint })]) - } -} - -impl ToInternal for Level { - fn to_internal(self) -> errors::Level { - match self { - Level::Error => errors::Level::Error, - Level::Warning => errors::Level::Warning, - Level::Note => errors::Level::Note, - Level::Help => errors::Level::Help, - _ => unreachable!("unknown proc_macro::Level variant: {:?}", self), - } - } -} - -#[derive(Clone)] -pub struct TokenStreamIter { - cursor: tokenstream::Cursor, - stack: Vec>, -} - -#[derive(Clone)] -pub struct Group { - delimiter: Delimiter, - stream: TokenStream, - span: DelimSpan, -} - -#[derive(Copy, Clone, PartialEq, Eq, Hash)] -pub struct Punct { - ch: char, - // NB. not using `Spacing` here because it doesn't implement `Hash`. - joint: bool, - span: Span, -} - -impl Punct { - fn new(ch: char, joint: bool, span: Span) -> Punct { - const LEGAL_CHARS: &[char] = &['=', '<', '>', '!', '~', '+', '-', '*', '/', '%', '^', - '&', '|', '@', '.', ',', ';', ':', '#', '$', '?', '\'']; - if !LEGAL_CHARS.contains(&ch) { - panic!("unsupported character `{:?}`", ch) - } - Punct { ch, joint, span } - } -} - -#[derive(Copy, Clone, PartialEq, Eq, Hash)] -pub struct Ident { - sym: Symbol, - is_raw: bool, - span: Span, -} - -impl Ident { - fn is_valid(string: &str) -> bool { - let mut chars = string.chars(); - if let Some(start) = chars.next() { - rustc_lexer::is_id_start(start) && chars.all(rustc_lexer::is_id_continue) - } else { - false - } - } - fn new(sym: Symbol, is_raw: bool, span: Span) -> Ident { - let string = sym.as_str(); - if !Self::is_valid(&string) { - panic!("`{:?}` is not a valid identifier", string) - } - if is_raw && !sym.can_be_raw() { - panic!("`{}` cannot be a raw identifier", string); - } - Ident { sym, is_raw, span } - } - fn dollar_crate(span: Span) -> Ident { - // `$crate` is accepted as an ident only if it comes from the compiler. - Ident { sym: kw::DollarCrate, is_raw: false, span } - } -} - -// FIXME(eddyb) `Literal` should not expose internal `Debug` impls. -#[derive(Clone, Debug)] -pub struct Literal { - lit: token::Lit, - span: Span, -} - -pub(crate) struct Rustc<'a> { - sess: &'a ParseSess, - def_site: Span, - call_site: Span, - mixed_site: Span, -} - -impl<'a> Rustc<'a> { - pub fn new(cx: &'a ExtCtxt<'_>) -> Self { - let expn_data = cx.current_expansion.id.expn_data(); - Rustc { - sess: cx.parse_sess, - def_site: cx.with_def_site_ctxt(expn_data.def_site), - call_site: cx.with_call_site_ctxt(expn_data.call_site), - mixed_site: cx.with_mixed_site_ctxt(expn_data.call_site), - } - } - - fn lit(&mut self, kind: token::LitKind, symbol: Symbol, suffix: Option) -> Literal { - Literal { - lit: token::Lit::new(kind, symbol, suffix), - span: server::Span::call_site(self), - } - } -} - -impl server::Types for Rustc<'_> { - type TokenStream = TokenStream; - type TokenStreamBuilder = tokenstream::TokenStreamBuilder; - type TokenStreamIter = TokenStreamIter; - type Group = Group; - type Punct = Punct; - type Ident = Ident; - type Literal = Literal; - type SourceFile = Lrc; - type MultiSpan = Vec; - type Diagnostic = Diagnostic; - type Span = Span; -} - -impl server::TokenStream for Rustc<'_> { - fn new(&mut self) -> Self::TokenStream { - TokenStream::default() - } - fn is_empty(&mut self, stream: &Self::TokenStream) -> bool { - stream.is_empty() - } - fn from_str(&mut self, src: &str) -> Self::TokenStream { - parse::parse_stream_from_source_str( - FileName::proc_macro_source_code(src), - src.to_string(), - self.sess, - Some(self.call_site), - ) - } - fn to_string(&mut self, stream: &Self::TokenStream) -> String { - pprust::tts_to_string(stream.clone()) - } - fn from_token_tree( - &mut self, - tree: TokenTree, - ) -> Self::TokenStream { - tree.to_internal() - } - fn into_iter(&mut self, stream: Self::TokenStream) -> Self::TokenStreamIter { - TokenStreamIter { - cursor: stream.trees(), - stack: vec![], - } - } -} - -impl server::TokenStreamBuilder for Rustc<'_> { - fn new(&mut self) -> Self::TokenStreamBuilder { - tokenstream::TokenStreamBuilder::new() - } - fn push(&mut self, builder: &mut Self::TokenStreamBuilder, stream: Self::TokenStream) { - builder.push(stream); - } - fn build(&mut self, builder: Self::TokenStreamBuilder) -> Self::TokenStream { - builder.build() - } -} - -impl server::TokenStreamIter for Rustc<'_> { - fn next( - &mut self, - iter: &mut Self::TokenStreamIter, - ) -> Option> { - loop { - let tree = iter.stack.pop().or_else(|| { - let next = iter.cursor.next_with_joint()?; - Some(TokenTree::from_internal((next, self.sess, &mut iter.stack))) - })?; - // HACK: The condition "dummy span + group with empty delimiter" represents an AST - // fragment approximately converted into a token stream. This may happen, for - // example, with inputs to proc macro attributes, including derives. Such "groups" - // need to flattened during iteration over stream's token trees. - // Eventually this needs to be removed in favor of keeping original token trees - // and not doing the roundtrip through AST. - if let TokenTree::Group(ref group) = tree { - if group.delimiter == Delimiter::None && group.span.entire().is_dummy() { - iter.cursor.append(group.stream.clone()); - continue; - } - } - return Some(tree); - } - } -} - -impl server::Group for Rustc<'_> { - fn new(&mut self, delimiter: Delimiter, stream: Self::TokenStream) -> Self::Group { - Group { - delimiter, - stream, - span: DelimSpan::from_single(server::Span::call_site(self)), - } - } - fn delimiter(&mut self, group: &Self::Group) -> Delimiter { - group.delimiter - } - fn stream(&mut self, group: &Self::Group) -> Self::TokenStream { - group.stream.clone() - } - fn span(&mut self, group: &Self::Group) -> Self::Span { - group.span.entire() - } - fn span_open(&mut self, group: &Self::Group) -> Self::Span { - group.span.open - } - fn span_close(&mut self, group: &Self::Group) -> Self::Span { - group.span.close - } - fn set_span(&mut self, group: &mut Self::Group, span: Self::Span) { - group.span = DelimSpan::from_single(span); - } -} - -impl server::Punct for Rustc<'_> { - fn new(&mut self, ch: char, spacing: Spacing) -> Self::Punct { - Punct::new(ch, spacing == Spacing::Joint, server::Span::call_site(self)) - } - fn as_char(&mut self, punct: Self::Punct) -> char { - punct.ch - } - fn spacing(&mut self, punct: Self::Punct) -> Spacing { - if punct.joint { - Spacing::Joint - } else { - Spacing::Alone - } - } - fn span(&mut self, punct: Self::Punct) -> Self::Span { - punct.span - } - fn with_span(&mut self, punct: Self::Punct, span: Self::Span) -> Self::Punct { - Punct { span, ..punct } - } -} - -impl server::Ident for Rustc<'_> { - fn new(&mut self, string: &str, span: Self::Span, is_raw: bool) -> Self::Ident { - Ident::new(Symbol::intern(string), is_raw, span) - } - fn span(&mut self, ident: Self::Ident) -> Self::Span { - ident.span - } - fn with_span(&mut self, ident: Self::Ident, span: Self::Span) -> Self::Ident { - Ident { span, ..ident } - } -} - -impl server::Literal for Rustc<'_> { - // FIXME(eddyb) `Literal` should not expose internal `Debug` impls. - fn debug(&mut self, literal: &Self::Literal) -> String { - format!("{:?}", literal) - } - fn integer(&mut self, n: &str) -> Self::Literal { - self.lit(token::Integer, Symbol::intern(n), None) - } - fn typed_integer(&mut self, n: &str, kind: &str) -> Self::Literal { - self.lit(token::Integer, Symbol::intern(n), Some(Symbol::intern(kind))) - } - fn float(&mut self, n: &str) -> Self::Literal { - self.lit(token::Float, Symbol::intern(n), None) - } - fn f32(&mut self, n: &str) -> Self::Literal { - self.lit(token::Float, Symbol::intern(n), Some(sym::f32)) - } - fn f64(&mut self, n: &str) -> Self::Literal { - self.lit(token::Float, Symbol::intern(n), Some(sym::f64)) - } - fn string(&mut self, string: &str) -> Self::Literal { - let mut escaped = String::new(); - for ch in string.chars() { - escaped.extend(ch.escape_debug()); - } - self.lit(token::Str, Symbol::intern(&escaped), None) - } - fn character(&mut self, ch: char) -> Self::Literal { - let mut escaped = String::new(); - escaped.extend(ch.escape_unicode()); - self.lit(token::Char, Symbol::intern(&escaped), None) - } - fn byte_string(&mut self, bytes: &[u8]) -> Self::Literal { - let string = bytes - .iter() - .cloned() - .flat_map(ascii::escape_default) - .map(Into::::into) - .collect::(); - self.lit(token::ByteStr, Symbol::intern(&string), None) - } - fn span(&mut self, literal: &Self::Literal) -> Self::Span { - literal.span - } - fn set_span(&mut self, literal: &mut Self::Literal, span: Self::Span) { - literal.span = span; - } - fn subspan( - &mut self, - literal: &Self::Literal, - start: Bound, - end: Bound, - ) -> Option { - let span = literal.span; - let length = span.hi().to_usize() - span.lo().to_usize(); - - let start = match start { - Bound::Included(lo) => lo, - Bound::Excluded(lo) => lo + 1, - Bound::Unbounded => 0, - }; - - let end = match end { - Bound::Included(hi) => hi + 1, - Bound::Excluded(hi) => hi, - Bound::Unbounded => length, - }; - - // Bounds check the values, preventing addition overflow and OOB spans. - if start > u32::max_value() as usize - || end > u32::max_value() as usize - || (u32::max_value() - start as u32) < span.lo().to_u32() - || (u32::max_value() - end as u32) < span.lo().to_u32() - || start >= end - || end > length - { - return None; - } - - let new_lo = span.lo() + BytePos::from_usize(start); - let new_hi = span.lo() + BytePos::from_usize(end); - Some(span.with_lo(new_lo).with_hi(new_hi)) - } -} - -impl server::SourceFile for Rustc<'_> { - fn eq(&mut self, file1: &Self::SourceFile, file2: &Self::SourceFile) -> bool { - Lrc::ptr_eq(file1, file2) - } - fn path(&mut self, file: &Self::SourceFile) -> String { - match file.name { - FileName::Real(ref path) => path - .to_str() - .expect("non-UTF8 file path in `proc_macro::SourceFile::path`") - .to_string(), - _ => file.name.to_string(), - } - } - fn is_real(&mut self, file: &Self::SourceFile) -> bool { - file.is_real_file() - } -} - -impl server::MultiSpan for Rustc<'_> { - fn new(&mut self) -> Self::MultiSpan { - vec![] - } - fn push(&mut self, spans: &mut Self::MultiSpan, span: Self::Span) { - spans.push(span) - } -} - -impl server::Diagnostic for Rustc<'_> { - fn new(&mut self, level: Level, msg: &str, spans: Self::MultiSpan) -> Self::Diagnostic { - let mut diag = Diagnostic::new(level.to_internal(), msg); - diag.set_span(MultiSpan::from_spans(spans)); - diag - } - fn sub( - &mut self, - diag: &mut Self::Diagnostic, - level: Level, - msg: &str, - spans: Self::MultiSpan, - ) { - diag.sub(level.to_internal(), msg, MultiSpan::from_spans(spans), None); - } - fn emit(&mut self, diag: Self::Diagnostic) { - self.sess.span_diagnostic.emit_diagnostic(&diag); - } -} - -impl server::Span for Rustc<'_> { - fn debug(&mut self, span: Self::Span) -> String { - format!("{:?} bytes({}..{})", span.ctxt(), span.lo().0, span.hi().0) - } - fn def_site(&mut self) -> Self::Span { - self.def_site - } - fn call_site(&mut self) -> Self::Span { - self.call_site - } - fn mixed_site(&mut self) -> Self::Span { - self.mixed_site - } - fn source_file(&mut self, span: Self::Span) -> Self::SourceFile { - self.sess.source_map().lookup_char_pos(span.lo()).file - } - fn parent(&mut self, span: Self::Span) -> Option { - span.parent() - } - fn source(&mut self, span: Self::Span) -> Self::Span { - span.source_callsite() - } - fn start(&mut self, span: Self::Span) -> LineColumn { - let loc = self.sess.source_map().lookup_char_pos(span.lo()); - LineColumn { - line: loc.line, - column: loc.col.to_usize(), - } - } - fn end(&mut self, span: Self::Span) -> LineColumn { - let loc = self.sess.source_map().lookup_char_pos(span.hi()); - LineColumn { - line: loc.line, - column: loc.col.to_usize(), - } - } - fn join(&mut self, first: Self::Span, second: Self::Span) -> Option { - let self_loc = self.sess.source_map().lookup_char_pos(first.lo()); - let other_loc = self.sess.source_map().lookup_char_pos(second.lo()); - - if self_loc.file.name != other_loc.file.name { - return None; - } - - Some(first.to(second)) - } - fn resolved_at(&mut self, span: Self::Span, at: Self::Span) -> Self::Span { - span.with_ctxt(at.ctxt()) - } - fn source_text(&mut self, span: Self::Span) -> Option { - self.sess.source_map().span_to_snippet(span).ok() - } -} diff --git a/src/libsyntax/feature_gate/check.rs b/src/libsyntax/feature_gate/check.rs index c07b6050afe..172511f0f09 100644 --- a/src/libsyntax/feature_gate/check.rs +++ b/src/libsyntax/feature_gate/check.rs @@ -56,7 +56,7 @@ macro_rules! gate_feature { }; } -crate fn check_attribute(attr: &ast::Attribute, parse_sess: &ParseSess, features: &Features) { +pub fn check_attribute(attr: &ast::Attribute, parse_sess: &ParseSess, features: &Features) { PostExpansionVisitor { parse_sess, features }.visit_attribute(attr) } diff --git a/src/libsyntax/feature_gate/mod.rs b/src/libsyntax/feature_gate/mod.rs index ca13ab36205..ba970618c0e 100644 --- a/src/libsyntax/feature_gate/mod.rs +++ b/src/libsyntax/feature_gate/mod.rs @@ -58,8 +58,7 @@ pub use builtin_attrs::{ deprecated_attributes, is_builtin_attr, is_builtin_attr_name, }; pub use check::{ - check_crate, get_features, feature_err, emit_feature_err, + check_crate, check_attribute, get_features, feature_err, emit_feature_err, Stability, GateIssue, UnstableFeatures, EXPLAIN_STMT_ATTR_SYNTAX, EXPLAIN_UNSIZED_TUPLE_COERCION, }; -crate use check::check_attribute; diff --git a/src/libsyntax/lib.rs b/src/libsyntax/lib.rs index fa75f5624f8..a68b7fdf931 100644 --- a/src/libsyntax/lib.rs +++ b/src/libsyntax/lib.rs @@ -13,17 +13,12 @@ #![feature(crate_visibility_modifier)] #![feature(label_break_value)] #![feature(nll)] -#![feature(proc_macro_diagnostic)] -#![feature(proc_macro_internals)] -#![feature(proc_macro_span)] #![feature(try_trait)] #![feature(slice_patterns)] #![feature(unicode_internals)] #![recursion_limit="256"] -extern crate proc_macro; - pub use errors; use rustc_data_structures::sync::Lock; use rustc_index::bit_set::GrowableBitSet; @@ -34,26 +29,7 @@ use syntax_pos::edition::Edition; #[cfg(test)] mod tests; -const MACRO_ARGUMENTS: Option<&'static str> = Some("macro arguments"); - -// A variant of 'try!' that panics on an Err. This is used as a crutch on the -// way towards a non-panic!-prone parser. It should be used for fatal parsing -// errors; eventually we plan to convert all code using panictry to just use -// normal try. -#[macro_export] -macro_rules! panictry { - ($e:expr) => ({ - use std::result::Result::{Ok, Err}; - use errors::FatalError; - match $e { - Ok(e) => e, - Err(mut e) => { - e.emit(); - FatalError.raise() - } - } - }) -} +pub const MACRO_ARGUMENTS: Option<&'static str> = Some("macro arguments"); // A variant of 'panictry!' that works on a Vec instead of a single DiagnosticBuilder. macro_rules! panictry_buffer { @@ -157,19 +133,4 @@ pub mod print { mod helpers; } -pub mod ext { - mod placeholders; - mod proc_macro_server; - - pub use syntax_pos::hygiene; - pub use mbe::macro_rules::compile_declarative_macro; - pub mod allocator; - pub mod base; - pub mod build; - pub mod expand; - pub mod proc_macro; - - crate mod mbe; -} - pub mod early_buffered_lints; diff --git a/src/libsyntax/parse/literal.rs b/src/libsyntax/parse/literal.rs index 54e523430e4..14e1696610a 100644 --- a/src/libsyntax/parse/literal.rs +++ b/src/libsyntax/parse/literal.rs @@ -212,7 +212,7 @@ impl Lit { /// Attempts to recover an AST literal from semantic literal. /// This function is used when the original token doesn't exist (e.g. the literal is created /// by an AST-based macro) or unavailable (e.g. from HIR pretty-printing). - crate fn from_lit_kind(kind: LitKind, span: Span) -> Lit { + pub fn from_lit_kind(kind: LitKind, span: Span) -> Lit { Lit { token: kind.to_lit_token(), kind, span } } diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index 7914fdbf978..64ec943294a 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -5,7 +5,7 @@ mod item; mod module; mod ty; mod path; -crate use path::PathStyle; +pub use path::PathStyle; mod stmt; mod generics; mod diagnostics; @@ -130,7 +130,7 @@ pub struct Parser<'a> { /// Name of the root module this parser originated from. If `None`, then the /// name is not known. This does not change while the parser is descending /// into modules, and sub-parsers have new values for this name. - crate root_module_name: Option, + pub root_module_name: Option, expected_tokens: Vec, token_cursor: TokenCursor, desugar_doc_comments: bool, @@ -148,7 +148,7 @@ pub struct Parser<'a> { /// error. pub(super) unclosed_delims: Vec, last_unexpected_token_span: Option, - crate last_type_ascription: Option<(Span, bool /* likely path typo */)>, + pub last_type_ascription: Option<(Span, bool /* likely path typo */)>, /// If present, this `Parser` is not parsing Rust code but rather a macro call. subparser_name: Option<&'static str>, } @@ -330,7 +330,7 @@ enum TokenExpectType { } impl<'a> Parser<'a> { - crate fn new( + pub fn new( sess: &'a ParseSess, tokens: TokenStream, directory: Option>, @@ -1003,7 +1003,7 @@ impl<'a> Parser<'a> { } } - crate fn process_potential_macro_variable(&mut self) { + pub fn process_potential_macro_variable(&mut self) { self.token = match self.token.kind { token::Dollar if self.token.span.from_expansion() && self.look_ahead(1, |t| t.is_ident()) => { @@ -1037,7 +1037,7 @@ impl<'a> Parser<'a> { } /// Parses a single token tree from the input. - crate fn parse_token_tree(&mut self) -> TokenTree { + pub fn parse_token_tree(&mut self) -> TokenTree { match self.token.kind { token::OpenDelim(..) => { let frame = mem::replace(&mut self.token_cursor.frame, @@ -1099,7 +1099,7 @@ impl<'a> Parser<'a> { /// If the following element can't be a tuple (i.e., it's a function definition), then /// it's not a tuple struct field), and the contents within the parentheses isn't valid, /// so emit a proper diagnostic. - crate fn parse_visibility(&mut self, can_take_tuple: bool) -> PResult<'a, Visibility> { + pub fn parse_visibility(&mut self, can_take_tuple: bool) -> PResult<'a, Visibility> { maybe_whole!(self, NtVis, |x| x); self.expected_tokens.push(TokenType::Keyword(kw::Crate)); diff --git a/src/libsyntax/parse/parser/diagnostics.rs b/src/libsyntax/parse/parser/diagnostics.rs index a6884ec2c72..32a41bb0e27 100644 --- a/src/libsyntax/parse/parser/diagnostics.rs +++ b/src/libsyntax/parse/parser/diagnostics.rs @@ -170,7 +170,7 @@ impl RecoverQPath for Expr { } impl<'a> Parser<'a> { - crate fn fatal(&self, m: &str) -> DiagnosticBuilder<'a> { + pub fn fatal(&self, m: &str) -> DiagnosticBuilder<'a> { self.span_fatal(self.token.span, m) } @@ -190,11 +190,11 @@ impl<'a> Parser<'a> { self.sess.span_diagnostic.span_err(sp, m) } - crate fn struct_span_err>(&self, sp: S, m: &str) -> DiagnosticBuilder<'a> { + pub fn struct_span_err>(&self, sp: S, m: &str) -> DiagnosticBuilder<'a> { self.sess.span_diagnostic.struct_span_err(sp, m) } - crate fn span_bug>(&self, sp: S, m: &str) -> ! { + pub fn span_bug>(&self, sp: S, m: &str) -> ! { self.sess.span_diagnostic.span_bug(sp, m) } diff --git a/src/libsyntax/parse/parser/expr.rs b/src/libsyntax/parse/parser/expr.rs index bf9526dece2..273f5a5ffa3 100644 --- a/src/libsyntax/parse/parser/expr.rs +++ b/src/libsyntax/parse/parser/expr.rs @@ -1233,7 +1233,7 @@ impl<'a> Parser<'a> { } /// Matches `'-' lit | lit` (cf. `ast_validation::AstValidator::check_expr_within_pat`). - crate fn parse_literal_maybe_minus(&mut self) -> PResult<'a, P> { + pub fn parse_literal_maybe_minus(&mut self) -> PResult<'a, P> { maybe_whole_expr!(self); let minus_lo = self.token.span; diff --git a/src/libsyntax/parse/parser/item.rs b/src/libsyntax/parse/parser/item.rs index 6a1cbd2ebb7..0acfd1450d8 100644 --- a/src/libsyntax/parse/parser/item.rs +++ b/src/libsyntax/parse/parser/item.rs @@ -674,7 +674,7 @@ impl<'a> Parser<'a> { } /// Parses an impl item. - crate fn parse_impl_item(&mut self, at_end: &mut bool) -> PResult<'a, ImplItem> { + pub fn parse_impl_item(&mut self, at_end: &mut bool) -> PResult<'a, ImplItem> { maybe_whole!(self, NtImplItem, |x| x); let attrs = self.parse_outer_attributes()?; let mut unclosed_delims = vec![]; @@ -850,7 +850,7 @@ impl<'a> Parser<'a> { } /// Parses the items in a trait declaration. - crate fn parse_trait_item(&mut self, at_end: &mut bool) -> PResult<'a, TraitItem> { + pub fn parse_trait_item(&mut self, at_end: &mut bool) -> PResult<'a, TraitItem> { maybe_whole!(self, NtTraitItem, |x| x); let attrs = self.parse_outer_attributes()?; let mut unclosed_delims = vec![]; @@ -1116,7 +1116,7 @@ impl<'a> Parser<'a> { } /// Parses a foreign item. - crate fn parse_foreign_item(&mut self, extern_sp: Span) -> PResult<'a, ForeignItem> { + pub fn parse_foreign_item(&mut self, extern_sp: Span) -> PResult<'a, ForeignItem> { maybe_whole!(self, NtForeignItem, |ni| ni); let attrs = self.parse_outer_attributes()?; diff --git a/src/libsyntax/parse/parser/pat.rs b/src/libsyntax/parse/parser/pat.rs index b30e0636bb1..af795e51792 100644 --- a/src/libsyntax/parse/parser/pat.rs +++ b/src/libsyntax/parse/parser/pat.rs @@ -34,7 +34,7 @@ impl<'a> Parser<'a> { /// Corresponds to `pat` in RFC 2535 and does not admit or-patterns /// at the top level. Used when parsing the parameters of lambda expressions, /// functions, function pointers, and `pat` macro fragments. - crate fn parse_pat(&mut self, expected: Expected) -> PResult<'a, P> { + pub fn parse_pat(&mut self, expected: Expected) -> PResult<'a, P> { self.parse_pat_with_range_pat(true, expected) } diff --git a/src/libsyntax/parse/parser/path.rs b/src/libsyntax/parse/parser/path.rs index 0f28fb11cde..639d61a2b5c 100644 --- a/src/libsyntax/parse/parser/path.rs +++ b/src/libsyntax/parse/parser/path.rs @@ -13,7 +13,7 @@ use errors::{Applicability, pluralise}; /// Specifies how to parse a path. #[derive(Copy, Clone, PartialEq)] -crate enum PathStyle { +pub enum PathStyle { /// In some contexts, notably in expressions, paths with generic arguments are ambiguous /// with something else. For example, in expressions `segment < ....` can be interpreted /// as a comparison and `segment ( ....` can be interpreted as a function call. @@ -88,7 +88,7 @@ impl<'a> Parser<'a> { /// `a::b::C::` (with disambiguator) /// `Fn(Args)` (without disambiguator) /// `Fn::(Args)` (with disambiguator) - crate fn parse_path(&mut self, style: PathStyle) -> PResult<'a, Path> { + pub fn parse_path(&mut self, style: PathStyle) -> PResult<'a, Path> { maybe_whole!(self, NtPath, |path| { if style == PathStyle::Mod && path.segments.iter().any(|segment| segment.args.is_some()) { diff --git a/src/libsyntax/parse/parser/stmt.rs b/src/libsyntax/parse/parser/stmt.rs index f656bc11e1c..4b1fdb2eef3 100644 --- a/src/libsyntax/parse/parser/stmt.rs +++ b/src/libsyntax/parse/parser/stmt.rs @@ -19,7 +19,7 @@ use errors::Applicability; impl<'a> Parser<'a> { /// Parses a statement. This stops just before trailing semicolons on everything but items. /// e.g., a `StmtKind::Semi` parses to a `StmtKind::Expr`, leaving the trailing `;` unconsumed. - crate fn parse_stmt(&mut self) -> PResult<'a, Option> { + pub fn parse_stmt(&mut self) -> PResult<'a, Option> { Ok(self.parse_stmt_(true)) } @@ -299,7 +299,7 @@ impl<'a> Parser<'a> { } /// Parses a block. No inner attributes are allowed. - crate fn parse_block(&mut self) -> PResult<'a, P> { + pub fn parse_block(&mut self) -> PResult<'a, P> { maybe_whole!(self, NtBlock, |x| x); let lo = self.token.span; @@ -421,7 +421,7 @@ impl<'a> Parser<'a> { } /// Parses a statement, including the trailing semicolon. - crate fn parse_full_stmt(&mut self, macro_legacy_warnings: bool) -> PResult<'a, Option> { + pub fn parse_full_stmt(&mut self, macro_legacy_warnings: bool) -> PResult<'a, Option> { // Skip looking for a trailing semicolon when we have an interpolated statement. maybe_whole!(self, NtStmt, |x| Some(x)); diff --git a/src/libsyntax/parse/parser/ty.rs b/src/libsyntax/parse/parser/ty.rs index ae9f4d06c05..86c94b680b2 100644 --- a/src/libsyntax/parse/parser/ty.rs +++ b/src/libsyntax/parse/parser/ty.rs @@ -433,13 +433,13 @@ impl<'a> Parser<'a> { } } - crate fn check_lifetime(&mut self) -> bool { + pub fn check_lifetime(&mut self) -> bool { self.expected_tokens.push(TokenType::Lifetime); self.token.is_lifetime() } /// Parses a single lifetime `'a` or panics. - crate fn expect_lifetime(&mut self) -> Lifetime { + pub fn expect_lifetime(&mut self) -> Lifetime { if let Some(ident) = self.token.lifetime() { let span = self.token.span; self.bump(); diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index eb74ab2b919..e527989fb0b 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -285,7 +285,7 @@ impl TokenKind { } impl Token { - crate fn new(kind: TokenKind, span: Span) -> Self { + pub fn new(kind: TokenKind, span: Span) -> Self { Token { kind, span } } @@ -295,12 +295,12 @@ impl Token { } /// Recovers a `Token` from an `ast::Ident`. This creates a raw identifier if necessary. - crate fn from_ast_ident(ident: ast::Ident) -> Self { + pub fn from_ast_ident(ident: ast::Ident) -> Self { Token::new(Ident(ident.name, ident.is_raw_guess()), ident.span) } /// Return this token by value and leave a dummy token in its place. - crate fn take(&mut self) -> Self { + pub fn take(&mut self) -> Self { mem::replace(self, Token::dummy()) } @@ -321,7 +321,7 @@ impl Token { } /// Returns `true` if the token can appear at the start of an expression. - crate fn can_begin_expr(&self) -> bool { + pub fn can_begin_expr(&self) -> bool { match self.kind { Ident(name, is_raw) => ident_can_begin_expr(name, self.span, is_raw), // value name or keyword @@ -353,7 +353,7 @@ impl Token { } /// Returns `true` if the token can appear at the start of a type. - crate fn can_begin_type(&self) -> bool { + pub fn can_begin_type(&self) -> bool { match self.kind { Ident(name, is_raw) => ident_can_begin_type(name, self.span, is_raw), // type name or keyword @@ -396,7 +396,7 @@ impl Token { } /// Returns `true` if the token is any literal - crate fn is_lit(&self) -> bool { + pub fn is_lit(&self) -> bool { match self.kind { Literal(..) => true, _ => false, @@ -412,7 +412,7 @@ impl Token { /// Returns `true` if the token is any literal, a minus (which can prefix a literal, /// for example a '-42', or one of the boolean idents). - crate fn can_begin_literal_or_bool(&self) -> bool { + pub fn can_begin_literal_or_bool(&self) -> bool { match self.kind { Literal(..) | BinOp(Minus) => true, Ident(name, false) if name.is_bool_lit() => true, diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index 0ff1c26bac2..db6832d6423 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -156,7 +156,7 @@ use IsJoint::*; impl TokenStream { /// Given a `TokenStream` with a `Stream` of only two arguments, return a new `TokenStream` /// separating the two arguments with a comma for diagnostic suggestions. - pub(crate) fn add_comma(&self) -> Option<(TokenStream, Span)> { + pub fn add_comma(&self) -> Option<(TokenStream, Span)> { // Used to suggest if a user writes `foo!(a b);` let mut suggestion = None; let mut iter = self.0.iter().enumerate().peekable(); diff --git a/src/libsyntax_expand/Cargo.toml b/src/libsyntax_expand/Cargo.toml new file mode 100644 index 00000000000..f063753f599 --- /dev/null +++ b/src/libsyntax_expand/Cargo.toml @@ -0,0 +1,26 @@ +[package] +authors = ["The Rust Project Developers"] +name = "syntax_expand" +version = "0.0.0" +edition = "2018" +build = false + +[lib] +name = "syntax_expand" +path = "lib.rs" +doctest = false + +[dependencies] +bitflags = "1.0" +rustc_serialize = { path = "../libserialize", package = "serialize" } +log = "0.4" +scoped-tls = "1.0" +lazy_static = "1.0.0" +syntax_pos = { path = "../libsyntax_pos" } +errors = { path = "../librustc_errors", package = "rustc_errors" } +rustc_data_structures = { path = "../librustc_data_structures" } +rustc_index = { path = "../librustc_index" } +rustc_lexer = { path = "../librustc_lexer" } +rustc_target = { path = "../librustc_target" } +smallvec = { version = "0.6.7", features = ["union", "may_dangle"] } +syntax = { path = "../libsyntax" } diff --git a/src/libsyntax_expand/allocator.rs b/src/libsyntax_expand/allocator.rs new file mode 100644 index 00000000000..3526be17721 --- /dev/null +++ b/src/libsyntax_expand/allocator.rs @@ -0,0 +1,75 @@ +use syntax::{ast, attr, visit}; +use syntax::symbol::{sym, Symbol}; +use syntax_pos::Span; + +#[derive(Clone, Copy)] +pub enum AllocatorKind { + Global, + DefaultLib, + DefaultExe, +} + +impl AllocatorKind { + pub fn fn_name(&self, base: &str) -> String { + match *self { + AllocatorKind::Global => format!("__rg_{}", base), + AllocatorKind::DefaultLib => format!("__rdl_{}", base), + AllocatorKind::DefaultExe => format!("__rde_{}", base), + } + } +} + +pub enum AllocatorTy { + Layout, + Ptr, + ResultPtr, + Unit, + Usize, +} + +pub struct AllocatorMethod { + pub name: &'static str, + pub inputs: &'static [AllocatorTy], + pub output: AllocatorTy, +} + +pub static ALLOCATOR_METHODS: &[AllocatorMethod] = &[ + AllocatorMethod { + name: "alloc", + inputs: &[AllocatorTy::Layout], + output: AllocatorTy::ResultPtr, + }, + AllocatorMethod { + name: "dealloc", + inputs: &[AllocatorTy::Ptr, AllocatorTy::Layout], + output: AllocatorTy::Unit, + }, + AllocatorMethod { + name: "realloc", + inputs: &[AllocatorTy::Ptr, AllocatorTy::Layout, AllocatorTy::Usize], + output: AllocatorTy::ResultPtr, + }, + AllocatorMethod { + name: "alloc_zeroed", + inputs: &[AllocatorTy::Layout], + output: AllocatorTy::ResultPtr, + }, +]; + +pub fn global_allocator_spans(krate: &ast::Crate) -> Vec { + struct Finder { name: Symbol, spans: Vec } + impl<'ast> visit::Visitor<'ast> for Finder { + fn visit_item(&mut self, item: &'ast ast::Item) { + if item.ident.name == self.name && + attr::contains_name(&item.attrs, sym::rustc_std_internal_symbol) { + self.spans.push(item.span); + } + visit::walk_item(self, item) + } + } + + let name = Symbol::intern(&AllocatorKind::Global.fn_name("alloc")); + let mut f = Finder { name, spans: Vec::new() }; + visit::walk_crate(&mut f, krate); + f.spans +} diff --git a/src/libsyntax_expand/base.rs b/src/libsyntax_expand/base.rs new file mode 100644 index 00000000000..593e06f29b9 --- /dev/null +++ b/src/libsyntax_expand/base.rs @@ -0,0 +1,1189 @@ +use crate::expand::{self, AstFragment, Invocation}; +use crate::hygiene::ExpnId; + +use syntax::ast::{self, NodeId, Attribute, Name, PatKind}; +use syntax::attr::{self, HasAttrs, Stability, Deprecation}; +use syntax::source_map::SourceMap; +use syntax::edition::Edition; +use syntax::mut_visit::{self, MutVisitor}; +use syntax::parse::{self, parser, DirectoryOwnership}; +use syntax::parse::token; +use syntax::ptr::P; +use syntax::sess::ParseSess; +use syntax::symbol::{kw, sym, Ident, Symbol}; +use syntax::{ThinVec, MACRO_ARGUMENTS}; +use syntax::tokenstream::{self, TokenStream}; +use syntax::visit::Visitor; + +use errors::{DiagnosticBuilder, DiagnosticId}; +use smallvec::{smallvec, SmallVec}; +use syntax_pos::{FileName, Span, MultiSpan, DUMMY_SP}; +use syntax_pos::hygiene::{AstPass, ExpnData, ExpnKind}; + +use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::sync::{self, Lrc}; +use std::iter; +use std::path::PathBuf; +use std::rc::Rc; +use std::default::Default; + +pub use syntax_pos::hygiene::MacroKind; + +#[derive(Debug,Clone)] +pub enum Annotatable { + Item(P), + TraitItem(P), + ImplItem(P), + ForeignItem(P), + Stmt(P), + Expr(P), + Arm(ast::Arm), + Field(ast::Field), + FieldPat(ast::FieldPat), + GenericParam(ast::GenericParam), + Param(ast::Param), + StructField(ast::StructField), + Variant(ast::Variant), +} + +impl HasAttrs for Annotatable { + fn attrs(&self) -> &[Attribute] { + match *self { + Annotatable::Item(ref item) => &item.attrs, + Annotatable::TraitItem(ref trait_item) => &trait_item.attrs, + Annotatable::ImplItem(ref impl_item) => &impl_item.attrs, + Annotatable::ForeignItem(ref foreign_item) => &foreign_item.attrs, + Annotatable::Stmt(ref stmt) => stmt.attrs(), + Annotatable::Expr(ref expr) => &expr.attrs, + Annotatable::Arm(ref arm) => &arm.attrs, + Annotatable::Field(ref field) => &field.attrs, + Annotatable::FieldPat(ref fp) => &fp.attrs, + Annotatable::GenericParam(ref gp) => &gp.attrs, + Annotatable::Param(ref p) => &p.attrs, + Annotatable::StructField(ref sf) => &sf.attrs, + Annotatable::Variant(ref v) => &v.attrs(), + } + } + + fn visit_attrs)>(&mut self, f: F) { + match self { + Annotatable::Item(item) => item.visit_attrs(f), + Annotatable::TraitItem(trait_item) => trait_item.visit_attrs(f), + Annotatable::ImplItem(impl_item) => impl_item.visit_attrs(f), + Annotatable::ForeignItem(foreign_item) => foreign_item.visit_attrs(f), + Annotatable::Stmt(stmt) => stmt.visit_attrs(f), + Annotatable::Expr(expr) => expr.visit_attrs(f), + Annotatable::Arm(arm) => arm.visit_attrs(f), + Annotatable::Field(field) => field.visit_attrs(f), + Annotatable::FieldPat(fp) => fp.visit_attrs(f), + Annotatable::GenericParam(gp) => gp.visit_attrs(f), + Annotatable::Param(p) => p.visit_attrs(f), + Annotatable::StructField(sf) => sf.visit_attrs(f), + Annotatable::Variant(v) => v.visit_attrs(f), + } + } +} + +impl Annotatable { + pub fn span(&self) -> Span { + match *self { + Annotatable::Item(ref item) => item.span, + Annotatable::TraitItem(ref trait_item) => trait_item.span, + Annotatable::ImplItem(ref impl_item) => impl_item.span, + Annotatable::ForeignItem(ref foreign_item) => foreign_item.span, + Annotatable::Stmt(ref stmt) => stmt.span, + Annotatable::Expr(ref expr) => expr.span, + Annotatable::Arm(ref arm) => arm.span, + Annotatable::Field(ref field) => field.span, + Annotatable::FieldPat(ref fp) => fp.pat.span, + Annotatable::GenericParam(ref gp) => gp.ident.span, + Annotatable::Param(ref p) => p.span, + Annotatable::StructField(ref sf) => sf.span, + Annotatable::Variant(ref v) => v.span, + } + } + + pub fn visit_with<'a, V: Visitor<'a>>(&'a self, visitor: &mut V) { + match self { + Annotatable::Item(item) => visitor.visit_item(item), + Annotatable::TraitItem(trait_item) => visitor.visit_trait_item(trait_item), + Annotatable::ImplItem(impl_item) => visitor.visit_impl_item(impl_item), + Annotatable::ForeignItem(foreign_item) => visitor.visit_foreign_item(foreign_item), + Annotatable::Stmt(stmt) => visitor.visit_stmt(stmt), + Annotatable::Expr(expr) => visitor.visit_expr(expr), + Annotatable::Arm(arm) => visitor.visit_arm(arm), + Annotatable::Field(field) => visitor.visit_field(field), + Annotatable::FieldPat(fp) => visitor.visit_field_pattern(fp), + Annotatable::GenericParam(gp) => visitor.visit_generic_param(gp), + Annotatable::Param(p) => visitor.visit_param(p), + Annotatable::StructField(sf) =>visitor.visit_struct_field(sf), + Annotatable::Variant(v) => visitor.visit_variant(v), + } + } + + pub fn expect_item(self) -> P { + match self { + Annotatable::Item(i) => i, + _ => panic!("expected Item") + } + } + + pub fn map_item_or(self, mut f: F, mut or: G) -> Annotatable + where F: FnMut(P) -> P, + G: FnMut(Annotatable) -> Annotatable + { + match self { + Annotatable::Item(i) => Annotatable::Item(f(i)), + _ => or(self) + } + } + + pub fn expect_trait_item(self) -> ast::TraitItem { + match self { + Annotatable::TraitItem(i) => i.into_inner(), + _ => panic!("expected Item") + } + } + + pub fn expect_impl_item(self) -> ast::ImplItem { + match self { + Annotatable::ImplItem(i) => i.into_inner(), + _ => panic!("expected Item") + } + } + + pub fn expect_foreign_item(self) -> ast::ForeignItem { + match self { + Annotatable::ForeignItem(i) => i.into_inner(), + _ => panic!("expected foreign item") + } + } + + pub fn expect_stmt(self) -> ast::Stmt { + match self { + Annotatable::Stmt(stmt) => stmt.into_inner(), + _ => panic!("expected statement"), + } + } + + pub fn expect_expr(self) -> P { + match self { + Annotatable::Expr(expr) => expr, + _ => panic!("expected expression"), + } + } + + pub fn expect_arm(self) -> ast::Arm { + match self { + Annotatable::Arm(arm) => arm, + _ => panic!("expected match arm") + } + } + + pub fn expect_field(self) -> ast::Field { + match self { + Annotatable::Field(field) => field, + _ => panic!("expected field") + } + } + + pub fn expect_field_pattern(self) -> ast::FieldPat { + match self { + Annotatable::FieldPat(fp) => fp, + _ => panic!("expected field pattern") + } + } + + pub fn expect_generic_param(self) -> ast::GenericParam { + match self { + Annotatable::GenericParam(gp) => gp, + _ => panic!("expected generic parameter") + } + } + + pub fn expect_param(self) -> ast::Param { + match self { + Annotatable::Param(param) => param, + _ => panic!("expected parameter") + } + } + + pub fn expect_struct_field(self) -> ast::StructField { + match self { + Annotatable::StructField(sf) => sf, + _ => panic!("expected struct field") + } + } + + pub fn expect_variant(self) -> ast::Variant { + match self { + Annotatable::Variant(v) => v, + _ => panic!("expected variant") + } + } + + pub fn derive_allowed(&self) -> bool { + match *self { + Annotatable::Item(ref item) => match item.kind { + ast::ItemKind::Struct(..) | + ast::ItemKind::Enum(..) | + ast::ItemKind::Union(..) => true, + _ => false, + }, + _ => false, + } + } +} + +// `meta_item` is the annotation, and `item` is the item being modified. +// FIXME Decorators should follow the same pattern too. +pub trait MultiItemModifier { + fn expand(&self, + ecx: &mut ExtCtxt<'_>, + span: Span, + meta_item: &ast::MetaItem, + item: Annotatable) + -> Vec; +} + +impl MultiItemModifier for F + where F: Fn(&mut ExtCtxt<'_>, Span, &ast::MetaItem, Annotatable) -> T, + T: Into>, +{ + fn expand(&self, + ecx: &mut ExtCtxt<'_>, + span: Span, + meta_item: &ast::MetaItem, + item: Annotatable) + -> Vec { + (*self)(ecx, span, meta_item, item).into() + } +} + +impl Into> for Annotatable { + fn into(self) -> Vec { + vec![self] + } +} + +pub trait ProcMacro { + fn expand<'cx>(&self, + ecx: &'cx mut ExtCtxt<'_>, + span: Span, + ts: TokenStream) + -> TokenStream; +} + +impl ProcMacro for F + where F: Fn(TokenStream) -> TokenStream +{ + fn expand<'cx>(&self, + _ecx: &'cx mut ExtCtxt<'_>, + _span: Span, + ts: TokenStream) + -> TokenStream { + // FIXME setup implicit context in TLS before calling self. + (*self)(ts) + } +} + +pub trait AttrProcMacro { + fn expand<'cx>(&self, + ecx: &'cx mut ExtCtxt<'_>, + span: Span, + annotation: TokenStream, + annotated: TokenStream) + -> TokenStream; +} + +impl AttrProcMacro for F + where F: Fn(TokenStream, TokenStream) -> TokenStream +{ + fn expand<'cx>(&self, + _ecx: &'cx mut ExtCtxt<'_>, + _span: Span, + annotation: TokenStream, + annotated: TokenStream) + -> TokenStream { + // FIXME setup implicit context in TLS before calling self. + (*self)(annotation, annotated) + } +} + +/// Represents a thing that maps token trees to Macro Results +pub trait TTMacroExpander { + fn expand<'cx>( + &self, + ecx: &'cx mut ExtCtxt<'_>, + span: Span, + input: TokenStream, + ) -> Box; +} + +pub type MacroExpanderFn = + for<'cx> fn(&'cx mut ExtCtxt<'_>, Span, TokenStream) + -> Box; + +impl TTMacroExpander for F + where F: for<'cx> Fn(&'cx mut ExtCtxt<'_>, Span, TokenStream) + -> Box +{ + fn expand<'cx>( + &self, + ecx: &'cx mut ExtCtxt<'_>, + span: Span, + mut input: TokenStream, + ) -> Box { + struct AvoidInterpolatedIdents; + + impl MutVisitor for AvoidInterpolatedIdents { + fn visit_tt(&mut self, tt: &mut tokenstream::TokenTree) { + if let tokenstream::TokenTree::Token(token) = tt { + if let token::Interpolated(nt) = &token.kind { + if let token::NtIdent(ident, is_raw) = **nt { + *tt = tokenstream::TokenTree::token( + token::Ident(ident.name, is_raw), ident.span + ); + } + } + } + mut_visit::noop_visit_tt(tt, self) + } + + fn visit_mac(&mut self, mac: &mut ast::Mac) { + mut_visit::noop_visit_mac(mac, self) + } + } + AvoidInterpolatedIdents.visit_tts(&mut input); + (*self)(ecx, span, input) + } +} + +// Use a macro because forwarding to a simple function has type system issues +macro_rules! make_stmts_default { + ($me:expr) => { + $me.make_expr().map(|e| smallvec![ast::Stmt { + id: ast::DUMMY_NODE_ID, + span: e.span, + kind: ast::StmtKind::Expr(e), + }]) + } +} + +/// The result of a macro expansion. The return values of the various +/// methods are spliced into the AST at the callsite of the macro. +pub trait MacResult { + /// Creates an expression. + fn make_expr(self: Box) -> Option> { + None + } + /// Creates zero or more items. + fn make_items(self: Box) -> Option; 1]>> { + None + } + + /// Creates zero or more impl items. + fn make_impl_items(self: Box) -> Option> { + None + } + + /// Creates zero or more trait items. + fn make_trait_items(self: Box) -> Option> { + None + } + + /// Creates zero or more items in an `extern {}` block + fn make_foreign_items(self: Box) -> Option> { None } + + /// Creates a pattern. + fn make_pat(self: Box) -> Option> { + None + } + + /// Creates zero or more statements. + /// + /// By default this attempts to create an expression statement, + /// returning None if that fails. + fn make_stmts(self: Box) -> Option> { + make_stmts_default!(self) + } + + fn make_ty(self: Box) -> Option> { + None + } + + fn make_arms(self: Box) -> Option> { + None + } + + fn make_fields(self: Box) -> Option> { + None + } + + fn make_field_patterns(self: Box) -> Option> { + None + } + + fn make_generic_params(self: Box) -> Option> { + None + } + + fn make_params(self: Box) -> Option> { + None + } + + fn make_struct_fields(self: Box) -> Option> { + None + } + + fn make_variants(self: Box) -> Option> { + None + } +} + +macro_rules! make_MacEager { + ( $( $fld:ident: $t:ty, )* ) => { + /// `MacResult` implementation for the common case where you've already + /// built each form of AST that you might return. + #[derive(Default)] + pub struct MacEager { + $( + pub $fld: Option<$t>, + )* + } + + impl MacEager { + $( + pub fn $fld(v: $t) -> Box { + Box::new(MacEager { + $fld: Some(v), + ..Default::default() + }) + } + )* + } + } +} + +make_MacEager! { + expr: P, + pat: P, + items: SmallVec<[P; 1]>, + impl_items: SmallVec<[ast::ImplItem; 1]>, + trait_items: SmallVec<[ast::TraitItem; 1]>, + foreign_items: SmallVec<[ast::ForeignItem; 1]>, + stmts: SmallVec<[ast::Stmt; 1]>, + ty: P, +} + +impl MacResult for MacEager { + fn make_expr(self: Box) -> Option> { + self.expr + } + + fn make_items(self: Box) -> Option; 1]>> { + self.items + } + + fn make_impl_items(self: Box) -> Option> { + self.impl_items + } + + fn make_trait_items(self: Box) -> Option> { + self.trait_items + } + + fn make_foreign_items(self: Box) -> Option> { + self.foreign_items + } + + fn make_stmts(self: Box) -> Option> { + match self.stmts.as_ref().map_or(0, |s| s.len()) { + 0 => make_stmts_default!(self), + _ => self.stmts, + } + } + + fn make_pat(self: Box) -> Option> { + if let Some(p) = self.pat { + return Some(p); + } + if let Some(e) = self.expr { + if let ast::ExprKind::Lit(_) = e.kind { + return Some(P(ast::Pat { + id: ast::DUMMY_NODE_ID, + span: e.span, + kind: PatKind::Lit(e), + })); + } + } + None + } + + fn make_ty(self: Box) -> Option> { + self.ty + } +} + +/// Fill-in macro expansion result, to allow compilation to continue +/// after hitting errors. +#[derive(Copy, Clone)] +pub struct DummyResult { + is_error: bool, + span: Span, +} + +impl DummyResult { + /// Creates a default MacResult that can be anything. + /// + /// Use this as a return value after hitting any errors and + /// calling `span_err`. + pub fn any(span: Span) -> Box { + Box::new(DummyResult { is_error: true, span }) + } + + /// Same as `any`, but must be a valid fragment, not error. + pub fn any_valid(span: Span) -> Box { + Box::new(DummyResult { is_error: false, span }) + } + + /// A plain dummy expression. + pub fn raw_expr(sp: Span, is_error: bool) -> P { + P(ast::Expr { + id: ast::DUMMY_NODE_ID, + kind: if is_error { ast::ExprKind::Err } else { ast::ExprKind::Tup(Vec::new()) }, + span: sp, + attrs: ThinVec::new(), + }) + } + + /// A plain dummy pattern. + pub fn raw_pat(sp: Span) -> ast::Pat { + ast::Pat { + id: ast::DUMMY_NODE_ID, + kind: PatKind::Wild, + span: sp, + } + } + + /// A plain dummy type. + pub fn raw_ty(sp: Span, is_error: bool) -> P { + P(ast::Ty { + id: ast::DUMMY_NODE_ID, + kind: if is_error { ast::TyKind::Err } else { ast::TyKind::Tup(Vec::new()) }, + span: sp + }) + } +} + +impl MacResult for DummyResult { + fn make_expr(self: Box) -> Option> { + Some(DummyResult::raw_expr(self.span, self.is_error)) + } + + fn make_pat(self: Box) -> Option> { + Some(P(DummyResult::raw_pat(self.span))) + } + + fn make_items(self: Box) -> Option; 1]>> { + Some(SmallVec::new()) + } + + fn make_impl_items(self: Box) -> Option> { + Some(SmallVec::new()) + } + + fn make_trait_items(self: Box) -> Option> { + Some(SmallVec::new()) + } + + fn make_foreign_items(self: Box) -> Option> { + Some(SmallVec::new()) + } + + fn make_stmts(self: Box) -> Option> { + Some(smallvec![ast::Stmt { + id: ast::DUMMY_NODE_ID, + kind: ast::StmtKind::Expr(DummyResult::raw_expr(self.span, self.is_error)), + span: self.span, + }]) + } + + fn make_ty(self: Box) -> Option> { + Some(DummyResult::raw_ty(self.span, self.is_error)) + } + + fn make_arms(self: Box) -> Option> { + Some(SmallVec::new()) + } + + fn make_fields(self: Box) -> Option> { + Some(SmallVec::new()) + } + + fn make_field_patterns(self: Box) -> Option> { + Some(SmallVec::new()) + } + + fn make_generic_params(self: Box) -> Option> { + Some(SmallVec::new()) + } + + fn make_params(self: Box) -> Option> { + Some(SmallVec::new()) + } + + fn make_struct_fields(self: Box) -> Option> { + Some(SmallVec::new()) + } + + fn make_variants(self: Box) -> Option> { + Some(SmallVec::new()) + } +} + +/// A syntax extension kind. +pub enum SyntaxExtensionKind { + /// A token-based function-like macro. + Bang( + /// An expander with signature TokenStream -> TokenStream. + Box, + ), + + /// An AST-based function-like macro. + LegacyBang( + /// An expander with signature TokenStream -> AST. + Box, + ), + + /// A token-based attribute macro. + Attr( + /// An expander with signature (TokenStream, TokenStream) -> TokenStream. + /// The first TokenSteam is the attribute itself, the second is the annotated item. + /// The produced TokenSteam replaces the input TokenSteam. + Box, + ), + + /// An AST-based attribute macro. + LegacyAttr( + /// An expander with signature (AST, AST) -> AST. + /// The first AST fragment is the attribute itself, the second is the annotated item. + /// The produced AST fragment replaces the input AST fragment. + Box, + ), + + /// A trivial attribute "macro" that does nothing, + /// only keeps the attribute and marks it as inert, + /// thus making it ineligible for further expansion. + NonMacroAttr { + /// Suppresses the `unused_attributes` lint for this attribute. + mark_used: bool, + }, + + /// A token-based derive macro. + Derive( + /// An expander with signature TokenStream -> TokenStream (not yet). + /// The produced TokenSteam is appended to the input TokenSteam. + Box, + ), + + /// An AST-based derive macro. + LegacyDerive( + /// An expander with signature AST -> AST. + /// The produced AST fragment is appended to the input AST fragment. + Box, + ), +} + +/// A struct representing a macro definition in "lowered" form ready for expansion. +pub struct SyntaxExtension { + /// A syntax extension kind. + pub kind: SyntaxExtensionKind, + /// Span of the macro definition. + pub span: Span, + /// Whitelist of unstable features that are treated as stable inside this macro. + pub allow_internal_unstable: Option>, + /// Suppresses the `unsafe_code` lint for code produced by this macro. + pub allow_internal_unsafe: bool, + /// Enables the macro helper hack (`ident!(...)` -> `$crate::ident!(...)`) for this macro. + pub local_inner_macros: bool, + /// The macro's stability info. + pub stability: Option, + /// The macro's deprecation info. + pub deprecation: Option, + /// Names of helper attributes registered by this macro. + pub helper_attrs: Vec, + /// Edition of the crate in which this macro is defined. + pub edition: Edition, + /// Built-in macros have a couple of special properties like availability + /// in `#[no_implicit_prelude]` modules, so we have to keep this flag. + pub is_builtin: bool, + /// We have to identify macros providing a `Copy` impl early for compatibility reasons. + pub is_derive_copy: bool, +} + +impl SyntaxExtension { + /// Returns which kind of macro calls this syntax extension. + pub fn macro_kind(&self) -> MacroKind { + match self.kind { + SyntaxExtensionKind::Bang(..) | + SyntaxExtensionKind::LegacyBang(..) => MacroKind::Bang, + SyntaxExtensionKind::Attr(..) | + SyntaxExtensionKind::LegacyAttr(..) | + SyntaxExtensionKind::NonMacroAttr { .. } => MacroKind::Attr, + SyntaxExtensionKind::Derive(..) | + SyntaxExtensionKind::LegacyDerive(..) => MacroKind::Derive, + } + } + + /// Constructs a syntax extension with default properties. + pub fn default(kind: SyntaxExtensionKind, edition: Edition) -> SyntaxExtension { + SyntaxExtension { + span: DUMMY_SP, + allow_internal_unstable: None, + allow_internal_unsafe: false, + local_inner_macros: false, + stability: None, + deprecation: None, + helper_attrs: Vec::new(), + edition, + is_builtin: false, + is_derive_copy: false, + kind, + } + } + + /// Constructs a syntax extension with the given properties + /// and other properties converted from attributes. + pub fn new( + sess: &ParseSess, + kind: SyntaxExtensionKind, + span: Span, + helper_attrs: Vec, + edition: Edition, + name: Name, + attrs: &[ast::Attribute], + ) -> SyntaxExtension { + let allow_internal_unstable = attr::allow_internal_unstable( + &attrs, &sess.span_diagnostic, + ).map(|features| features.collect::>().into()); + + let mut local_inner_macros = false; + if let Some(macro_export) = attr::find_by_name(attrs, sym::macro_export) { + if let Some(l) = macro_export.meta_item_list() { + local_inner_macros = attr::list_contains_name(&l, sym::local_inner_macros); + } + } + + let is_builtin = attr::contains_name(attrs, sym::rustc_builtin_macro); + + SyntaxExtension { + kind, + span, + allow_internal_unstable, + allow_internal_unsafe: attr::contains_name(attrs, sym::allow_internal_unsafe), + local_inner_macros, + stability: attr::find_stability(&sess, attrs, span), + deprecation: attr::find_deprecation(&sess, attrs, span), + helper_attrs, + edition, + is_builtin, + is_derive_copy: is_builtin && name == sym::Copy, + } + } + + pub fn dummy_bang(edition: Edition) -> SyntaxExtension { + fn expander<'cx>(_: &'cx mut ExtCtxt<'_>, span: Span, _: TokenStream) + -> Box { + DummyResult::any(span) + } + SyntaxExtension::default(SyntaxExtensionKind::LegacyBang(Box::new(expander)), edition) + } + + pub fn dummy_derive(edition: Edition) -> SyntaxExtension { + fn expander(_: &mut ExtCtxt<'_>, _: Span, _: &ast::MetaItem, _: Annotatable) + -> Vec { + Vec::new() + } + SyntaxExtension::default(SyntaxExtensionKind::Derive(Box::new(expander)), edition) + } + + pub fn non_macro_attr(mark_used: bool, edition: Edition) -> SyntaxExtension { + SyntaxExtension::default(SyntaxExtensionKind::NonMacroAttr { mark_used }, edition) + } + + pub fn expn_data(&self, parent: ExpnId, call_site: Span, descr: Symbol) -> ExpnData { + ExpnData { + kind: ExpnKind::Macro(self.macro_kind(), descr), + parent, + call_site, + def_site: self.span, + allow_internal_unstable: self.allow_internal_unstable.clone(), + allow_internal_unsafe: self.allow_internal_unsafe, + local_inner_macros: self.local_inner_macros, + edition: self.edition, + } + } +} + +pub type NamedSyntaxExtension = (Name, SyntaxExtension); + +/// Result of resolving a macro invocation. +pub enum InvocationRes { + Single(Lrc), + DeriveContainer(Vec>), +} + +/// Error type that denotes indeterminacy. +pub struct Indeterminate; + +bitflags::bitflags! { + /// Built-in derives that need some extra tracking beyond the usual macro functionality. + #[derive(Default)] + pub struct SpecialDerives: u8 { + const PARTIAL_EQ = 1 << 0; + const EQ = 1 << 1; + const COPY = 1 << 2; + } +} + +pub trait Resolver { + fn next_node_id(&mut self) -> NodeId; + + fn resolve_dollar_crates(&mut self); + fn visit_ast_fragment_with_placeholders(&mut self, expn_id: ExpnId, fragment: &AstFragment, + extra_placeholders: &[NodeId]); + fn register_builtin_macro(&mut self, ident: ast::Ident, ext: SyntaxExtension); + + fn expansion_for_ast_pass( + &mut self, + call_site: Span, + pass: AstPass, + features: &[Symbol], + parent_module_id: Option, + ) -> ExpnId; + + fn resolve_imports(&mut self); + + fn resolve_macro_invocation( + &mut self, invoc: &Invocation, eager_expansion_root: ExpnId, force: bool + ) -> Result; + + fn check_unused_macros(&self); + + fn has_derives(&self, expn_id: ExpnId, derives: SpecialDerives) -> bool; + fn add_derives(&mut self, expn_id: ExpnId, derives: SpecialDerives); +} + +#[derive(Clone)] +pub struct ModuleData { + pub mod_path: Vec, + pub directory: PathBuf, +} + +#[derive(Clone)] +pub struct ExpansionData { + pub id: ExpnId, + pub depth: usize, + pub module: Rc, + pub directory_ownership: DirectoryOwnership, + pub prior_type_ascription: Option<(Span, bool)>, +} + +/// One of these is made during expansion and incrementally updated as we go; +/// when a macro expansion occurs, the resulting nodes have the `backtrace() +/// -> expn_data` of their expansion context stored into their span. +pub struct ExtCtxt<'a> { + pub parse_sess: &'a ParseSess, + pub ecfg: expand::ExpansionConfig<'a>, + pub root_path: PathBuf, + pub resolver: &'a mut dyn Resolver, + pub current_expansion: ExpansionData, + pub expansions: FxHashMap>, +} + +impl<'a> ExtCtxt<'a> { + pub fn new(parse_sess: &'a ParseSess, + ecfg: expand::ExpansionConfig<'a>, + resolver: &'a mut dyn Resolver) + -> ExtCtxt<'a> { + ExtCtxt { + parse_sess, + ecfg, + root_path: PathBuf::new(), + resolver, + current_expansion: ExpansionData { + id: ExpnId::root(), + depth: 0, + module: Rc::new(ModuleData { mod_path: Vec::new(), directory: PathBuf::new() }), + directory_ownership: DirectoryOwnership::Owned { relative: None }, + prior_type_ascription: None, + }, + expansions: FxHashMap::default(), + } + } + + /// Returns a `Folder` for deeply expanding all macros in an AST node. + pub fn expander<'b>(&'b mut self) -> expand::MacroExpander<'b, 'a> { + expand::MacroExpander::new(self, false) + } + + /// Returns a `Folder` that deeply expands all macros and assigns all `NodeId`s in an AST node. + /// Once `NodeId`s are assigned, the node may not be expanded, removed, or otherwise modified. + pub fn monotonic_expander<'b>(&'b mut self) -> expand::MacroExpander<'b, 'a> { + expand::MacroExpander::new(self, true) + } + pub fn new_parser_from_tts(&self, stream: TokenStream) -> parser::Parser<'a> { + parse::stream_to_parser(self.parse_sess, stream, MACRO_ARGUMENTS) + } + pub fn source_map(&self) -> &'a SourceMap { self.parse_sess.source_map() } + pub fn parse_sess(&self) -> &'a ParseSess { self.parse_sess } + pub fn cfg(&self) -> &ast::CrateConfig { &self.parse_sess.config } + pub fn call_site(&self) -> Span { + self.current_expansion.id.expn_data().call_site + } + + /// Equivalent of `Span::def_site` from the proc macro API, + /// except that the location is taken from the span passed as an argument. + pub fn with_def_site_ctxt(&self, span: Span) -> Span { + span.with_def_site_ctxt(self.current_expansion.id) + } + + /// Equivalent of `Span::call_site` from the proc macro API, + /// except that the location is taken from the span passed as an argument. + pub fn with_call_site_ctxt(&self, span: Span) -> Span { + span.with_call_site_ctxt(self.current_expansion.id) + } + + /// Equivalent of `Span::mixed_site` from the proc macro API, + /// except that the location is taken from the span passed as an argument. + pub fn with_mixed_site_ctxt(&self, span: Span) -> Span { + span.with_mixed_site_ctxt(self.current_expansion.id) + } + + /// Returns span for the macro which originally caused the current expansion to happen. + /// + /// Stops backtracing at include! boundary. + pub fn expansion_cause(&self) -> Option { + let mut expn_id = self.current_expansion.id; + let mut last_macro = None; + loop { + let expn_data = expn_id.expn_data(); + // Stop going up the backtrace once include! is encountered + if expn_data.is_root() || expn_data.kind.descr() == sym::include { + break; + } + expn_id = expn_data.call_site.ctxt().outer_expn(); + last_macro = Some(expn_data.call_site); + } + last_macro + } + + pub fn struct_span_warn>(&self, + sp: S, + msg: &str) + -> DiagnosticBuilder<'a> { + self.parse_sess.span_diagnostic.struct_span_warn(sp, msg) + } + pub fn struct_span_err>(&self, + sp: S, + msg: &str) + -> DiagnosticBuilder<'a> { + self.parse_sess.span_diagnostic.struct_span_err(sp, msg) + } + pub fn struct_span_fatal>(&self, + sp: S, + msg: &str) + -> DiagnosticBuilder<'a> { + self.parse_sess.span_diagnostic.struct_span_fatal(sp, msg) + } + + /// Emit `msg` attached to `sp`, and stop compilation immediately. + /// + /// `span_err` should be strongly preferred where-ever possible: + /// this should *only* be used when: + /// + /// - continuing has a high risk of flow-on errors (e.g., errors in + /// declaring a macro would cause all uses of that macro to + /// complain about "undefined macro"), or + /// - there is literally nothing else that can be done (however, + /// in most cases one can construct a dummy expression/item to + /// substitute; we never hit resolve/type-checking so the dummy + /// value doesn't have to match anything) + pub fn span_fatal>(&self, sp: S, msg: &str) -> ! { + self.parse_sess.span_diagnostic.span_fatal(sp, msg).raise(); + } + + /// Emit `msg` attached to `sp`, without immediately stopping + /// compilation. + /// + /// Compilation will be stopped in the near future (at the end of + /// the macro expansion phase). + pub fn span_err>(&self, sp: S, msg: &str) { + self.parse_sess.span_diagnostic.span_err(sp, msg); + } + pub fn span_err_with_code>(&self, sp: S, msg: &str, code: DiagnosticId) { + self.parse_sess.span_diagnostic.span_err_with_code(sp, msg, code); + } + pub fn span_warn>(&self, sp: S, msg: &str) { + self.parse_sess.span_diagnostic.span_warn(sp, msg); + } + pub fn span_bug>(&self, sp: S, msg: &str) -> ! { + self.parse_sess.span_diagnostic.span_bug(sp, msg); + } + pub fn trace_macros_diag(&mut self) { + for (sp, notes) in self.expansions.iter() { + let mut db = self.parse_sess.span_diagnostic.span_note_diag(*sp, "trace_macro"); + for note in notes { + db.note(note); + } + db.emit(); + } + // Fixme: does this result in errors? + self.expansions.clear(); + } + pub fn bug(&self, msg: &str) -> ! { + self.parse_sess.span_diagnostic.bug(msg); + } + pub fn trace_macros(&self) -> bool { + self.ecfg.trace_mac + } + pub fn set_trace_macros(&mut self, x: bool) { + self.ecfg.trace_mac = x + } + pub fn ident_of(&self, st: &str, sp: Span) -> ast::Ident { + ast::Ident::from_str_and_span(st, sp) + } + pub fn std_path(&self, components: &[Symbol]) -> Vec { + let def_site = self.with_def_site_ctxt(DUMMY_SP); + iter::once(Ident::new(kw::DollarCrate, def_site)) + .chain(components.iter().map(|&s| Ident::with_dummy_span(s))) + .collect() + } + pub fn name_of(&self, st: &str) -> ast::Name { + Symbol::intern(st) + } + + pub fn check_unused_macros(&self) { + self.resolver.check_unused_macros(); + } + + /// Resolves a path mentioned inside Rust code. + /// + /// This unifies the logic used for resolving `include_X!`, and `#[doc(include)]` file paths. + /// + /// Returns an absolute path to the file that `path` refers to. + pub fn resolve_path(&self, path: impl Into, span: Span) -> PathBuf { + let path = path.into(); + + // Relative paths are resolved relative to the file in which they are found + // after macro expansion (that is, they are unhygienic). + if !path.is_absolute() { + let callsite = span.source_callsite(); + let mut result = match self.source_map().span_to_unmapped_path(callsite) { + FileName::Real(path) => path, + FileName::DocTest(path, _) => path, + other => panic!("cannot resolve relative path in non-file source `{}`", other), + }; + result.pop(); + result.push(path); + result + } else { + path + } + } +} + +/// Extracts a string literal from the macro expanded version of `expr`, +/// emitting `err_msg` if `expr` is not a string literal. This does not stop +/// compilation on error, merely emits a non-fatal error and returns `None`. +pub fn expr_to_spanned_string<'a>( + cx: &'a mut ExtCtxt<'_>, + expr: P, + err_msg: &str, +) -> Result<(Symbol, ast::StrStyle, Span), Option>> { + // Perform eager expansion on the expression. + // We want to be able to handle e.g., `concat!("foo", "bar")`. + let expr = cx.expander().fully_expand_fragment(AstFragment::Expr(expr)).make_expr(); + + Err(match expr.kind { + ast::ExprKind::Lit(ref l) => match l.kind { + ast::LitKind::Str(s, style) => return Ok((s, style, expr.span)), + ast::LitKind::Err(_) => None, + _ => Some(cx.struct_span_err(l.span, err_msg)) + }, + ast::ExprKind::Err => None, + _ => Some(cx.struct_span_err(expr.span, err_msg)) + }) +} + +pub fn expr_to_string(cx: &mut ExtCtxt<'_>, expr: P, err_msg: &str) + -> Option<(Symbol, ast::StrStyle)> { + expr_to_spanned_string(cx, expr, err_msg) + .map_err(|err| err.map(|mut err| err.emit())) + .ok() + .map(|(symbol, style, _)| (symbol, style)) +} + +/// Non-fatally assert that `tts` is empty. Note that this function +/// returns even when `tts` is non-empty, macros that *need* to stop +/// compilation should call +/// `cx.parse_sess.span_diagnostic.abort_if_errors()` (this should be +/// done as rarely as possible). +pub fn check_zero_tts(cx: &ExtCtxt<'_>, + sp: Span, + tts: TokenStream, + name: &str) { + if !tts.is_empty() { + cx.span_err(sp, &format!("{} takes no arguments", name)); + } +} + +/// Interpreting `tts` as a comma-separated sequence of expressions, +/// expect exactly one string literal, or emit an error and return `None`. +pub fn get_single_str_from_tts(cx: &mut ExtCtxt<'_>, + sp: Span, + tts: TokenStream, + name: &str) + -> Option { + let mut p = cx.new_parser_from_tts(tts); + if p.token == token::Eof { + cx.span_err(sp, &format!("{} takes 1 argument", name)); + return None + } + let ret = panictry!(p.parse_expr()); + let _ = p.eat(&token::Comma); + + if p.token != token::Eof { + cx.span_err(sp, &format!("{} takes 1 argument", name)); + } + expr_to_string(cx, ret, "argument must be a string literal").map(|(s, _)| { + s.to_string() + }) +} + +/// Extracts comma-separated expressions from `tts`. If there is a +/// parsing error, emit a non-fatal error and return `None`. +pub fn get_exprs_from_tts(cx: &mut ExtCtxt<'_>, + sp: Span, + tts: TokenStream) -> Option>> { + let mut p = cx.new_parser_from_tts(tts); + let mut es = Vec::new(); + while p.token != token::Eof { + let expr = panictry!(p.parse_expr()); + + // Perform eager expansion on the expression. + // We want to be able to handle e.g., `concat!("foo", "bar")`. + let expr = cx.expander().fully_expand_fragment(AstFragment::Expr(expr)).make_expr(); + + es.push(expr); + if p.eat(&token::Comma) { + continue; + } + if p.token != token::Eof { + cx.span_err(sp, "expected token: `,`"); + return None; + } + } + Some(es) +} diff --git a/src/libsyntax_expand/build.rs b/src/libsyntax_expand/build.rs new file mode 100644 index 00000000000..105ffe3ee8a --- /dev/null +++ b/src/libsyntax_expand/build.rs @@ -0,0 +1,640 @@ +use crate::base::ExtCtxt; + +use syntax::ast::{self, Ident, Expr, BlockCheckMode, UnOp, PatKind}; +use syntax::attr; +use syntax::source_map::{respan, Spanned}; +use syntax::ptr::P; +use syntax::symbol::{kw, sym, Symbol}; +use syntax::ThinVec; + +use syntax_pos::{Pos, Span}; + +impl<'a> ExtCtxt<'a> { + pub fn path(&self, span: Span, strs: Vec ) -> ast::Path { + self.path_all(span, false, strs, vec![]) + } + pub fn path_ident(&self, span: Span, id: ast::Ident) -> ast::Path { + self.path(span, vec![id]) + } + pub fn path_global(&self, span: Span, strs: Vec ) -> ast::Path { + self.path_all(span, true, strs, vec![]) + } + pub fn path_all(&self, + span: Span, + global: bool, + mut idents: Vec , + args: Vec) + -> ast::Path { + assert!(!idents.is_empty()); + let add_root = global && !idents[0].is_path_segment_keyword(); + let mut segments = Vec::with_capacity(idents.len() + add_root as usize); + if add_root { + segments.push(ast::PathSegment::path_root(span)); + } + let last_ident = idents.pop().unwrap(); + segments.extend(idents.into_iter().map(|ident| { + ast::PathSegment::from_ident(ident.with_span_pos(span)) + })); + let args = if !args.is_empty() { + ast::AngleBracketedArgs { args, constraints: Vec::new(), span }.into() + } else { + None + }; + segments.push(ast::PathSegment { + ident: last_ident.with_span_pos(span), + id: ast::DUMMY_NODE_ID, + args, + }); + ast::Path { span, segments } + } + + pub fn ty_mt(&self, ty: P, mutbl: ast::Mutability) -> ast::MutTy { + ast::MutTy { + ty, + mutbl, + } + } + + pub fn ty(&self, span: Span, kind: ast::TyKind) -> P { + P(ast::Ty { + id: ast::DUMMY_NODE_ID, + span, + kind, + }) + } + + pub fn ty_path(&self, path: ast::Path) -> P { + self.ty(path.span, ast::TyKind::Path(None, path)) + } + + // Might need to take bounds as an argument in the future, if you ever want + // to generate a bounded existential trait type. + pub fn ty_ident(&self, span: Span, ident: ast::Ident) + -> P { + self.ty_path(self.path_ident(span, ident)) + } + + pub fn anon_const(&self, span: Span, kind: ast::ExprKind) -> ast::AnonConst { + ast::AnonConst { + id: ast::DUMMY_NODE_ID, + value: P(ast::Expr { + id: ast::DUMMY_NODE_ID, + kind, + span, + attrs: ThinVec::new(), + }) + } + } + + pub fn const_ident(&self, span: Span, ident: ast::Ident) -> ast::AnonConst { + self.anon_const(span, ast::ExprKind::Path(None, self.path_ident(span, ident))) + } + + pub fn ty_rptr(&self, + span: Span, + ty: P, + lifetime: Option, + mutbl: ast::Mutability) + -> P { + self.ty(span, + ast::TyKind::Rptr(lifetime, self.ty_mt(ty, mutbl))) + } + + pub fn ty_ptr(&self, + span: Span, + ty: P, + mutbl: ast::Mutability) + -> P { + self.ty(span, + ast::TyKind::Ptr(self.ty_mt(ty, mutbl))) + } + + pub fn typaram(&self, + span: Span, + ident: ast::Ident, + attrs: Vec, + bounds: ast::GenericBounds, + default: Option>) -> ast::GenericParam { + ast::GenericParam { + ident: ident.with_span_pos(span), + id: ast::DUMMY_NODE_ID, + attrs: attrs.into(), + bounds, + kind: ast::GenericParamKind::Type { + default, + }, + is_placeholder: false + } + } + + pub fn trait_ref(&self, path: ast::Path) -> ast::TraitRef { + ast::TraitRef { + path, + ref_id: ast::DUMMY_NODE_ID, + } + } + + pub fn poly_trait_ref(&self, span: Span, path: ast::Path) -> ast::PolyTraitRef { + ast::PolyTraitRef { + bound_generic_params: Vec::new(), + trait_ref: self.trait_ref(path), + span, + } + } + + pub fn trait_bound(&self, path: ast::Path) -> ast::GenericBound { + ast::GenericBound::Trait(self.poly_trait_ref(path.span, path), + ast::TraitBoundModifier::None) + } + + pub fn lifetime(&self, span: Span, ident: ast::Ident) -> ast::Lifetime { + ast::Lifetime { id: ast::DUMMY_NODE_ID, ident: ident.with_span_pos(span) } + } + + pub fn lifetime_def(&self, + span: Span, + ident: ast::Ident, + attrs: Vec, + bounds: ast::GenericBounds) + -> ast::GenericParam { + let lifetime = self.lifetime(span, ident); + ast::GenericParam { + ident: lifetime.ident, + id: lifetime.id, + attrs: attrs.into(), + bounds, + kind: ast::GenericParamKind::Lifetime, + is_placeholder: false + } + } + + pub fn stmt_expr(&self, expr: P) -> ast::Stmt { + ast::Stmt { + id: ast::DUMMY_NODE_ID, + span: expr.span, + kind: ast::StmtKind::Expr(expr), + } + } + + pub fn stmt_let(&self, sp: Span, mutbl: bool, ident: ast::Ident, + ex: P) -> ast::Stmt { + let pat = if mutbl { + let binding_mode = ast::BindingMode::ByValue(ast::Mutability::Mutable); + self.pat_ident_binding_mode(sp, ident, binding_mode) + } else { + self.pat_ident(sp, ident) + }; + let local = P(ast::Local { + pat, + ty: None, + init: Some(ex), + id: ast::DUMMY_NODE_ID, + span: sp, + attrs: ThinVec::new(), + }); + ast::Stmt { + id: ast::DUMMY_NODE_ID, + kind: ast::StmtKind::Local(local), + span: sp, + } + } + + // Generates `let _: Type;`, which is usually used for type assertions. + pub fn stmt_let_type_only(&self, span: Span, ty: P) -> ast::Stmt { + let local = P(ast::Local { + pat: self.pat_wild(span), + ty: Some(ty), + init: None, + id: ast::DUMMY_NODE_ID, + span, + attrs: ThinVec::new(), + }); + ast::Stmt { + id: ast::DUMMY_NODE_ID, + kind: ast::StmtKind::Local(local), + span, + } + } + + pub fn stmt_item(&self, sp: Span, item: P) -> ast::Stmt { + ast::Stmt { + id: ast::DUMMY_NODE_ID, + kind: ast::StmtKind::Item(item), + span: sp, + } + } + + pub fn block_expr(&self, expr: P) -> P { + self.block(expr.span, vec![ast::Stmt { + id: ast::DUMMY_NODE_ID, + span: expr.span, + kind: ast::StmtKind::Expr(expr), + }]) + } + pub fn block(&self, span: Span, stmts: Vec) -> P { + P(ast::Block { + stmts, + id: ast::DUMMY_NODE_ID, + rules: BlockCheckMode::Default, + span, + }) + } + + pub fn expr(&self, span: Span, kind: ast::ExprKind) -> P { + P(ast::Expr { + id: ast::DUMMY_NODE_ID, + kind, + span, + attrs: ThinVec::new(), + }) + } + + pub fn expr_path(&self, path: ast::Path) -> P { + self.expr(path.span, ast::ExprKind::Path(None, path)) + } + + pub fn expr_ident(&self, span: Span, id: ast::Ident) -> P { + self.expr_path(self.path_ident(span, id)) + } + pub fn expr_self(&self, span: Span) -> P { + self.expr_ident(span, Ident::with_dummy_span(kw::SelfLower)) + } + + pub fn expr_binary(&self, sp: Span, op: ast::BinOpKind, + lhs: P, rhs: P) -> P { + self.expr(sp, ast::ExprKind::Binary(Spanned { node: op, span: sp }, lhs, rhs)) + } + + pub fn expr_deref(&self, sp: Span, e: P) -> P { + self.expr(sp, ast::ExprKind::Unary(UnOp::Deref, e)) + } + + pub fn expr_addr_of(&self, sp: Span, e: P) -> P { + self.expr(sp, ast::ExprKind::AddrOf(ast::Mutability::Immutable, e)) + } + + pub fn expr_call( + &self, span: Span, expr: P, args: Vec>, + ) -> P { + self.expr(span, ast::ExprKind::Call(expr, args)) + } + pub fn expr_call_ident(&self, span: Span, id: ast::Ident, + args: Vec>) -> P { + self.expr(span, ast::ExprKind::Call(self.expr_ident(span, id), args)) + } + pub fn expr_call_global(&self, sp: Span, fn_path: Vec , + args: Vec> ) -> P { + let pathexpr = self.expr_path(self.path_global(sp, fn_path)); + self.expr_call(sp, pathexpr, args) + } + pub fn expr_method_call(&self, span: Span, + expr: P, + ident: ast::Ident, + mut args: Vec> ) -> P { + args.insert(0, expr); + let segment = ast::PathSegment::from_ident(ident.with_span_pos(span)); + self.expr(span, ast::ExprKind::MethodCall(segment, args)) + } + pub fn expr_block(&self, b: P) -> P { + self.expr(b.span, ast::ExprKind::Block(b, None)) + } + pub fn field_imm(&self, span: Span, ident: Ident, e: P) -> ast::Field { + ast::Field { + ident: ident.with_span_pos(span), + expr: e, + span, + is_shorthand: false, + attrs: ThinVec::new(), + id: ast::DUMMY_NODE_ID, + is_placeholder: false, + } + } + pub fn expr_struct( + &self, span: Span, path: ast::Path, fields: Vec + ) -> P { + self.expr(span, ast::ExprKind::Struct(path, fields, None)) + } + pub fn expr_struct_ident(&self, span: Span, + id: ast::Ident, fields: Vec) -> P { + self.expr_struct(span, self.path_ident(span, id), fields) + } + + pub fn expr_lit(&self, span: Span, lit_kind: ast::LitKind) -> P { + let lit = ast::Lit::from_lit_kind(lit_kind, span); + self.expr(span, ast::ExprKind::Lit(lit)) + } + pub fn expr_usize(&self, span: Span, i: usize) -> P { + self.expr_lit(span, ast::LitKind::Int(i as u128, + ast::LitIntType::Unsigned(ast::UintTy::Usize))) + } + pub fn expr_u32(&self, sp: Span, u: u32) -> P { + self.expr_lit(sp, ast::LitKind::Int(u as u128, + ast::LitIntType::Unsigned(ast::UintTy::U32))) + } + pub fn expr_bool(&self, sp: Span, value: bool) -> P { + self.expr_lit(sp, ast::LitKind::Bool(value)) + } + + pub fn expr_vec(&self, sp: Span, exprs: Vec>) -> P { + self.expr(sp, ast::ExprKind::Array(exprs)) + } + pub fn expr_vec_slice(&self, sp: Span, exprs: Vec>) -> P { + self.expr_addr_of(sp, self.expr_vec(sp, exprs)) + } + pub fn expr_str(&self, sp: Span, s: Symbol) -> P { + self.expr_lit(sp, ast::LitKind::Str(s, ast::StrStyle::Cooked)) + } + + pub fn expr_cast(&self, sp: Span, expr: P, ty: P) -> P { + self.expr(sp, ast::ExprKind::Cast(expr, ty)) + } + + pub fn expr_some(&self, sp: Span, expr: P) -> P { + let some = self.std_path(&[sym::option, sym::Option, sym::Some]); + self.expr_call_global(sp, some, vec![expr]) + } + + pub fn expr_tuple(&self, sp: Span, exprs: Vec>) -> P { + self.expr(sp, ast::ExprKind::Tup(exprs)) + } + + pub fn expr_fail(&self, span: Span, msg: Symbol) -> P { + let loc = self.source_map().lookup_char_pos(span.lo()); + let expr_file = self.expr_str(span, Symbol::intern(&loc.file.name.to_string())); + let expr_line = self.expr_u32(span, loc.line as u32); + let expr_col = self.expr_u32(span, loc.col.to_usize() as u32 + 1); + let expr_loc_tuple = self.expr_tuple(span, vec![expr_file, expr_line, expr_col]); + let expr_loc_ptr = self.expr_addr_of(span, expr_loc_tuple); + self.expr_call_global( + span, + [sym::std, sym::rt, sym::begin_panic].iter().map(|s| Ident::new(*s, span)).collect(), + vec![ + self.expr_str(span, msg), + expr_loc_ptr]) + } + + pub fn expr_unreachable(&self, span: Span) -> P { + self.expr_fail(span, Symbol::intern("internal error: entered unreachable code")) + } + + pub fn expr_ok(&self, sp: Span, expr: P) -> P { + let ok = self.std_path(&[sym::result, sym::Result, sym::Ok]); + self.expr_call_global(sp, ok, vec![expr]) + } + + pub fn expr_try(&self, sp: Span, head: P) -> P { + let ok = self.std_path(&[sym::result, sym::Result, sym::Ok]); + let ok_path = self.path_global(sp, ok); + let err = self.std_path(&[sym::result, sym::Result, sym::Err]); + let err_path = self.path_global(sp, err); + + let binding_variable = self.ident_of("__try_var", sp); + let binding_pat = self.pat_ident(sp, binding_variable); + let binding_expr = self.expr_ident(sp, binding_variable); + + // `Ok(__try_var)` pattern + let ok_pat = self.pat_tuple_struct(sp, ok_path, vec![binding_pat.clone()]); + + // `Err(__try_var)` (pattern and expression respectively) + let err_pat = self.pat_tuple_struct(sp, err_path.clone(), vec![binding_pat]); + let err_inner_expr = self.expr_call(sp, self.expr_path(err_path), + vec![binding_expr.clone()]); + // `return Err(__try_var)` + let err_expr = self.expr(sp, ast::ExprKind::Ret(Some(err_inner_expr))); + + // `Ok(__try_var) => __try_var` + let ok_arm = self.arm(sp, ok_pat, binding_expr); + // `Err(__try_var) => return Err(__try_var)` + let err_arm = self.arm(sp, err_pat, err_expr); + + // `match head { Ok() => ..., Err() => ... }` + self.expr_match(sp, head, vec![ok_arm, err_arm]) + } + + + pub fn pat(&self, span: Span, kind: PatKind) -> P { + P(ast::Pat { id: ast::DUMMY_NODE_ID, kind, span }) + } + pub fn pat_wild(&self, span: Span) -> P { + self.pat(span, PatKind::Wild) + } + pub fn pat_lit(&self, span: Span, expr: P) -> P { + self.pat(span, PatKind::Lit(expr)) + } + pub fn pat_ident(&self, span: Span, ident: ast::Ident) -> P { + let binding_mode = ast::BindingMode::ByValue(ast::Mutability::Immutable); + self.pat_ident_binding_mode(span, ident, binding_mode) + } + + pub fn pat_ident_binding_mode(&self, + span: Span, + ident: ast::Ident, + bm: ast::BindingMode) -> P { + let pat = PatKind::Ident(bm, ident.with_span_pos(span), None); + self.pat(span, pat) + } + pub fn pat_path(&self, span: Span, path: ast::Path) -> P { + self.pat(span, PatKind::Path(None, path)) + } + pub fn pat_tuple_struct(&self, span: Span, path: ast::Path, + subpats: Vec>) -> P { + self.pat(span, PatKind::TupleStruct(path, subpats)) + } + pub fn pat_struct(&self, span: Span, path: ast::Path, + field_pats: Vec) -> P { + self.pat(span, PatKind::Struct(path, field_pats, false)) + } + pub fn pat_tuple(&self, span: Span, pats: Vec>) -> P { + self.pat(span, PatKind::Tuple(pats)) + } + + pub fn pat_some(&self, span: Span, pat: P) -> P { + let some = self.std_path(&[sym::option, sym::Option, sym::Some]); + let path = self.path_global(span, some); + self.pat_tuple_struct(span, path, vec![pat]) + } + + pub fn pat_none(&self, span: Span) -> P { + let some = self.std_path(&[sym::option, sym::Option, sym::None]); + let path = self.path_global(span, some); + self.pat_path(span, path) + } + + pub fn pat_ok(&self, span: Span, pat: P) -> P { + let some = self.std_path(&[sym::result, sym::Result, sym::Ok]); + let path = self.path_global(span, some); + self.pat_tuple_struct(span, path, vec![pat]) + } + + pub fn pat_err(&self, span: Span, pat: P) -> P { + let some = self.std_path(&[sym::result, sym::Result, sym::Err]); + let path = self.path_global(span, some); + self.pat_tuple_struct(span, path, vec![pat]) + } + + pub fn arm(&self, span: Span, pat: P, expr: P) -> ast::Arm { + ast::Arm { + attrs: vec![], + pat, + guard: None, + body: expr, + span, + id: ast::DUMMY_NODE_ID, + is_placeholder: false, + } + } + + pub fn arm_unreachable(&self, span: Span) -> ast::Arm { + self.arm(span, self.pat_wild(span), self.expr_unreachable(span)) + } + + pub fn expr_match(&self, span: Span, arg: P, arms: Vec) -> P { + self.expr(span, ast::ExprKind::Match(arg, arms)) + } + + pub fn expr_if(&self, span: Span, cond: P, + then: P, els: Option>) -> P { + let els = els.map(|x| self.expr_block(self.block_expr(x))); + self.expr(span, ast::ExprKind::If(cond, self.block_expr(then), els)) + } + + pub fn lambda_fn_decl(&self, + span: Span, + fn_decl: P, + body: P, + fn_decl_span: Span) // span of the `|...|` part + -> P { + self.expr(span, ast::ExprKind::Closure(ast::CaptureBy::Ref, + ast::IsAsync::NotAsync, + ast::Movability::Movable, + fn_decl, + body, + fn_decl_span)) + } + + pub fn lambda(&self, + span: Span, + ids: Vec, + body: P) + -> P { + let fn_decl = self.fn_decl( + ids.iter().map(|id| self.param(span, *id, self.ty(span, ast::TyKind::Infer))).collect(), + ast::FunctionRetTy::Default(span)); + + // FIXME -- We are using `span` as the span of the `|...|` + // part of the lambda, but it probably (maybe?) corresponds to + // the entire lambda body. Probably we should extend the API + // here, but that's not entirely clear. + self.expr(span, ast::ExprKind::Closure(ast::CaptureBy::Ref, + ast::IsAsync::NotAsync, + ast::Movability::Movable, + fn_decl, + body, + span)) + } + + pub fn lambda0(&self, span: Span, body: P) -> P { + self.lambda(span, Vec::new(), body) + } + + pub fn lambda1(&self, span: Span, body: P, ident: ast::Ident) -> P { + self.lambda(span, vec![ident], body) + } + + pub fn lambda_stmts_1(&self, span: Span, stmts: Vec, + ident: ast::Ident) -> P { + self.lambda1(span, self.expr_block(self.block(span, stmts)), ident) + } + + pub fn param(&self, span: Span, ident: ast::Ident, ty: P) -> ast::Param { + let arg_pat = self.pat_ident(span, ident); + ast::Param { + attrs: ThinVec::default(), + id: ast::DUMMY_NODE_ID, + pat: arg_pat, + span, + ty, + is_placeholder: false, + } + } + + // FIXME: unused `self` + pub fn fn_decl(&self, inputs: Vec, output: ast::FunctionRetTy) -> P { + P(ast::FnDecl { + inputs, + output, + }) + } + + pub fn item(&self, span: Span, name: Ident, + attrs: Vec, kind: ast::ItemKind) -> P { + // FIXME: Would be nice if our generated code didn't violate + // Rust coding conventions + P(ast::Item { + ident: name, + attrs, + id: ast::DUMMY_NODE_ID, + kind, + vis: respan(span.shrink_to_lo(), ast::VisibilityKind::Inherited), + span, + tokens: None, + }) + } + + pub fn variant(&self, span: Span, ident: Ident, tys: Vec> ) -> ast::Variant { + let fields: Vec<_> = tys.into_iter().map(|ty| { + ast::StructField { + span: ty.span, + ty, + ident: None, + vis: respan(span.shrink_to_lo(), ast::VisibilityKind::Inherited), + attrs: Vec::new(), + id: ast::DUMMY_NODE_ID, + is_placeholder: false, + } + }).collect(); + + let vdata = if fields.is_empty() { + ast::VariantData::Unit(ast::DUMMY_NODE_ID) + } else { + ast::VariantData::Tuple(fields, ast::DUMMY_NODE_ID) + }; + + ast::Variant { + attrs: Vec::new(), + data: vdata, + disr_expr: None, + id: ast::DUMMY_NODE_ID, + ident, + span, + is_placeholder: false, + } + } + + pub fn item_static(&self, + span: Span, + name: Ident, + ty: P, + mutbl: ast::Mutability, + expr: P) + -> P { + self.item(span, name, Vec::new(), ast::ItemKind::Static(ty, mutbl, expr)) + } + + pub fn item_const(&self, + span: Span, + name: Ident, + ty: P, + expr: P) + -> P { + self.item(span, name, Vec::new(), ast::ItemKind::Const(ty, expr)) + } + + pub fn attribute(&self, mi: ast::MetaItem) -> ast::Attribute { + attr::mk_attr_outer(mi) + } + + pub fn meta_word(&self, sp: Span, w: ast::Name) -> ast::MetaItem { + attr::mk_word_item(Ident::new(w, sp)) + } +} diff --git a/src/libsyntax_expand/expand.rs b/src/libsyntax_expand/expand.rs new file mode 100644 index 00000000000..47b4bca314a --- /dev/null +++ b/src/libsyntax_expand/expand.rs @@ -0,0 +1,1551 @@ +use crate::base::*; +use crate::proc_macro::{collect_derives, MarkAttrs}; +use crate::hygiene::{ExpnId, SyntaxContext, ExpnData, ExpnKind}; +use crate::mbe::macro_rules::annotate_err_with_kind; +use crate::placeholders::{placeholder, PlaceholderExpander}; + +use syntax::ast::{self, AttrItem, Block, Ident, LitKind, NodeId, PatKind, Path}; +use syntax::ast::{MacStmtStyle, StmtKind, ItemKind}; +use syntax::attr::{self, HasAttrs}; +use syntax::source_map::respan; +use syntax::configure; +use syntax::config::StripUnconfigured; +use syntax::feature_gate::{self, Features, GateIssue, is_builtin_attr, emit_feature_err}; +use syntax::mut_visit::*; +use syntax::parse::{DirectoryOwnership, PResult}; +use syntax::parse::token; +use syntax::parse::parser::Parser; +use syntax::print::pprust; +use syntax::ptr::P; +use syntax::symbol::{sym, Symbol}; +use syntax::tokenstream::{TokenStream, TokenTree}; +use syntax::visit::Visitor; +use syntax::util::map_in_place::MapInPlace; + +use errors::{Applicability, FatalError}; +use smallvec::{smallvec, SmallVec}; +use syntax_pos::{Span, DUMMY_SP, FileName}; + +use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::sync::Lrc; +use std::io::ErrorKind; +use std::{iter, mem, slice}; +use std::ops::DerefMut; +use std::rc::Rc; +use std::path::PathBuf; + +macro_rules! ast_fragments { + ( + $($Kind:ident($AstTy:ty) { + $kind_name:expr; + $(one fn $mut_visit_ast:ident; fn $visit_ast:ident;)? + $(many fn $flat_map_ast_elt:ident; fn $visit_ast_elt:ident;)? + fn $make_ast:ident; + })* + ) => { + /// A fragment of AST that can be produced by a single macro expansion. + /// Can also serve as an input and intermediate result for macro expansion operations. + pub enum AstFragment { + OptExpr(Option>), + $($Kind($AstTy),)* + } + + /// "Discriminant" of an AST fragment. + #[derive(Copy, Clone, PartialEq, Eq)] + pub enum AstFragmentKind { + OptExpr, + $($Kind,)* + } + + impl AstFragmentKind { + pub fn name(self) -> &'static str { + match self { + AstFragmentKind::OptExpr => "expression", + $(AstFragmentKind::$Kind => $kind_name,)* + } + } + + fn make_from<'a>(self, result: Box) -> Option { + match self { + AstFragmentKind::OptExpr => + result.make_expr().map(Some).map(AstFragment::OptExpr), + $(AstFragmentKind::$Kind => result.$make_ast().map(AstFragment::$Kind),)* + } + } + } + + impl AstFragment { + pub fn make_opt_expr(self) -> Option> { + match self { + AstFragment::OptExpr(expr) => expr, + _ => panic!("AstFragment::make_* called on the wrong kind of fragment"), + } + } + + $(pub fn $make_ast(self) -> $AstTy { + match self { + AstFragment::$Kind(ast) => ast, + _ => panic!("AstFragment::make_* called on the wrong kind of fragment"), + } + })* + + pub fn mut_visit_with(&mut self, vis: &mut F) { + match self { + AstFragment::OptExpr(opt_expr) => { + visit_clobber(opt_expr, |opt_expr| { + if let Some(expr) = opt_expr { + vis.filter_map_expr(expr) + } else { + None + } + }); + } + $($(AstFragment::$Kind(ast) => vis.$mut_visit_ast(ast),)?)* + $($(AstFragment::$Kind(ast) => + ast.flat_map_in_place(|ast| vis.$flat_map_ast_elt(ast)),)?)* + } + } + + pub fn visit_with<'a, V: Visitor<'a>>(&'a self, visitor: &mut V) { + match *self { + AstFragment::OptExpr(Some(ref expr)) => visitor.visit_expr(expr), + AstFragment::OptExpr(None) => {} + $($(AstFragment::$Kind(ref ast) => visitor.$visit_ast(ast),)?)* + $($(AstFragment::$Kind(ref ast) => for ast_elt in &ast[..] { + visitor.$visit_ast_elt(ast_elt); + })?)* + } + } + } + + impl<'a> MacResult for crate::mbe::macro_rules::ParserAnyMacro<'a> { + $(fn $make_ast(self: Box>) + -> Option<$AstTy> { + Some(self.make(AstFragmentKind::$Kind).$make_ast()) + })* + } + } +} + +ast_fragments! { + Expr(P) { "expression"; one fn visit_expr; fn visit_expr; fn make_expr; } + Pat(P) { "pattern"; one fn visit_pat; fn visit_pat; fn make_pat; } + Ty(P) { "type"; one fn visit_ty; fn visit_ty; fn make_ty; } + Stmts(SmallVec<[ast::Stmt; 1]>) { + "statement"; many fn flat_map_stmt; fn visit_stmt; fn make_stmts; + } + Items(SmallVec<[P; 1]>) { + "item"; many fn flat_map_item; fn visit_item; fn make_items; + } + TraitItems(SmallVec<[ast::TraitItem; 1]>) { + "trait item"; many fn flat_map_trait_item; fn visit_trait_item; fn make_trait_items; + } + ImplItems(SmallVec<[ast::ImplItem; 1]>) { + "impl item"; many fn flat_map_impl_item; fn visit_impl_item; fn make_impl_items; + } + ForeignItems(SmallVec<[ast::ForeignItem; 1]>) { + "foreign item"; + many fn flat_map_foreign_item; + fn visit_foreign_item; + fn make_foreign_items; + } + Arms(SmallVec<[ast::Arm; 1]>) { + "match arm"; many fn flat_map_arm; fn visit_arm; fn make_arms; + } + Fields(SmallVec<[ast::Field; 1]>) { + "field expression"; many fn flat_map_field; fn visit_field; fn make_fields; + } + FieldPats(SmallVec<[ast::FieldPat; 1]>) { + "field pattern"; + many fn flat_map_field_pattern; + fn visit_field_pattern; + fn make_field_patterns; + } + GenericParams(SmallVec<[ast::GenericParam; 1]>) { + "generic parameter"; + many fn flat_map_generic_param; + fn visit_generic_param; + fn make_generic_params; + } + Params(SmallVec<[ast::Param; 1]>) { + "function parameter"; many fn flat_map_param; fn visit_param; fn make_params; + } + StructFields(SmallVec<[ast::StructField; 1]>) { + "field"; + many fn flat_map_struct_field; + fn visit_struct_field; + fn make_struct_fields; + } + Variants(SmallVec<[ast::Variant; 1]>) { + "variant"; many fn flat_map_variant; fn visit_variant; fn make_variants; + } +} + +impl AstFragmentKind { + fn dummy(self, span: Span) -> AstFragment { + self.make_from(DummyResult::any(span)).expect("couldn't create a dummy AST fragment") + } + + fn expect_from_annotatables>(self, items: I) + -> AstFragment { + let mut items = items.into_iter(); + match self { + AstFragmentKind::Arms => + AstFragment::Arms(items.map(Annotatable::expect_arm).collect()), + AstFragmentKind::Fields => + AstFragment::Fields(items.map(Annotatable::expect_field).collect()), + AstFragmentKind::FieldPats => + AstFragment::FieldPats(items.map(Annotatable::expect_field_pattern).collect()), + AstFragmentKind::GenericParams => + AstFragment::GenericParams(items.map(Annotatable::expect_generic_param).collect()), + AstFragmentKind::Params => + AstFragment::Params(items.map(Annotatable::expect_param).collect()), + AstFragmentKind::StructFields => AstFragment::StructFields( + items.map(Annotatable::expect_struct_field).collect() + ), + AstFragmentKind::Variants => + AstFragment::Variants(items.map(Annotatable::expect_variant).collect()), + AstFragmentKind::Items => + AstFragment::Items(items.map(Annotatable::expect_item).collect()), + AstFragmentKind::ImplItems => + AstFragment::ImplItems(items.map(Annotatable::expect_impl_item).collect()), + AstFragmentKind::TraitItems => + AstFragment::TraitItems(items.map(Annotatable::expect_trait_item).collect()), + AstFragmentKind::ForeignItems => + AstFragment::ForeignItems(items.map(Annotatable::expect_foreign_item).collect()), + AstFragmentKind::Stmts => + AstFragment::Stmts(items.map(Annotatable::expect_stmt).collect()), + AstFragmentKind::Expr => AstFragment::Expr( + items.next().expect("expected exactly one expression").expect_expr() + ), + AstFragmentKind::OptExpr => + AstFragment::OptExpr(items.next().map(Annotatable::expect_expr)), + AstFragmentKind::Pat | AstFragmentKind::Ty => + panic!("patterns and types aren't annotatable"), + } + } +} + +pub struct Invocation { + pub kind: InvocationKind, + pub fragment_kind: AstFragmentKind, + pub expansion_data: ExpansionData, +} + +pub enum InvocationKind { + Bang { + mac: ast::Mac, + span: Span, + }, + Attr { + attr: ast::Attribute, + item: Annotatable, + // Required for resolving derive helper attributes. + derives: Vec, + // We temporarily report errors for attribute macros placed after derives + after_derive: bool, + }, + Derive { + path: Path, + item: Annotatable, + }, + /// "Invocation" that contains all derives from an item, + /// broken into multiple `Derive` invocations when expanded. + /// FIXME: Find a way to remove it. + DeriveContainer { + derives: Vec, + item: Annotatable, + }, +} + +impl Invocation { + pub fn span(&self) -> Span { + match &self.kind { + InvocationKind::Bang { span, .. } => *span, + InvocationKind::Attr { attr, .. } => attr.span, + InvocationKind::Derive { path, .. } => path.span, + InvocationKind::DeriveContainer { item, .. } => item.span(), + } + } +} + +pub struct MacroExpander<'a, 'b> { + pub cx: &'a mut ExtCtxt<'b>, + monotonic: bool, // cf. `cx.monotonic_expander()` +} + +impl<'a, 'b> MacroExpander<'a, 'b> { + pub fn new(cx: &'a mut ExtCtxt<'b>, monotonic: bool) -> Self { + MacroExpander { cx, monotonic } + } + + pub fn expand_crate(&mut self, mut krate: ast::Crate) -> ast::Crate { + let mut module = ModuleData { + mod_path: vec![Ident::from_str(&self.cx.ecfg.crate_name)], + directory: match self.cx.source_map().span_to_unmapped_path(krate.span) { + FileName::Real(path) => path, + other => PathBuf::from(other.to_string()), + }, + }; + module.directory.pop(); + self.cx.root_path = module.directory.clone(); + self.cx.current_expansion.module = Rc::new(module); + + let orig_mod_span = krate.module.inner; + + let krate_item = AstFragment::Items(smallvec![P(ast::Item { + attrs: krate.attrs, + span: krate.span, + kind: ast::ItemKind::Mod(krate.module), + ident: Ident::invalid(), + id: ast::DUMMY_NODE_ID, + vis: respan(krate.span.shrink_to_lo(), ast::VisibilityKind::Public), + tokens: None, + })]); + + match self.fully_expand_fragment(krate_item).make_items().pop().map(P::into_inner) { + Some(ast::Item { attrs, kind: ast::ItemKind::Mod(module), .. }) => { + krate.attrs = attrs; + krate.module = module; + }, + None => { + // Resolution failed so we return an empty expansion + krate.attrs = vec![]; + krate.module = ast::Mod { + inner: orig_mod_span, + items: vec![], + inline: true, + }; + }, + _ => unreachable!(), + }; + self.cx.trace_macros_diag(); + krate + } + + // Recursively expand all macro invocations in this AST fragment. + pub fn fully_expand_fragment(&mut self, input_fragment: AstFragment) -> AstFragment { + let orig_expansion_data = self.cx.current_expansion.clone(); + self.cx.current_expansion.depth = 0; + + // Collect all macro invocations and replace them with placeholders. + let (mut fragment_with_placeholders, mut invocations) + = self.collect_invocations(input_fragment, &[]); + + // Optimization: if we resolve all imports now, + // we'll be able to immediately resolve most of imported macros. + self.resolve_imports(); + + // Resolve paths in all invocations and produce output expanded fragments for them, but + // do not insert them into our input AST fragment yet, only store in `expanded_fragments`. + // The output fragments also go through expansion recursively until no invocations are left. + // Unresolved macros produce dummy outputs as a recovery measure. + invocations.reverse(); + let mut expanded_fragments = Vec::new(); + let mut all_derive_placeholders: FxHashMap> = FxHashMap::default(); + let mut undetermined_invocations = Vec::new(); + let (mut progress, mut force) = (false, !self.monotonic); + loop { + let invoc = if let Some(invoc) = invocations.pop() { + invoc + } else { + self.resolve_imports(); + if undetermined_invocations.is_empty() { break } + invocations = mem::take(&mut undetermined_invocations); + force = !mem::replace(&mut progress, false); + continue + }; + + let eager_expansion_root = + if self.monotonic { invoc.expansion_data.id } else { orig_expansion_data.id }; + let res = match self.cx.resolver.resolve_macro_invocation( + &invoc, eager_expansion_root, force + ) { + Ok(res) => res, + Err(Indeterminate) => { + undetermined_invocations.push(invoc); + continue + } + }; + + progress = true; + let ExpansionData { depth, id: expn_id, .. } = invoc.expansion_data; + self.cx.current_expansion = invoc.expansion_data.clone(); + + // FIXME(jseyfried): Refactor out the following logic + let (expanded_fragment, new_invocations) = match res { + InvocationRes::Single(ext) => { + let fragment = self.expand_invoc(invoc, &ext.kind); + self.collect_invocations(fragment, &[]) + } + InvocationRes::DeriveContainer(exts) => { + let (derives, item) = match invoc.kind { + InvocationKind::DeriveContainer { derives, item } => (derives, item), + _ => unreachable!(), + }; + if !item.derive_allowed() { + let attr = attr::find_by_name(item.attrs(), sym::derive) + .expect("`derive` attribute should exist"); + let span = attr.span; + let mut err = self.cx.struct_span_err(span, + "`derive` may only be applied to structs, enums and unions"); + if let ast::AttrStyle::Inner = attr.style { + let trait_list = derives.iter() + .map(|t| pprust::path_to_string(t)) + .collect::>(); + let suggestion = format!("#[derive({})]", trait_list.join(", ")); + err.span_suggestion( + span, "try an outer attribute", suggestion, + // We don't 𝑘𝑛𝑜𝑤 that the following item is an ADT + Applicability::MaybeIncorrect + ); + } + err.emit(); + } + + let mut item = self.fully_configure(item); + item.visit_attrs(|attrs| attrs.retain(|a| a.path != sym::derive)); + let mut helper_attrs = Vec::new(); + let mut has_copy = false; + for ext in exts { + helper_attrs.extend(&ext.helper_attrs); + has_copy |= ext.is_derive_copy; + } + // Mark derive helpers inside this item as known and used. + // FIXME: This is a hack, derive helpers should be integrated with regular name + // resolution instead. For example, helpers introduced by a derive container + // can be in scope for all code produced by that container's expansion. + item.visit_with(&mut MarkAttrs(&helper_attrs)); + if has_copy { + self.cx.resolver.add_derives(invoc.expansion_data.id, SpecialDerives::COPY); + } + + let derive_placeholders = + all_derive_placeholders.entry(invoc.expansion_data.id).or_default(); + derive_placeholders.reserve(derives.len()); + invocations.reserve(derives.len()); + for path in derives { + let expn_id = ExpnId::fresh(None); + derive_placeholders.push(NodeId::placeholder_from_expn_id(expn_id)); + invocations.push(Invocation { + kind: InvocationKind::Derive { path, item: item.clone() }, + fragment_kind: invoc.fragment_kind, + expansion_data: ExpansionData { + id: expn_id, + ..invoc.expansion_data.clone() + }, + }); + } + let fragment = invoc.fragment_kind + .expect_from_annotatables(::std::iter::once(item)); + self.collect_invocations(fragment, derive_placeholders) + } + }; + + if expanded_fragments.len() < depth { + expanded_fragments.push(Vec::new()); + } + expanded_fragments[depth - 1].push((expn_id, expanded_fragment)); + if !self.cx.ecfg.single_step { + invocations.extend(new_invocations.into_iter().rev()); + } + } + + self.cx.current_expansion = orig_expansion_data; + + // Finally incorporate all the expanded macros into the input AST fragment. + let mut placeholder_expander = PlaceholderExpander::new(self.cx, self.monotonic); + while let Some(expanded_fragments) = expanded_fragments.pop() { + for (expn_id, expanded_fragment) in expanded_fragments.into_iter().rev() { + let derive_placeholders = + all_derive_placeholders.remove(&expn_id).unwrap_or_else(Vec::new); + placeholder_expander.add(NodeId::placeholder_from_expn_id(expn_id), + expanded_fragment, derive_placeholders); + } + } + fragment_with_placeholders.mut_visit_with(&mut placeholder_expander); + fragment_with_placeholders + } + + fn resolve_imports(&mut self) { + if self.monotonic { + self.cx.resolver.resolve_imports(); + } + } + + /// Collects all macro invocations reachable at this time in this AST fragment, and replace + /// them with "placeholders" - dummy macro invocations with specially crafted `NodeId`s. + /// Then call into resolver that builds a skeleton ("reduced graph") of the fragment and + /// prepares data for resolving paths of macro invocations. + fn collect_invocations(&mut self, mut fragment: AstFragment, extra_placeholders: &[NodeId]) + -> (AstFragment, Vec) { + // Resolve `$crate`s in the fragment for pretty-printing. + self.cx.resolver.resolve_dollar_crates(); + + let invocations = { + let mut collector = InvocationCollector { + cfg: StripUnconfigured { + sess: self.cx.parse_sess, + features: self.cx.ecfg.features, + }, + cx: self.cx, + invocations: Vec::new(), + monotonic: self.monotonic, + }; + fragment.mut_visit_with(&mut collector); + collector.invocations + }; + + // FIXME: Merge `extra_placeholders` into the `fragment` as regular placeholders. + if self.monotonic { + self.cx.resolver.visit_ast_fragment_with_placeholders( + self.cx.current_expansion.id, &fragment, extra_placeholders); + } + + (fragment, invocations) + } + + fn fully_configure(&mut self, item: Annotatable) -> Annotatable { + let mut cfg = StripUnconfigured { + sess: self.cx.parse_sess, + features: self.cx.ecfg.features, + }; + // Since the item itself has already been configured by the InvocationCollector, + // we know that fold result vector will contain exactly one element + match item { + Annotatable::Item(item) => { + Annotatable::Item(cfg.flat_map_item(item).pop().unwrap()) + } + Annotatable::TraitItem(item) => { + Annotatable::TraitItem( + item.map(|item| cfg.flat_map_trait_item(item).pop().unwrap())) + } + Annotatable::ImplItem(item) => { + Annotatable::ImplItem(item.map(|item| cfg.flat_map_impl_item(item).pop().unwrap())) + } + Annotatable::ForeignItem(item) => { + Annotatable::ForeignItem( + item.map(|item| cfg.flat_map_foreign_item(item).pop().unwrap()) + ) + } + Annotatable::Stmt(stmt) => { + Annotatable::Stmt(stmt.map(|stmt| cfg.flat_map_stmt(stmt).pop().unwrap())) + } + Annotatable::Expr(mut expr) => { + Annotatable::Expr({ cfg.visit_expr(&mut expr); expr }) + } + Annotatable::Arm(arm) => { + Annotatable::Arm(cfg.flat_map_arm(arm).pop().unwrap()) + } + Annotatable::Field(field) => { + Annotatable::Field(cfg.flat_map_field(field).pop().unwrap()) + } + Annotatable::FieldPat(fp) => { + Annotatable::FieldPat(cfg.flat_map_field_pattern(fp).pop().unwrap()) + } + Annotatable::GenericParam(param) => { + Annotatable::GenericParam(cfg.flat_map_generic_param(param).pop().unwrap()) + } + Annotatable::Param(param) => { + Annotatable::Param(cfg.flat_map_param(param).pop().unwrap()) + } + Annotatable::StructField(sf) => { + Annotatable::StructField(cfg.flat_map_struct_field(sf).pop().unwrap()) + } + Annotatable::Variant(v) => { + Annotatable::Variant(cfg.flat_map_variant(v).pop().unwrap()) + } + } + } + + fn expand_invoc(&mut self, invoc: Invocation, ext: &SyntaxExtensionKind) -> AstFragment { + if self.cx.current_expansion.depth > self.cx.ecfg.recursion_limit { + let expn_data = self.cx.current_expansion.id.expn_data(); + let suggested_limit = self.cx.ecfg.recursion_limit * 2; + let mut err = self.cx.struct_span_err(expn_data.call_site, + &format!("recursion limit reached while expanding the macro `{}`", + expn_data.kind.descr())); + err.help(&format!( + "consider adding a `#![recursion_limit=\"{}\"]` attribute to your crate", + suggested_limit)); + err.emit(); + self.cx.trace_macros_diag(); + FatalError.raise(); + } + + let (fragment_kind, span) = (invoc.fragment_kind, invoc.span()); + match invoc.kind { + InvocationKind::Bang { mac, .. } => match ext { + SyntaxExtensionKind::Bang(expander) => { + self.gate_proc_macro_expansion_kind(span, fragment_kind); + let tok_result = expander.expand(self.cx, span, mac.stream()); + self.parse_ast_fragment(tok_result, fragment_kind, &mac.path, span) + } + SyntaxExtensionKind::LegacyBang(expander) => { + let prev = self.cx.current_expansion.prior_type_ascription; + self.cx.current_expansion.prior_type_ascription = mac.prior_type_ascription; + let tok_result = expander.expand(self.cx, span, mac.stream()); + let result = if let Some(result) = fragment_kind.make_from(tok_result) { + result + } else { + let msg = format!( + "non-{kind} macro in {kind} position: {path}", + kind = fragment_kind.name(), + path = pprust::path_to_string(&mac.path), + ); + self.cx.span_err(span, &msg); + self.cx.trace_macros_diag(); + fragment_kind.dummy(span) + }; + self.cx.current_expansion.prior_type_ascription = prev; + result + } + _ => unreachable!() + } + InvocationKind::Attr { attr, mut item, .. } => match ext { + SyntaxExtensionKind::Attr(expander) => { + self.gate_proc_macro_attr_item(span, &item); + let item_tok = TokenTree::token(token::Interpolated(Lrc::new(match item { + Annotatable::Item(item) => token::NtItem(item), + Annotatable::TraitItem(item) => token::NtTraitItem(item.into_inner()), + Annotatable::ImplItem(item) => token::NtImplItem(item.into_inner()), + Annotatable::ForeignItem(item) => token::NtForeignItem(item.into_inner()), + Annotatable::Stmt(stmt) => token::NtStmt(stmt.into_inner()), + Annotatable::Expr(expr) => token::NtExpr(expr), + Annotatable::Arm(..) + | Annotatable::Field(..) + | Annotatable::FieldPat(..) + | Annotatable::GenericParam(..) + | Annotatable::Param(..) + | Annotatable::StructField(..) + | Annotatable::Variant(..) + => panic!("unexpected annotatable"), + })), DUMMY_SP).into(); + let input = self.extract_proc_macro_attr_input(attr.item.tokens, span); + let tok_result = expander.expand(self.cx, span, input, item_tok); + self.parse_ast_fragment(tok_result, fragment_kind, &attr.item.path, span) + } + SyntaxExtensionKind::LegacyAttr(expander) => { + match attr.parse_meta(self.cx.parse_sess) { + Ok(meta) => { + let item = expander.expand(self.cx, span, &meta, item); + fragment_kind.expect_from_annotatables(item) + } + Err(mut err) => { + err.emit(); + fragment_kind.dummy(span) + } + } + } + SyntaxExtensionKind::NonMacroAttr { mark_used } => { + attr::mark_known(&attr); + if *mark_used { + attr::mark_used(&attr); + } + item.visit_attrs(|attrs| attrs.push(attr)); + fragment_kind.expect_from_annotatables(iter::once(item)) + } + _ => unreachable!() + } + InvocationKind::Derive { path, item } => match ext { + SyntaxExtensionKind::Derive(expander) | + SyntaxExtensionKind::LegacyDerive(expander) => { + if !item.derive_allowed() { + return fragment_kind.dummy(span); + } + let meta = ast::MetaItem { kind: ast::MetaItemKind::Word, span, path }; + let items = expander.expand(self.cx, span, &meta, item); + fragment_kind.expect_from_annotatables(items) + } + _ => unreachable!() + } + InvocationKind::DeriveContainer { .. } => unreachable!() + } + } + + fn extract_proc_macro_attr_input(&self, tokens: TokenStream, span: Span) -> TokenStream { + let mut trees = tokens.trees(); + match trees.next() { + Some(TokenTree::Delimited(_, _, tts)) => { + if trees.next().is_none() { + return tts.into() + } + } + Some(TokenTree::Token(..)) => {} + None => return TokenStream::default(), + } + self.cx.span_err(span, "custom attribute invocations must be \ + of the form `#[foo]` or `#[foo(..)]`, the macro name must only be \ + followed by a delimiter token"); + TokenStream::default() + } + + fn gate_proc_macro_attr_item(&self, span: Span, item: &Annotatable) { + let (kind, gate) = match *item { + Annotatable::Item(ref item) => { + match item.kind { + ItemKind::Mod(_) if self.cx.ecfg.proc_macro_hygiene() => return, + ItemKind::Mod(_) => ("modules", sym::proc_macro_hygiene), + _ => return, + } + } + Annotatable::TraitItem(_) => return, + Annotatable::ImplItem(_) => return, + Annotatable::ForeignItem(_) => return, + Annotatable::Stmt(_) | + Annotatable::Expr(_) if self.cx.ecfg.proc_macro_hygiene() => return, + Annotatable::Stmt(_) => ("statements", sym::proc_macro_hygiene), + Annotatable::Expr(_) => ("expressions", sym::proc_macro_hygiene), + Annotatable::Arm(..) + | Annotatable::Field(..) + | Annotatable::FieldPat(..) + | Annotatable::GenericParam(..) + | Annotatable::Param(..) + | Annotatable::StructField(..) + | Annotatable::Variant(..) + => panic!("unexpected annotatable"), + }; + emit_feature_err( + self.cx.parse_sess, + gate, + span, + GateIssue::Language, + &format!("custom attributes cannot be applied to {}", kind), + ); + } + + fn gate_proc_macro_expansion_kind(&self, span: Span, kind: AstFragmentKind) { + let kind = match kind { + AstFragmentKind::Expr | + AstFragmentKind::OptExpr => "expressions", + AstFragmentKind::Pat => "patterns", + AstFragmentKind::Stmts => "statements", + AstFragmentKind::Ty | + AstFragmentKind::Items | + AstFragmentKind::TraitItems | + AstFragmentKind::ImplItems | + AstFragmentKind::ForeignItems => return, + AstFragmentKind::Arms + | AstFragmentKind::Fields + | AstFragmentKind::FieldPats + | AstFragmentKind::GenericParams + | AstFragmentKind::Params + | AstFragmentKind::StructFields + | AstFragmentKind::Variants + => panic!("unexpected AST fragment kind"), + }; + if self.cx.ecfg.proc_macro_hygiene() { + return + } + emit_feature_err( + self.cx.parse_sess, + sym::proc_macro_hygiene, + span, + GateIssue::Language, + &format!("procedural macros cannot be expanded to {}", kind), + ); + } + + fn parse_ast_fragment( + &mut self, + toks: TokenStream, + kind: AstFragmentKind, + path: &Path, + span: Span, + ) -> AstFragment { + let mut parser = self.cx.new_parser_from_tts(toks); + match parse_ast_fragment(&mut parser, kind, false) { + Ok(fragment) => { + ensure_complete_parse(&mut parser, path, kind.name(), span); + fragment + } + Err(mut err) => { + err.set_span(span); + annotate_err_with_kind(&mut err, kind, span); + err.emit(); + self.cx.trace_macros_diag(); + kind.dummy(span) + } + } + } +} + +pub fn parse_ast_fragment<'a>( + this: &mut Parser<'a>, + kind: AstFragmentKind, + macro_legacy_warnings: bool, +) -> PResult<'a, AstFragment> { + Ok(match kind { + AstFragmentKind::Items => { + let mut items = SmallVec::new(); + while let Some(item) = this.parse_item()? { + items.push(item); + } + AstFragment::Items(items) + } + AstFragmentKind::TraitItems => { + let mut items = SmallVec::new(); + while this.token != token::Eof { + items.push(this.parse_trait_item(&mut false)?); + } + AstFragment::TraitItems(items) + } + AstFragmentKind::ImplItems => { + let mut items = SmallVec::new(); + while this.token != token::Eof { + items.push(this.parse_impl_item(&mut false)?); + } + AstFragment::ImplItems(items) + } + AstFragmentKind::ForeignItems => { + let mut items = SmallVec::new(); + while this.token != token::Eof { + items.push(this.parse_foreign_item(DUMMY_SP)?); + } + AstFragment::ForeignItems(items) + } + AstFragmentKind::Stmts => { + let mut stmts = SmallVec::new(); + while this.token != token::Eof && + // won't make progress on a `}` + this.token != token::CloseDelim(token::Brace) { + if let Some(stmt) = this.parse_full_stmt(macro_legacy_warnings)? { + stmts.push(stmt); + } + } + AstFragment::Stmts(stmts) + } + AstFragmentKind::Expr => AstFragment::Expr(this.parse_expr()?), + AstFragmentKind::OptExpr => { + if this.token != token::Eof { + AstFragment::OptExpr(Some(this.parse_expr()?)) + } else { + AstFragment::OptExpr(None) + } + }, + AstFragmentKind::Ty => AstFragment::Ty(this.parse_ty()?), + AstFragmentKind::Pat => AstFragment::Pat(this.parse_pat(None)?), + AstFragmentKind::Arms + | AstFragmentKind::Fields + | AstFragmentKind::FieldPats + | AstFragmentKind::GenericParams + | AstFragmentKind::Params + | AstFragmentKind::StructFields + | AstFragmentKind::Variants + => panic!("unexpected AST fragment kind"), + }) +} + +pub fn ensure_complete_parse<'a>( + this: &mut Parser<'a>, + macro_path: &Path, + kind_name: &str, + span: Span, +) { + if this.token != token::Eof { + let msg = format!("macro expansion ignores token `{}` and any following", + this.this_token_to_string()); + // Avoid emitting backtrace info twice. + let def_site_span = this.token.span.with_ctxt(SyntaxContext::root()); + let mut err = this.struct_span_err(def_site_span, &msg); + err.span_label(span, "caused by the macro expansion here"); + let msg = format!( + "the usage of `{}!` is likely invalid in {} context", + pprust::path_to_string(macro_path), + kind_name, + ); + err.note(&msg); + let semi_span = this.sess.source_map().next_point(span); + + let semi_full_span = semi_span.to(this.sess.source_map().next_point(semi_span)); + match this.sess.source_map().span_to_snippet(semi_full_span) { + Ok(ref snippet) if &snippet[..] != ";" && kind_name == "expression" => { + err.span_suggestion( + semi_span, + "you might be missing a semicolon here", + ";".to_owned(), + Applicability::MaybeIncorrect, + ); + } + _ => {} + } + err.emit(); + } +} + +struct InvocationCollector<'a, 'b> { + cx: &'a mut ExtCtxt<'b>, + cfg: StripUnconfigured<'a>, + invocations: Vec, + monotonic: bool, +} + +impl<'a, 'b> InvocationCollector<'a, 'b> { + fn collect(&mut self, fragment_kind: AstFragmentKind, kind: InvocationKind) -> AstFragment { + // Expansion data for all the collected invocations is set upon their resolution, + // with exception of the derive container case which is not resolved and can get + // its expansion data immediately. + let expn_data = match &kind { + InvocationKind::DeriveContainer { item, .. } => Some(ExpnData { + parent: self.cx.current_expansion.id, + ..ExpnData::default( + ExpnKind::Macro(MacroKind::Attr, sym::derive), + item.span(), self.cx.parse_sess.edition, + ) + }), + _ => None, + }; + let expn_id = ExpnId::fresh(expn_data); + self.invocations.push(Invocation { + kind, + fragment_kind, + expansion_data: ExpansionData { + id: expn_id, + depth: self.cx.current_expansion.depth + 1, + ..self.cx.current_expansion.clone() + }, + }); + placeholder(fragment_kind, NodeId::placeholder_from_expn_id(expn_id)) + } + + fn collect_bang(&mut self, mac: ast::Mac, span: Span, kind: AstFragmentKind) -> AstFragment { + self.collect(kind, InvocationKind::Bang { mac, span }) + } + + fn collect_attr(&mut self, + attr: Option, + derives: Vec, + item: Annotatable, + kind: AstFragmentKind, + after_derive: bool) + -> AstFragment { + self.collect(kind, match attr { + Some(attr) => InvocationKind::Attr { attr, item, derives, after_derive }, + None => InvocationKind::DeriveContainer { derives, item }, + }) + } + + fn find_attr_invoc(&self, attrs: &mut Vec, after_derive: &mut bool) + -> Option { + let attr = attrs.iter() + .position(|a| { + if a.path == sym::derive { + *after_derive = true; + } + !attr::is_known(a) && !is_builtin_attr(a) + }) + .map(|i| attrs.remove(i)); + if let Some(attr) = &attr { + if !self.cx.ecfg.custom_inner_attributes() && + attr.style == ast::AttrStyle::Inner && attr.path != sym::test { + emit_feature_err(&self.cx.parse_sess, sym::custom_inner_attributes, + attr.span, GateIssue::Language, + "non-builtin inner attributes are unstable"); + } + } + attr + } + + /// If `item` is an attr invocation, remove and return the macro attribute and derive traits. + fn classify_item(&mut self, item: &mut T) + -> (Option, Vec, /* after_derive */ bool) + where T: HasAttrs, + { + let (mut attr, mut traits, mut after_derive) = (None, Vec::new(), false); + + item.visit_attrs(|mut attrs| { + attr = self.find_attr_invoc(&mut attrs, &mut after_derive); + traits = collect_derives(&mut self.cx, &mut attrs); + }); + + (attr, traits, after_derive) + } + + /// Alternative to `classify_item()` that ignores `#[derive]` so invocations fallthrough + /// to the unused-attributes lint (making it an error on statements and expressions + /// is a breaking change) + fn classify_nonitem(&mut self, nonitem: &mut T) + -> (Option, /* after_derive */ bool) { + let (mut attr, mut after_derive) = (None, false); + + nonitem.visit_attrs(|mut attrs| { + attr = self.find_attr_invoc(&mut attrs, &mut after_derive); + }); + + (attr, after_derive) + } + + fn configure(&mut self, node: T) -> Option { + self.cfg.configure(node) + } + + // Detect use of feature-gated or invalid attributes on macro invocations + // since they will not be detected after macro expansion. + fn check_attributes(&mut self, attrs: &[ast::Attribute]) { + let features = self.cx.ecfg.features.unwrap(); + for attr in attrs.iter() { + feature_gate::check_attribute(attr, self.cx.parse_sess, features); + + // macros are expanded before any lint passes so this warning has to be hardcoded + if attr.path == sym::derive { + self.cx.struct_span_warn(attr.span, "`#[derive]` does nothing on macro invocations") + .note("this may become a hard error in a future release") + .emit(); + } + } + } +} + +impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> { + fn visit_expr(&mut self, expr: &mut P) { + self.cfg.configure_expr(expr); + visit_clobber(expr.deref_mut(), |mut expr| { + self.cfg.configure_expr_kind(&mut expr.kind); + + // ignore derives so they remain unused + let (attr, after_derive) = self.classify_nonitem(&mut expr); + + if attr.is_some() { + // Collect the invoc regardless of whether or not attributes are permitted here + // expansion will eat the attribute so it won't error later. + attr.as_ref().map(|a| self.cfg.maybe_emit_expr_attr_err(a)); + + // AstFragmentKind::Expr requires the macro to emit an expression. + return self.collect_attr(attr, vec![], Annotatable::Expr(P(expr)), + AstFragmentKind::Expr, after_derive) + .make_expr() + .into_inner() + } + + if let ast::ExprKind::Mac(mac) = expr.kind { + self.check_attributes(&expr.attrs); + self.collect_bang(mac, expr.span, AstFragmentKind::Expr) + .make_expr() + .into_inner() + } else { + noop_visit_expr(&mut expr, self); + expr + } + }); + } + + fn flat_map_arm(&mut self, arm: ast::Arm) -> SmallVec<[ast::Arm; 1]> { + let mut arm = configure!(self, arm); + + let (attr, traits, after_derive) = self.classify_item(&mut arm); + if attr.is_some() || !traits.is_empty() { + return self.collect_attr(attr, traits, Annotatable::Arm(arm), + AstFragmentKind::Arms, after_derive) + .make_arms(); + } + + noop_flat_map_arm(arm, self) + } + + fn flat_map_field(&mut self, field: ast::Field) -> SmallVec<[ast::Field; 1]> { + let mut field = configure!(self, field); + + let (attr, traits, after_derive) = self.classify_item(&mut field); + if attr.is_some() || !traits.is_empty() { + return self.collect_attr(attr, traits, Annotatable::Field(field), + AstFragmentKind::Fields, after_derive) + .make_fields(); + } + + noop_flat_map_field(field, self) + } + + fn flat_map_field_pattern(&mut self, fp: ast::FieldPat) -> SmallVec<[ast::FieldPat; 1]> { + let mut fp = configure!(self, fp); + + let (attr, traits, after_derive) = self.classify_item(&mut fp); + if attr.is_some() || !traits.is_empty() { + return self.collect_attr(attr, traits, Annotatable::FieldPat(fp), + AstFragmentKind::FieldPats, after_derive) + .make_field_patterns(); + } + + noop_flat_map_field_pattern(fp, self) + } + + fn flat_map_param(&mut self, p: ast::Param) -> SmallVec<[ast::Param; 1]> { + let mut p = configure!(self, p); + + let (attr, traits, after_derive) = self.classify_item(&mut p); + if attr.is_some() || !traits.is_empty() { + return self.collect_attr(attr, traits, Annotatable::Param(p), + AstFragmentKind::Params, after_derive) + .make_params(); + } + + noop_flat_map_param(p, self) + } + + fn flat_map_struct_field(&mut self, sf: ast::StructField) -> SmallVec<[ast::StructField; 1]> { + let mut sf = configure!(self, sf); + + let (attr, traits, after_derive) = self.classify_item(&mut sf); + if attr.is_some() || !traits.is_empty() { + return self.collect_attr(attr, traits, Annotatable::StructField(sf), + AstFragmentKind::StructFields, after_derive) + .make_struct_fields(); + } + + noop_flat_map_struct_field(sf, self) + } + + fn flat_map_variant(&mut self, variant: ast::Variant) -> SmallVec<[ast::Variant; 1]> { + let mut variant = configure!(self, variant); + + let (attr, traits, after_derive) = self.classify_item(&mut variant); + if attr.is_some() || !traits.is_empty() { + return self.collect_attr(attr, traits, Annotatable::Variant(variant), + AstFragmentKind::Variants, after_derive) + .make_variants(); + } + + noop_flat_map_variant(variant, self) + } + + fn filter_map_expr(&mut self, expr: P) -> Option> { + let expr = configure!(self, expr); + expr.filter_map(|mut expr| { + self.cfg.configure_expr_kind(&mut expr.kind); + + // Ignore derives so they remain unused. + let (attr, after_derive) = self.classify_nonitem(&mut expr); + + if attr.is_some() { + attr.as_ref().map(|a| self.cfg.maybe_emit_expr_attr_err(a)); + + return self.collect_attr(attr, vec![], Annotatable::Expr(P(expr)), + AstFragmentKind::OptExpr, after_derive) + .make_opt_expr() + .map(|expr| expr.into_inner()) + } + + if let ast::ExprKind::Mac(mac) = expr.kind { + self.check_attributes(&expr.attrs); + self.collect_bang(mac, expr.span, AstFragmentKind::OptExpr) + .make_opt_expr() + .map(|expr| expr.into_inner()) + } else { + Some({ noop_visit_expr(&mut expr, self); expr }) + } + }) + } + + fn visit_pat(&mut self, pat: &mut P) { + self.cfg.configure_pat(pat); + match pat.kind { + PatKind::Mac(_) => {} + _ => return noop_visit_pat(pat, self), + } + + visit_clobber(pat, |mut pat| { + match mem::replace(&mut pat.kind, PatKind::Wild) { + PatKind::Mac(mac) => + self.collect_bang(mac, pat.span, AstFragmentKind::Pat).make_pat(), + _ => unreachable!(), + } + }); + } + + fn flat_map_stmt(&mut self, stmt: ast::Stmt) -> SmallVec<[ast::Stmt; 1]> { + let mut stmt = configure!(self, stmt); + + // we'll expand attributes on expressions separately + if !stmt.is_expr() { + let (attr, derives, after_derive) = if stmt.is_item() { + self.classify_item(&mut stmt) + } else { + // ignore derives on non-item statements so it falls through + // to the unused-attributes lint + let (attr, after_derive) = self.classify_nonitem(&mut stmt); + (attr, vec![], after_derive) + }; + + if attr.is_some() || !derives.is_empty() { + return self.collect_attr(attr, derives, Annotatable::Stmt(P(stmt)), + AstFragmentKind::Stmts, after_derive).make_stmts(); + } + } + + if let StmtKind::Mac(mac) = stmt.kind { + let (mac, style, attrs) = mac.into_inner(); + self.check_attributes(&attrs); + let mut placeholder = self.collect_bang(mac, stmt.span, AstFragmentKind::Stmts) + .make_stmts(); + + // If this is a macro invocation with a semicolon, then apply that + // semicolon to the final statement produced by expansion. + if style == MacStmtStyle::Semicolon { + if let Some(stmt) = placeholder.pop() { + placeholder.push(stmt.add_trailing_semicolon()); + } + } + + return placeholder; + } + + // The placeholder expander gives ids to statements, so we avoid folding the id here. + let ast::Stmt { id, kind, span } = stmt; + noop_flat_map_stmt_kind(kind, self).into_iter().map(|kind| { + ast::Stmt { id, kind, span } + }).collect() + + } + + fn visit_block(&mut self, block: &mut P) { + let old_directory_ownership = self.cx.current_expansion.directory_ownership; + self.cx.current_expansion.directory_ownership = DirectoryOwnership::UnownedViaBlock; + noop_visit_block(block, self); + self.cx.current_expansion.directory_ownership = old_directory_ownership; + } + + fn flat_map_item(&mut self, item: P) -> SmallVec<[P; 1]> { + let mut item = configure!(self, item); + + let (attr, traits, after_derive) = self.classify_item(&mut item); + if attr.is_some() || !traits.is_empty() { + return self.collect_attr(attr, traits, Annotatable::Item(item), + AstFragmentKind::Items, after_derive).make_items(); + } + + match item.kind { + ast::ItemKind::Mac(..) => { + self.check_attributes(&item.attrs); + item.and_then(|item| match item.kind { + ItemKind::Mac(mac) => self.collect( + AstFragmentKind::Items, InvocationKind::Bang { mac, span: item.span } + ).make_items(), + _ => unreachable!(), + }) + } + ast::ItemKind::Mod(ast::Mod { inner, .. }) => { + if item.ident == Ident::invalid() { + return noop_flat_map_item(item, self); + } + + let orig_directory_ownership = self.cx.current_expansion.directory_ownership; + let mut module = (*self.cx.current_expansion.module).clone(); + module.mod_path.push(item.ident); + + // Detect if this is an inline module (`mod m { ... }` as opposed to `mod m;`). + // In the non-inline case, `inner` is never the dummy span (cf. `parse_item_mod`). + // Thus, if `inner` is the dummy span, we know the module is inline. + let inline_module = item.span.contains(inner) || inner.is_dummy(); + + if inline_module { + if let Some(path) = attr::first_attr_value_str_by_name(&item.attrs, sym::path) { + self.cx.current_expansion.directory_ownership = + DirectoryOwnership::Owned { relative: None }; + module.directory.push(&*path.as_str()); + } else { + module.directory.push(&*item.ident.as_str()); + } + } else { + let path = self.cx.parse_sess.source_map().span_to_unmapped_path(inner); + let mut path = match path { + FileName::Real(path) => path, + other => PathBuf::from(other.to_string()), + }; + let directory_ownership = match path.file_name().unwrap().to_str() { + Some("mod.rs") => DirectoryOwnership::Owned { relative: None }, + Some(_) => DirectoryOwnership::Owned { + relative: Some(item.ident), + }, + None => DirectoryOwnership::UnownedViaMod(false), + }; + path.pop(); + module.directory = path; + self.cx.current_expansion.directory_ownership = directory_ownership; + } + + let orig_module = + mem::replace(&mut self.cx.current_expansion.module, Rc::new(module)); + let result = noop_flat_map_item(item, self); + self.cx.current_expansion.module = orig_module; + self.cx.current_expansion.directory_ownership = orig_directory_ownership; + result + } + + _ => noop_flat_map_item(item, self), + } + } + + fn flat_map_trait_item(&mut self, item: ast::TraitItem) -> SmallVec<[ast::TraitItem; 1]> { + let mut item = configure!(self, item); + + let (attr, traits, after_derive) = self.classify_item(&mut item); + if attr.is_some() || !traits.is_empty() { + return self.collect_attr(attr, traits, Annotatable::TraitItem(P(item)), + AstFragmentKind::TraitItems, after_derive).make_trait_items() + } + + match item.kind { + ast::TraitItemKind::Macro(mac) => { + let ast::TraitItem { attrs, span, .. } = item; + self.check_attributes(&attrs); + self.collect_bang(mac, span, AstFragmentKind::TraitItems).make_trait_items() + } + _ => noop_flat_map_trait_item(item, self), + } + } + + fn flat_map_impl_item(&mut self, item: ast::ImplItem) -> SmallVec<[ast::ImplItem; 1]> { + let mut item = configure!(self, item); + + let (attr, traits, after_derive) = self.classify_item(&mut item); + if attr.is_some() || !traits.is_empty() { + return self.collect_attr(attr, traits, Annotatable::ImplItem(P(item)), + AstFragmentKind::ImplItems, after_derive).make_impl_items(); + } + + match item.kind { + ast::ImplItemKind::Macro(mac) => { + let ast::ImplItem { attrs, span, .. } = item; + self.check_attributes(&attrs); + self.collect_bang(mac, span, AstFragmentKind::ImplItems).make_impl_items() + } + _ => noop_flat_map_impl_item(item, self), + } + } + + fn visit_ty(&mut self, ty: &mut P) { + match ty.kind { + ast::TyKind::Mac(_) => {} + _ => return noop_visit_ty(ty, self), + }; + + visit_clobber(ty, |mut ty| { + match mem::replace(&mut ty.kind, ast::TyKind::Err) { + ast::TyKind::Mac(mac) => + self.collect_bang(mac, ty.span, AstFragmentKind::Ty).make_ty(), + _ => unreachable!(), + } + }); + } + + fn visit_foreign_mod(&mut self, foreign_mod: &mut ast::ForeignMod) { + self.cfg.configure_foreign_mod(foreign_mod); + noop_visit_foreign_mod(foreign_mod, self); + } + + fn flat_map_foreign_item(&mut self, mut foreign_item: ast::ForeignItem) + -> SmallVec<[ast::ForeignItem; 1]> + { + let (attr, traits, after_derive) = self.classify_item(&mut foreign_item); + + if attr.is_some() || !traits.is_empty() { + return self.collect_attr(attr, traits, Annotatable::ForeignItem(P(foreign_item)), + AstFragmentKind::ForeignItems, after_derive) + .make_foreign_items(); + } + + if let ast::ForeignItemKind::Macro(mac) = foreign_item.kind { + self.check_attributes(&foreign_item.attrs); + return self.collect_bang(mac, foreign_item.span, AstFragmentKind::ForeignItems) + .make_foreign_items(); + } + + noop_flat_map_foreign_item(foreign_item, self) + } + + fn visit_item_kind(&mut self, item: &mut ast::ItemKind) { + match item { + ast::ItemKind::MacroDef(..) => {} + _ => { + self.cfg.configure_item_kind(item); + noop_visit_item_kind(item, self); + } + } + } + + fn flat_map_generic_param( + &mut self, + param: ast::GenericParam + ) -> SmallVec<[ast::GenericParam; 1]> + { + let mut param = configure!(self, param); + + let (attr, traits, after_derive) = self.classify_item(&mut param); + if attr.is_some() || !traits.is_empty() { + return self.collect_attr(attr, traits, Annotatable::GenericParam(param), + AstFragmentKind::GenericParams, after_derive) + .make_generic_params(); + } + + noop_flat_map_generic_param(param, self) + } + + fn visit_attribute(&mut self, at: &mut ast::Attribute) { + // turn `#[doc(include="filename")]` attributes into `#[doc(include(file="filename", + // contents="file contents")]` attributes + if !at.check_name(sym::doc) { + return noop_visit_attribute(at, self); + } + + if let Some(list) = at.meta_item_list() { + if !list.iter().any(|it| it.check_name(sym::include)) { + return noop_visit_attribute(at, self); + } + + let mut items = vec![]; + + for mut it in list { + if !it.check_name(sym::include) { + items.push({ noop_visit_meta_list_item(&mut it, self); it }); + continue; + } + + if let Some(file) = it.value_str() { + let err_count = self.cx.parse_sess.span_diagnostic.err_count(); + self.check_attributes(slice::from_ref(at)); + if self.cx.parse_sess.span_diagnostic.err_count() > err_count { + // avoid loading the file if they haven't enabled the feature + return noop_visit_attribute(at, self); + } + + let filename = self.cx.resolve_path(&*file.as_str(), it.span()); + match self.cx.source_map().load_file(&filename) { + Ok(source_file) => { + let src = source_file.src.as_ref() + .expect("freshly loaded file should have a source"); + let src_interned = Symbol::intern(src.as_str()); + + let include_info = vec![ + ast::NestedMetaItem::MetaItem( + attr::mk_name_value_item_str( + Ident::with_dummy_span(sym::file), + file, + DUMMY_SP, + ), + ), + ast::NestedMetaItem::MetaItem( + attr::mk_name_value_item_str( + Ident::with_dummy_span(sym::contents), + src_interned, + DUMMY_SP, + ), + ), + ]; + + let include_ident = Ident::with_dummy_span(sym::include); + let item = attr::mk_list_item(include_ident, include_info); + items.push(ast::NestedMetaItem::MetaItem(item)); + } + Err(e) => { + let lit = it + .meta_item() + .and_then(|item| item.name_value_literal()) + .unwrap(); + + if e.kind() == ErrorKind::InvalidData { + self.cx + .struct_span_err( + lit.span, + &format!("{} wasn't a utf-8 file", filename.display()), + ) + .span_label(lit.span, "contains invalid utf-8") + .emit(); + } else { + let mut err = self.cx.struct_span_err( + lit.span, + &format!("couldn't read {}: {}", filename.display(), e), + ); + err.span_label(lit.span, "couldn't read file"); + + err.emit(); + } + } + } + } else { + let mut err = self.cx.struct_span_err( + it.span(), + &format!("expected path to external documentation"), + ); + + // Check if the user erroneously used `doc(include(...))` syntax. + let literal = it.meta_item_list().and_then(|list| { + if list.len() == 1 { + list[0].literal().map(|literal| &literal.kind) + } else { + None + } + }); + + let (path, applicability) = match &literal { + Some(LitKind::Str(path, ..)) => { + (path.to_string(), Applicability::MachineApplicable) + } + _ => (String::from(""), Applicability::HasPlaceholders), + }; + + err.span_suggestion( + it.span(), + "provide a file path with `=`", + format!("include = \"{}\"", path), + applicability, + ); + + err.emit(); + } + } + + let meta = attr::mk_list_item(Ident::with_dummy_span(sym::doc), items); + *at = attr::Attribute { + item: AttrItem { path: meta.path, tokens: meta.kind.tokens(meta.span) }, + span: at.span, + id: at.id, + style: at.style, + is_sugared_doc: false, + }; + } else { + noop_visit_attribute(at, self) + } + } + + fn visit_id(&mut self, id: &mut ast::NodeId) { + if self.monotonic { + debug_assert_eq!(*id, ast::DUMMY_NODE_ID); + *id = self.cx.resolver.next_node_id() + } + } + + fn visit_fn_decl(&mut self, mut fn_decl: &mut P) { + self.cfg.configure_fn_decl(&mut fn_decl); + noop_visit_fn_decl(fn_decl, self); + } +} + +pub struct ExpansionConfig<'feat> { + pub crate_name: String, + pub features: Option<&'feat Features>, + pub recursion_limit: usize, + pub trace_mac: bool, + pub should_test: bool, // If false, strip `#[test]` nodes + pub single_step: bool, + pub keep_macs: bool, +} + +impl<'feat> ExpansionConfig<'feat> { + pub fn default(crate_name: String) -> ExpansionConfig<'static> { + ExpansionConfig { + crate_name, + features: None, + recursion_limit: 1024, + trace_mac: false, + should_test: false, + single_step: false, + keep_macs: false, + } + } + + fn proc_macro_hygiene(&self) -> bool { + self.features.map_or(false, |features| features.proc_macro_hygiene) + } + fn custom_inner_attributes(&self) -> bool { + self.features.map_or(false, |features| features.custom_inner_attributes) + } +} diff --git a/src/libsyntax_expand/lib.rs b/src/libsyntax_expand/lib.rs new file mode 100644 index 00000000000..88e69d79397 --- /dev/null +++ b/src/libsyntax_expand/lib.rs @@ -0,0 +1,38 @@ +#![feature(crate_visibility_modifier)] +#![feature(proc_macro_diagnostic)] +#![feature(proc_macro_internals)] +#![feature(proc_macro_span)] + +extern crate proc_macro as pm; + +// A variant of 'try!' that panics on an Err. This is used as a crutch on the +// way towards a non-panic!-prone parser. It should be used for fatal parsing +// errors; eventually we plan to convert all code using panictry to just use +// normal try. +#[macro_export] +macro_rules! panictry { + ($e:expr) => ({ + use std::result::Result::{Ok, Err}; + use errors::FatalError; + match $e { + Ok(e) => e, + Err(mut e) => { + e.emit(); + FatalError.raise() + } + } + }) +} + +mod placeholders; +mod proc_macro_server; + +pub use syntax_pos::hygiene; +pub use mbe::macro_rules::compile_declarative_macro; +pub mod allocator; +pub mod base; +pub mod build; +pub mod expand; +pub mod proc_macro; + +crate mod mbe; diff --git a/src/libsyntax_expand/mbe.rs b/src/libsyntax_expand/mbe.rs new file mode 100644 index 00000000000..453fe94f1de --- /dev/null +++ b/src/libsyntax_expand/mbe.rs @@ -0,0 +1,166 @@ +//! This module implements declarative macros: old `macro_rules` and the newer +//! `macro`. Declarative macros are also known as "macro by example", and that's +//! why we call this module `mbe`. For external documentation, prefer the +//! official terminology: "declarative macros". + +crate mod transcribe; +crate mod macro_check; +crate mod macro_parser; +crate mod macro_rules; +crate mod quoted; + +use syntax::ast; +use syntax::parse::token::{self, Token, TokenKind}; +use syntax::tokenstream::{DelimSpan}; + +use syntax_pos::{BytePos, Span}; + +use rustc_data_structures::sync::Lrc; + +/// Contains the sub-token-trees of a "delimited" token tree, such as the contents of `(`. Note +/// that the delimiter itself might be `NoDelim`. +#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug)] +struct Delimited { + delim: token::DelimToken, + tts: Vec, +} + +impl Delimited { + /// Returns a `self::TokenTree` with a `Span` corresponding to the opening delimiter. + fn open_tt(&self, span: Span) -> TokenTree { + let open_span = if span.is_dummy() { + span + } else { + span.with_hi(span.lo() + BytePos(self.delim.len() as u32)) + }; + TokenTree::token(token::OpenDelim(self.delim), open_span) + } + + /// Returns a `self::TokenTree` with a `Span` corresponding to the closing delimiter. + fn close_tt(&self, span: Span) -> TokenTree { + let close_span = if span.is_dummy() { + span + } else { + span.with_lo(span.hi() - BytePos(self.delim.len() as u32)) + }; + TokenTree::token(token::CloseDelim(self.delim), close_span) + } +} + +#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug)] +struct SequenceRepetition { + /// The sequence of token trees + tts: Vec, + /// The optional separator + separator: Option, + /// Whether the sequence can be repeated zero (*), or one or more times (+) + kleene: KleeneToken, + /// The number of `Match`s that appear in the sequence (and subsequences) + num_captures: usize, +} + +#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug, Copy)] +struct KleeneToken { + span: Span, + op: KleeneOp, +} + +impl KleeneToken { + fn new(op: KleeneOp, span: Span) -> KleeneToken { + KleeneToken { span, op } + } +} + +/// A Kleene-style [repetition operator](http://en.wikipedia.org/wiki/Kleene_star) +/// for token sequences. +#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] +enum KleeneOp { + /// Kleene star (`*`) for zero or more repetitions + ZeroOrMore, + /// Kleene plus (`+`) for one or more repetitions + OneOrMore, + /// Kleene optional (`?`) for zero or one reptitions + ZeroOrOne, +} + +/// Similar to `tokenstream::TokenTree`, except that `$i`, `$i:ident`, and `$(...)` +/// are "first-class" token trees. Useful for parsing macros. +#[derive(Debug, Clone, PartialEq, RustcEncodable, RustcDecodable)] +enum TokenTree { + Token(Token), + Delimited(DelimSpan, Lrc), + /// A kleene-style repetition sequence + Sequence(DelimSpan, Lrc), + /// e.g., `$var` + MetaVar(Span, ast::Ident), + /// e.g., `$var:expr`. This is only used in the left hand side of MBE macros. + MetaVarDecl( + Span, + ast::Ident, /* name to bind */ + ast::Ident, /* kind of nonterminal */ + ), +} + +impl TokenTree { + /// Return the number of tokens in the tree. + fn len(&self) -> usize { + match *self { + TokenTree::Delimited(_, ref delimed) => match delimed.delim { + token::NoDelim => delimed.tts.len(), + _ => delimed.tts.len() + 2, + }, + TokenTree::Sequence(_, ref seq) => seq.tts.len(), + _ => 0, + } + } + + /// Returns `true` if the given token tree is delimited. + fn is_delimited(&self) -> bool { + match *self { + TokenTree::Delimited(..) => true, + _ => false, + } + } + + /// Returns `true` if the given token tree is a token of the given kind. + fn is_token(&self, expected_kind: &TokenKind) -> bool { + match self { + TokenTree::Token(Token { kind: actual_kind, .. }) => actual_kind == expected_kind, + _ => false, + } + } + + /// Gets the `index`-th sub-token-tree. This only makes sense for delimited trees and sequences. + fn get_tt(&self, index: usize) -> TokenTree { + match (self, index) { + (&TokenTree::Delimited(_, ref delimed), _) if delimed.delim == token::NoDelim => { + delimed.tts[index].clone() + } + (&TokenTree::Delimited(span, ref delimed), _) => { + if index == 0 { + return delimed.open_tt(span.open); + } + if index == delimed.tts.len() + 1 { + return delimed.close_tt(span.close); + } + delimed.tts[index - 1].clone() + } + (&TokenTree::Sequence(_, ref seq), _) => seq.tts[index].clone(), + _ => panic!("Cannot expand a token tree"), + } + } + + /// Retrieves the `TokenTree`'s span. + fn span(&self) -> Span { + match *self { + TokenTree::Token(Token { span, .. }) + | TokenTree::MetaVar(span, _) + | TokenTree::MetaVarDecl(span, _, _) => span, + TokenTree::Delimited(span, _) | TokenTree::Sequence(span, _) => span.entire(), + } + } + + fn token(kind: TokenKind, span: Span) -> TokenTree { + TokenTree::Token(Token::new(kind, span)) + } +} diff --git a/src/libsyntax_expand/mbe/macro_check.rs b/src/libsyntax_expand/mbe/macro_check.rs new file mode 100644 index 00000000000..50abda8d45e --- /dev/null +++ b/src/libsyntax_expand/mbe/macro_check.rs @@ -0,0 +1,626 @@ +//! Checks that meta-variables in macro definition are correctly declared and used. +//! +//! # What is checked +//! +//! ## Meta-variables must not be bound twice +//! +//! ``` +//! macro_rules! foo { ($x:tt $x:tt) => { $x }; } +//! ``` +//! +//! This check is sound (no false-negative) and complete (no false-positive). +//! +//! ## Meta-variables must not be free +//! +//! ``` +//! macro_rules! foo { () => { $x }; } +//! ``` +//! +//! This check is also done at macro instantiation but only if the branch is taken. +//! +//! ## Meta-variables must repeat at least as many times as their binder +//! +//! ``` +//! macro_rules! foo { ($($x:tt)*) => { $x }; } +//! ``` +//! +//! This check is also done at macro instantiation but only if the branch is taken. +//! +//! ## Meta-variables must repeat with the same Kleene operators as their binder +//! +//! ``` +//! macro_rules! foo { ($($x:tt)+) => { $($x)* }; } +//! ``` +//! +//! This check is not done at macro instantiation. +//! +//! # Disclaimer +//! +//! In the presence of nested macros (a macro defined in a macro), those checks may have false +//! positives and false negatives. We try to detect those cases by recognizing potential macro +//! definitions in RHSes, but nested macros may be hidden through the use of particular values of +//! meta-variables. +//! +//! ## Examples of false positive +//! +//! False positives can come from cases where we don't recognize a nested macro, because it depends +//! on particular values of meta-variables. In the following example, we think both instances of +//! `$x` are free, which is a correct statement if `$name` is anything but `macro_rules`. But when +//! `$name` is `macro_rules`, like in the instantiation below, then `$x:tt` is actually a binder of +//! the nested macro and `$x` is bound to it. +//! +//! ``` +//! macro_rules! foo { ($name:ident) => { $name! bar { ($x:tt) => { $x }; } }; } +//! foo!(macro_rules); +//! ``` +//! +//! False positives can also come from cases where we think there is a nested macro while there +//! isn't. In the following example, we think `$x` is free, which is incorrect because `bar` is not +//! a nested macro since it is not evaluated as code by `stringify!`. +//! +//! ``` +//! macro_rules! foo { () => { stringify!(macro_rules! bar { () => { $x }; }) }; } +//! ``` +//! +//! ## Examples of false negative +//! +//! False negatives can come from cases where we don't recognize a meta-variable, because it depends +//! on particular values of meta-variables. In the following examples, we don't see that if `$d` is +//! instantiated with `$` then `$d z` becomes `$z` in the nested macro definition and is thus a free +//! meta-variable. Note however, that if `foo` is instantiated, then we would check the definition +//! of `bar` and would see the issue. +//! +//! ``` +//! macro_rules! foo { ($d:tt) => { macro_rules! bar { ($y:tt) => { $d z }; } }; } +//! ``` +//! +//! # How it is checked +//! +//! There are 3 main functions: `check_binders`, `check_occurrences`, and `check_nested_macro`. They +//! all need some kind of environment. +//! +//! ## Environments +//! +//! Environments are used to pass information. +//! +//! ### From LHS to RHS +//! +//! When checking a LHS with `check_binders`, we produce (and use) an environment for binders, +//! namely `Binders`. This is a mapping from binder name to information about that binder: the span +//! of the binder for error messages and the stack of Kleene operators under which it was bound in +//! the LHS. +//! +//! This environment is used by both the LHS and RHS. The LHS uses it to detect duplicate binders. +//! The RHS uses it to detect the other errors. +//! +//! ### From outer macro to inner macro +//! +//! When checking the RHS of an outer macro and we detect a nested macro definition, we push the +//! current state, namely `MacroState`, to an environment of nested macro definitions. Each state +//! stores the LHS binders when entering the macro definition as well as the stack of Kleene +//! operators under which the inner macro is defined in the RHS. +//! +//! This environment is a stack representing the nesting of macro definitions. As such, the stack of +//! Kleene operators under which a meta-variable is repeating is the concatenation of the stacks +//! stored when entering a macro definition starting from the state in which the meta-variable is +//! bound. +use crate::mbe::{KleeneToken, TokenTree}; + +use syntax::ast::NodeId; +use syntax::early_buffered_lints::BufferedEarlyLintId; +use syntax::parse::token::{DelimToken, Token, TokenKind}; +use syntax::sess::ParseSess; +use syntax::symbol::{kw, sym}; + +use rustc_data_structures::fx::FxHashMap; +use smallvec::SmallVec; +use syntax_pos::{symbol::Ident, MultiSpan, Span}; + +/// Stack represented as linked list. +/// +/// Those are used for environments because they grow incrementally and are not mutable. +enum Stack<'a, T> { + /// Empty stack. + Empty, + /// A non-empty stack. + Push { + /// The top element. + top: T, + /// The previous elements. + prev: &'a Stack<'a, T>, + }, +} + +impl<'a, T> Stack<'a, T> { + /// Returns whether a stack is empty. + fn is_empty(&self) -> bool { + match *self { + Stack::Empty => true, + _ => false, + } + } + + /// Returns a new stack with an element of top. + fn push(&'a self, top: T) -> Stack<'a, T> { + Stack::Push { top, prev: self } + } +} + +impl<'a, T> Iterator for &'a Stack<'a, T> { + type Item = &'a T; + + // Iterates from top to bottom of the stack. + fn next(&mut self) -> Option<&'a T> { + match *self { + Stack::Empty => None, + Stack::Push { ref top, ref prev } => { + *self = prev; + Some(top) + } + } + } +} + +impl From<&Stack<'_, KleeneToken>> for SmallVec<[KleeneToken; 1]> { + fn from(ops: &Stack<'_, KleeneToken>) -> SmallVec<[KleeneToken; 1]> { + let mut ops: SmallVec<[KleeneToken; 1]> = ops.cloned().collect(); + // The stack is innermost on top. We want outermost first. + ops.reverse(); + ops + } +} + +/// Information attached to a meta-variable binder in LHS. +struct BinderInfo { + /// The span of the meta-variable in LHS. + span: Span, + /// The stack of Kleene operators (outermost first). + ops: SmallVec<[KleeneToken; 1]>, +} + +/// An environment of meta-variables to their binder information. +type Binders = FxHashMap; + +/// The state at which we entered a macro definition in the RHS of another macro definition. +struct MacroState<'a> { + /// The binders of the branch where we entered the macro definition. + binders: &'a Binders, + /// The stack of Kleene operators (outermost first) where we entered the macro definition. + ops: SmallVec<[KleeneToken; 1]>, +} + +/// Checks that meta-variables are used correctly in a macro definition. +/// +/// Arguments: +/// - `sess` is used to emit diagnostics and lints +/// - `node_id` is used to emit lints +/// - `span` is used when no spans are available +/// - `lhses` and `rhses` should have the same length and represent the macro definition +pub(super) fn check_meta_variables( + sess: &ParseSess, + node_id: NodeId, + span: Span, + lhses: &[TokenTree], + rhses: &[TokenTree], +) -> bool { + if lhses.len() != rhses.len() { + sess.span_diagnostic.span_bug(span, "length mismatch between LHSes and RHSes") + } + let mut valid = true; + for (lhs, rhs) in lhses.iter().zip(rhses.iter()) { + let mut binders = Binders::default(); + check_binders(sess, node_id, lhs, &Stack::Empty, &mut binders, &Stack::Empty, &mut valid); + check_occurrences(sess, node_id, rhs, &Stack::Empty, &binders, &Stack::Empty, &mut valid); + } + valid +} + +/// Checks `lhs` as part of the LHS of a macro definition, extends `binders` with new binders, and +/// sets `valid` to false in case of errors. +/// +/// Arguments: +/// - `sess` is used to emit diagnostics and lints +/// - `node_id` is used to emit lints +/// - `lhs` is checked as part of a LHS +/// - `macros` is the stack of possible outer macros +/// - `binders` contains the binders of the LHS +/// - `ops` is the stack of Kleene operators from the LHS +/// - `valid` is set in case of errors +fn check_binders( + sess: &ParseSess, + node_id: NodeId, + lhs: &TokenTree, + macros: &Stack<'_, MacroState<'_>>, + binders: &mut Binders, + ops: &Stack<'_, KleeneToken>, + valid: &mut bool, +) { + match *lhs { + TokenTree::Token(..) => {} + // This can only happen when checking a nested macro because this LHS is then in the RHS of + // the outer macro. See ui/macros/macro-of-higher-order.rs where $y:$fragment in the + // LHS of the nested macro (and RHS of the outer macro) is parsed as MetaVar(y) Colon + // MetaVar(fragment) and not as MetaVarDecl(y, fragment). + TokenTree::MetaVar(span, name) => { + if macros.is_empty() { + sess.span_diagnostic.span_bug(span, "unexpected MetaVar in lhs"); + } + // There are 3 possibilities: + if let Some(prev_info) = binders.get(&name) { + // 1. The meta-variable is already bound in the current LHS: This is an error. + let mut span = MultiSpan::from_span(span); + span.push_span_label(prev_info.span, "previous declaration".into()); + buffer_lint(sess, span, node_id, "duplicate matcher binding"); + } else if get_binder_info(macros, binders, name).is_none() { + // 2. The meta-variable is free: This is a binder. + binders.insert(name, BinderInfo { span, ops: ops.into() }); + } else { + // 3. The meta-variable is bound: This is an occurrence. + check_occurrences(sess, node_id, lhs, macros, binders, ops, valid); + } + } + // Similarly, this can only happen when checking a toplevel macro. + TokenTree::MetaVarDecl(span, name, _kind) => { + if !macros.is_empty() { + sess.span_diagnostic.span_bug(span, "unexpected MetaVarDecl in nested lhs"); + } + if let Some(prev_info) = get_binder_info(macros, binders, name) { + // Duplicate binders at the top-level macro definition are errors. The lint is only + // for nested macro definitions. + sess.span_diagnostic + .struct_span_err(span, "duplicate matcher binding") + .span_note(prev_info.span, "previous declaration was here") + .emit(); + *valid = false; + } else { + binders.insert(name, BinderInfo { span, ops: ops.into() }); + } + } + TokenTree::Delimited(_, ref del) => { + for tt in &del.tts { + check_binders(sess, node_id, tt, macros, binders, ops, valid); + } + } + TokenTree::Sequence(_, ref seq) => { + let ops = ops.push(seq.kleene); + for tt in &seq.tts { + check_binders(sess, node_id, tt, macros, binders, &ops, valid); + } + } + } +} + +/// Returns the binder information of a meta-variable. +/// +/// Arguments: +/// - `macros` is the stack of possible outer macros +/// - `binders` contains the current binders +/// - `name` is the name of the meta-variable we are looking for +fn get_binder_info<'a>( + mut macros: &'a Stack<'a, MacroState<'a>>, + binders: &'a Binders, + name: Ident, +) -> Option<&'a BinderInfo> { + binders.get(&name).or_else(|| macros.find_map(|state| state.binders.get(&name))) +} + +/// Checks `rhs` as part of the RHS of a macro definition and sets `valid` to false in case of +/// errors. +/// +/// Arguments: +/// - `sess` is used to emit diagnostics and lints +/// - `node_id` is used to emit lints +/// - `rhs` is checked as part of a RHS +/// - `macros` is the stack of possible outer macros +/// - `binders` contains the binders of the associated LHS +/// - `ops` is the stack of Kleene operators from the RHS +/// - `valid` is set in case of errors +fn check_occurrences( + sess: &ParseSess, + node_id: NodeId, + rhs: &TokenTree, + macros: &Stack<'_, MacroState<'_>>, + binders: &Binders, + ops: &Stack<'_, KleeneToken>, + valid: &mut bool, +) { + match *rhs { + TokenTree::Token(..) => {} + TokenTree::MetaVarDecl(span, _name, _kind) => { + sess.span_diagnostic.span_bug(span, "unexpected MetaVarDecl in rhs") + } + TokenTree::MetaVar(span, name) => { + check_ops_is_prefix(sess, node_id, macros, binders, ops, span, name); + } + TokenTree::Delimited(_, ref del) => { + check_nested_occurrences(sess, node_id, &del.tts, macros, binders, ops, valid); + } + TokenTree::Sequence(_, ref seq) => { + let ops = ops.push(seq.kleene); + check_nested_occurrences(sess, node_id, &seq.tts, macros, binders, &ops, valid); + } + } +} + +/// Represents the processed prefix of a nested macro. +#[derive(Clone, Copy, PartialEq, Eq)] +enum NestedMacroState { + /// Nothing that matches a nested macro definition was processed yet. + Empty, + /// The token `macro_rules` was processed. + MacroRules, + /// The tokens `macro_rules!` were processed. + MacroRulesNot, + /// The tokens `macro_rules!` followed by a name were processed. The name may be either directly + /// an identifier or a meta-variable (that hopefully would be instantiated by an identifier). + MacroRulesNotName, + /// The keyword `macro` was processed. + Macro, + /// The keyword `macro` followed by a name was processed. + MacroName, + /// The keyword `macro` followed by a name and a token delimited by parentheses was processed. + MacroNameParen, +} + +/// Checks `tts` as part of the RHS of a macro definition, tries to recognize nested macro +/// definitions, and sets `valid` to false in case of errors. +/// +/// Arguments: +/// - `sess` is used to emit diagnostics and lints +/// - `node_id` is used to emit lints +/// - `tts` is checked as part of a RHS and may contain macro definitions +/// - `macros` is the stack of possible outer macros +/// - `binders` contains the binders of the associated LHS +/// - `ops` is the stack of Kleene operators from the RHS +/// - `valid` is set in case of errors +fn check_nested_occurrences( + sess: &ParseSess, + node_id: NodeId, + tts: &[TokenTree], + macros: &Stack<'_, MacroState<'_>>, + binders: &Binders, + ops: &Stack<'_, KleeneToken>, + valid: &mut bool, +) { + let mut state = NestedMacroState::Empty; + let nested_macros = macros.push(MacroState { binders, ops: ops.into() }); + let mut nested_binders = Binders::default(); + for tt in tts { + match (state, tt) { + ( + NestedMacroState::Empty, + &TokenTree::Token(Token { kind: TokenKind::Ident(name, false), .. }), + ) => { + if name == sym::macro_rules { + state = NestedMacroState::MacroRules; + } else if name == kw::Macro { + state = NestedMacroState::Macro; + } + } + ( + NestedMacroState::MacroRules, + &TokenTree::Token(Token { kind: TokenKind::Not, .. }), + ) => { + state = NestedMacroState::MacroRulesNot; + } + ( + NestedMacroState::MacroRulesNot, + &TokenTree::Token(Token { kind: TokenKind::Ident(..), .. }), + ) => { + state = NestedMacroState::MacroRulesNotName; + } + (NestedMacroState::MacroRulesNot, &TokenTree::MetaVar(..)) => { + state = NestedMacroState::MacroRulesNotName; + // We check that the meta-variable is correctly used. + check_occurrences(sess, node_id, tt, macros, binders, ops, valid); + } + (NestedMacroState::MacroRulesNotName, &TokenTree::Delimited(_, ref del)) + | (NestedMacroState::MacroName, &TokenTree::Delimited(_, ref del)) + if del.delim == DelimToken::Brace => + { + let legacy = state == NestedMacroState::MacroRulesNotName; + state = NestedMacroState::Empty; + let rest = + check_nested_macro(sess, node_id, legacy, &del.tts, &nested_macros, valid); + // If we did not check the whole macro definition, then check the rest as if outside + // the macro definition. + check_nested_occurrences( + sess, + node_id, + &del.tts[rest..], + macros, + binders, + ops, + valid, + ); + } + ( + NestedMacroState::Macro, + &TokenTree::Token(Token { kind: TokenKind::Ident(..), .. }), + ) => { + state = NestedMacroState::MacroName; + } + (NestedMacroState::Macro, &TokenTree::MetaVar(..)) => { + state = NestedMacroState::MacroName; + // We check that the meta-variable is correctly used. + check_occurrences(sess, node_id, tt, macros, binders, ops, valid); + } + (NestedMacroState::MacroName, &TokenTree::Delimited(_, ref del)) + if del.delim == DelimToken::Paren => + { + state = NestedMacroState::MacroNameParen; + nested_binders = Binders::default(); + check_binders( + sess, + node_id, + tt, + &nested_macros, + &mut nested_binders, + &Stack::Empty, + valid, + ); + } + (NestedMacroState::MacroNameParen, &TokenTree::Delimited(_, ref del)) + if del.delim == DelimToken::Brace => + { + state = NestedMacroState::Empty; + check_occurrences( + sess, + node_id, + tt, + &nested_macros, + &nested_binders, + &Stack::Empty, + valid, + ); + } + (_, ref tt) => { + state = NestedMacroState::Empty; + check_occurrences(sess, node_id, tt, macros, binders, ops, valid); + } + } + } +} + +/// Checks the body of nested macro, returns where the check stopped, and sets `valid` to false in +/// case of errors. +/// +/// The token trees are checked as long as they look like a list of (LHS) => {RHS} token trees. This +/// check is a best-effort to detect a macro definition. It returns the position in `tts` where we +/// stopped checking because we detected we were not in a macro definition anymore. +/// +/// Arguments: +/// - `sess` is used to emit diagnostics and lints +/// - `node_id` is used to emit lints +/// - `legacy` specifies whether the macro is legacy +/// - `tts` is checked as a list of (LHS) => {RHS} +/// - `macros` is the stack of outer macros +/// - `valid` is set in case of errors +fn check_nested_macro( + sess: &ParseSess, + node_id: NodeId, + legacy: bool, + tts: &[TokenTree], + macros: &Stack<'_, MacroState<'_>>, + valid: &mut bool, +) -> usize { + let n = tts.len(); + let mut i = 0; + let separator = if legacy { TokenKind::Semi } else { TokenKind::Comma }; + loop { + // We expect 3 token trees: `(LHS) => {RHS}`. The separator is checked after. + if i + 2 >= n + || !tts[i].is_delimited() + || !tts[i + 1].is_token(&TokenKind::FatArrow) + || !tts[i + 2].is_delimited() + { + break; + } + let lhs = &tts[i]; + let rhs = &tts[i + 2]; + let mut binders = Binders::default(); + check_binders(sess, node_id, lhs, macros, &mut binders, &Stack::Empty, valid); + check_occurrences(sess, node_id, rhs, macros, &binders, &Stack::Empty, valid); + // Since the last semicolon is optional for legacy macros and decl_macro are not terminated, + // we increment our checked position by how many token trees we already checked (the 3 + // above) before checking for the separator. + i += 3; + if i == n || !tts[i].is_token(&separator) { + break; + } + // We increment our checked position for the semicolon. + i += 1; + } + i +} + +/// Checks that a meta-variable occurrence is valid. +/// +/// Arguments: +/// - `sess` is used to emit diagnostics and lints +/// - `node_id` is used to emit lints +/// - `macros` is the stack of possible outer macros +/// - `binders` contains the binders of the associated LHS +/// - `ops` is the stack of Kleene operators from the RHS +/// - `span` is the span of the meta-variable to check +/// - `name` is the name of the meta-variable to check +fn check_ops_is_prefix( + sess: &ParseSess, + node_id: NodeId, + macros: &Stack<'_, MacroState<'_>>, + binders: &Binders, + ops: &Stack<'_, KleeneToken>, + span: Span, + name: Ident, +) { + let macros = macros.push(MacroState { binders, ops: ops.into() }); + // Accumulates the stacks the operators of each state until (and including when) the + // meta-variable is found. The innermost stack is first. + let mut acc: SmallVec<[&SmallVec<[KleeneToken; 1]>; 1]> = SmallVec::new(); + for state in ¯os { + acc.push(&state.ops); + if let Some(binder) = state.binders.get(&name) { + // This variable concatenates the stack of operators from the RHS of the LHS where the + // meta-variable was defined to where it is used (in possibly nested macros). The + // outermost operator is first. + let mut occurrence_ops: SmallVec<[KleeneToken; 2]> = SmallVec::new(); + // We need to iterate from the end to start with outermost stack. + for ops in acc.iter().rev() { + occurrence_ops.extend_from_slice(ops); + } + ops_is_prefix(sess, node_id, span, name, &binder.ops, &occurrence_ops); + return; + } + } + buffer_lint(sess, span.into(), node_id, &format!("unknown macro variable `{}`", name)); +} + +/// Returns whether `binder_ops` is a prefix of `occurrence_ops`. +/// +/// The stack of Kleene operators of a meta-variable occurrence just needs to have the stack of +/// Kleene operators of its binder as a prefix. +/// +/// Consider $i in the following example: +/// +/// ( $( $i:ident = $($j:ident),+ );* ) => { $($( $i += $j; )+)* } +/// +/// It occurs under the Kleene stack ["*", "+"] and is bound under ["*"] only. +/// +/// Arguments: +/// - `sess` is used to emit diagnostics and lints +/// - `node_id` is used to emit lints +/// - `span` is the span of the meta-variable being check +/// - `name` is the name of the meta-variable being check +/// - `binder_ops` is the stack of Kleene operators for the binder +/// - `occurrence_ops` is the stack of Kleene operators for the occurrence +fn ops_is_prefix( + sess: &ParseSess, + node_id: NodeId, + span: Span, + name: Ident, + binder_ops: &[KleeneToken], + occurrence_ops: &[KleeneToken], +) { + for (i, binder) in binder_ops.iter().enumerate() { + if i >= occurrence_ops.len() { + let mut span = MultiSpan::from_span(span); + span.push_span_label(binder.span, "expected repetition".into()); + let message = &format!("variable '{}' is still repeating at this depth", name); + buffer_lint(sess, span, node_id, message); + return; + } + let occurrence = &occurrence_ops[i]; + if occurrence.op != binder.op { + let mut span = MultiSpan::from_span(span); + span.push_span_label(binder.span, "expected repetition".into()); + span.push_span_label(occurrence.span, "conflicting repetition".into()); + let message = "meta-variable repeats with different Kleene operator"; + buffer_lint(sess, span, node_id, message); + return; + } + } +} + +fn buffer_lint(sess: &ParseSess, span: MultiSpan, node_id: NodeId, message: &str) { + sess.buffer_lint(BufferedEarlyLintId::MetaVariableMisuse, span, node_id, message); +} diff --git a/src/libsyntax_expand/mbe/macro_parser.rs b/src/libsyntax_expand/mbe/macro_parser.rs new file mode 100644 index 00000000000..2edb4925512 --- /dev/null +++ b/src/libsyntax_expand/mbe/macro_parser.rs @@ -0,0 +1,944 @@ +//! This is an NFA-based parser, which calls out to the main rust parser for named non-terminals +//! (which it commits to fully when it hits one in a grammar). There's a set of current NFA threads +//! and a set of next ones. Instead of NTs, we have a special case for Kleene star. The big-O, in +//! pathological cases, is worse than traditional use of NFA or Earley parsing, but it's an easier +//! fit for Macro-by-Example-style rules. +//! +//! (In order to prevent the pathological case, we'd need to lazily construct the resulting +//! `NamedMatch`es at the very end. It'd be a pain, and require more memory to keep around old +//! items, but it would also save overhead) +//! +//! We don't say this parser uses the Earley algorithm, because it's unnecessarily inaccurate. +//! The macro parser restricts itself to the features of finite state automata. Earley parsers +//! can be described as an extension of NFAs with completion rules, prediction rules, and recursion. +//! +//! Quick intro to how the parser works: +//! +//! A 'position' is a dot in the middle of a matcher, usually represented as a +//! dot. For example `· a $( a )* a b` is a position, as is `a $( · a )* a b`. +//! +//! The parser walks through the input a character at a time, maintaining a list +//! of threads consistent with the current position in the input string: `cur_items`. +//! +//! As it processes them, it fills up `eof_items` with threads that would be valid if +//! the macro invocation is now over, `bb_items` with threads that are waiting on +//! a Rust non-terminal like `$e:expr`, and `next_items` with threads that are waiting +//! on a particular token. Most of the logic concerns moving the · through the +//! repetitions indicated by Kleene stars. The rules for moving the · without +//! consuming any input are called epsilon transitions. It only advances or calls +//! out to the real Rust parser when no `cur_items` threads remain. +//! +//! Example: +//! +//! ```text, ignore +//! Start parsing a a a a b against [· a $( a )* a b]. +//! +//! Remaining input: a a a a b +//! next: [· a $( a )* a b] +//! +//! - - - Advance over an a. - - - +//! +//! Remaining input: a a a b +//! cur: [a · $( a )* a b] +//! Descend/Skip (first item). +//! next: [a $( · a )* a b] [a $( a )* · a b]. +//! +//! - - - Advance over an a. - - - +//! +//! Remaining input: a a b +//! cur: [a $( a · )* a b] [a $( a )* a · b] +//! Follow epsilon transition: Finish/Repeat (first item) +//! next: [a $( a )* · a b] [a $( · a )* a b] [a $( a )* a · b] +//! +//! - - - Advance over an a. - - - (this looks exactly like the last step) +//! +//! Remaining input: a b +//! cur: [a $( a · )* a b] [a $( a )* a · b] +//! Follow epsilon transition: Finish/Repeat (first item) +//! next: [a $( a )* · a b] [a $( · a )* a b] [a $( a )* a · b] +//! +//! - - - Advance over an a. - - - (this looks exactly like the last step) +//! +//! Remaining input: b +//! cur: [a $( a · )* a b] [a $( a )* a · b] +//! Follow epsilon transition: Finish/Repeat (first item) +//! next: [a $( a )* · a b] [a $( · a )* a b] [a $( a )* a · b] +//! +//! - - - Advance over a b. - - - +//! +//! Remaining input: '' +//! eof: [a $( a )* a b ·] +//! ``` + +crate use NamedMatch::*; +crate use ParseResult::*; +use TokenTreeOrTokenTreeSlice::*; + +use crate::mbe::{self, TokenTree}; + +use syntax::ast::{Ident, Name}; +use syntax::parse::{Directory, PResult}; +use syntax::parse::parser::{Parser, PathStyle}; +use syntax::parse::token::{self, DocComment, Nonterminal, Token}; +use syntax::print::pprust; +use syntax::sess::ParseSess; +use syntax::symbol::{kw, sym, Symbol}; +use syntax::tokenstream::{DelimSpan, TokenStream}; + +use errors::FatalError; +use smallvec::{smallvec, SmallVec}; +use syntax_pos::Span; + +use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::sync::Lrc; +use std::collections::hash_map::Entry::{Occupied, Vacant}; +use std::mem; +use std::ops::{Deref, DerefMut}; + +// To avoid costly uniqueness checks, we require that `MatchSeq` always has a nonempty body. + +/// Either a sequence of token trees or a single one. This is used as the representation of the +/// sequence of tokens that make up a matcher. +#[derive(Clone)] +enum TokenTreeOrTokenTreeSlice<'tt> { + Tt(TokenTree), + TtSeq(&'tt [TokenTree]), +} + +impl<'tt> TokenTreeOrTokenTreeSlice<'tt> { + /// Returns the number of constituent top-level token trees of `self` (top-level in that it + /// will not recursively descend into subtrees). + fn len(&self) -> usize { + match *self { + TtSeq(ref v) => v.len(), + Tt(ref tt) => tt.len(), + } + } + + /// The `index`-th token tree of `self`. + fn get_tt(&self, index: usize) -> TokenTree { + match *self { + TtSeq(ref v) => v[index].clone(), + Tt(ref tt) => tt.get_tt(index), + } + } +} + +/// An unzipping of `TokenTree`s... see the `stack` field of `MatcherPos`. +/// +/// This is used by `inner_parse_loop` to keep track of delimited submatchers that we have +/// descended into. +#[derive(Clone)] +struct MatcherTtFrame<'tt> { + /// The "parent" matcher that we are descending into. + elts: TokenTreeOrTokenTreeSlice<'tt>, + /// The position of the "dot" in `elts` at the time we descended. + idx: usize, +} + +type NamedMatchVec = SmallVec<[NamedMatch; 4]>; + +/// Represents a single "position" (aka "matcher position", aka "item"), as +/// described in the module documentation. +/// +/// Here: +/// +/// - `'root` represents the lifetime of the stack slot that holds the root +/// `MatcherPos`. As described in `MatcherPosHandle`, the root `MatcherPos` +/// structure is stored on the stack, but subsequent instances are put into +/// the heap. +/// - `'tt` represents the lifetime of the token trees that this matcher +/// position refers to. +/// +/// It is important to distinguish these two lifetimes because we have a +/// `SmallVec>` below, and the destructor of +/// that is considered to possibly access the data from its elements (it lacks +/// a `#[may_dangle]` attribute). As a result, the compiler needs to know that +/// all the elements in that `SmallVec` strictly outlive the root stack slot +/// lifetime. By separating `'tt` from `'root`, we can show that. +#[derive(Clone)] +struct MatcherPos<'root, 'tt> { + /// The token or sequence of tokens that make up the matcher + top_elts: TokenTreeOrTokenTreeSlice<'tt>, + + /// The position of the "dot" in this matcher + idx: usize, + + /// The first span of source that the beginning of this matcher corresponds to. In other + /// words, the token in the source whose span is `sp_open` is matched against the first token of + /// the matcher. + sp_open: Span, + + /// For each named metavar in the matcher, we keep track of token trees matched against the + /// metavar by the black box parser. In particular, there may be more than one match per + /// metavar if we are in a repetition (each repetition matches each of the variables). + /// Moreover, matchers and repetitions can be nested; the `matches` field is shared (hence the + /// `Rc`) among all "nested" matchers. `match_lo`, `match_cur`, and `match_hi` keep track of + /// the current position of the `self` matcher position in the shared `matches` list. + /// + /// Also, note that while we are descending into a sequence, matchers are given their own + /// `matches` vector. Only once we reach the end of a full repetition of the sequence do we add + /// all bound matches from the submatcher into the shared top-level `matches` vector. If `sep` + /// and `up` are `Some`, then `matches` is _not_ the shared top-level list. Instead, if one + /// wants the shared `matches`, one should use `up.matches`. + matches: Box<[Lrc]>, + /// The position in `matches` corresponding to the first metavar in this matcher's sequence of + /// token trees. In other words, the first metavar in the first token of `top_elts` corresponds + /// to `matches[match_lo]`. + match_lo: usize, + /// The position in `matches` corresponding to the metavar we are currently trying to match + /// against the source token stream. `match_lo <= match_cur <= match_hi`. + match_cur: usize, + /// Similar to `match_lo` except `match_hi` is the position in `matches` of the _last_ metavar + /// in this matcher. + match_hi: usize, + + // The following fields are used if we are matching a repetition. If we aren't, they should be + // `None`. + + /// The KleeneOp of this sequence if we are in a repetition. + seq_op: Option, + + /// The separator if we are in a repetition. + sep: Option, + + /// The "parent" matcher position if we are in a repetition. That is, the matcher position just + /// before we enter the sequence. + up: Option>, + + /// Specifically used to "unzip" token trees. By "unzip", we mean to unwrap the delimiters from + /// a delimited token tree (e.g., something wrapped in `(` `)`) or to get the contents of a doc + /// comment... + /// + /// When matching against matchers with nested delimited submatchers (e.g., `pat ( pat ( .. ) + /// pat ) pat`), we need to keep track of the matchers we are descending into. This stack does + /// that where the bottom of the stack is the outermost matcher. + /// Also, throughout the comments, this "descent" is often referred to as "unzipping"... + stack: SmallVec<[MatcherTtFrame<'tt>; 1]>, +} + +impl<'root, 'tt> MatcherPos<'root, 'tt> { + /// Adds `m` as a named match for the `idx`-th metavar. + fn push_match(&mut self, idx: usize, m: NamedMatch) { + let matches = Lrc::make_mut(&mut self.matches[idx]); + matches.push(m); + } +} + +// Lots of MatcherPos instances are created at runtime. Allocating them on the +// heap is slow. Furthermore, using SmallVec to allocate them all +// on the stack is also slow, because MatcherPos is quite a large type and +// instances get moved around a lot between vectors, which requires lots of +// slow memcpy calls. +// +// Therefore, the initial MatcherPos is always allocated on the stack, +// subsequent ones (of which there aren't that many) are allocated on the heap, +// and this type is used to encapsulate both cases. +enum MatcherPosHandle<'root, 'tt> { + Ref(&'root mut MatcherPos<'root, 'tt>), + Box(Box>), +} + +impl<'root, 'tt> Clone for MatcherPosHandle<'root, 'tt> { + // This always produces a new Box. + fn clone(&self) -> Self { + MatcherPosHandle::Box(match *self { + MatcherPosHandle::Ref(ref r) => Box::new((**r).clone()), + MatcherPosHandle::Box(ref b) => b.clone(), + }) + } +} + +impl<'root, 'tt> Deref for MatcherPosHandle<'root, 'tt> { + type Target = MatcherPos<'root, 'tt>; + fn deref(&self) -> &Self::Target { + match *self { + MatcherPosHandle::Ref(ref r) => r, + MatcherPosHandle::Box(ref b) => b, + } + } +} + +impl<'root, 'tt> DerefMut for MatcherPosHandle<'root, 'tt> { + fn deref_mut(&mut self) -> &mut MatcherPos<'root, 'tt> { + match *self { + MatcherPosHandle::Ref(ref mut r) => r, + MatcherPosHandle::Box(ref mut b) => b, + } + } +} + +/// Represents the possible results of an attempted parse. +crate enum ParseResult { + /// Parsed successfully. + Success(T), + /// Arm failed to match. If the second parameter is `token::Eof`, it indicates an unexpected + /// end of macro invocation. Otherwise, it indicates that no rules expected the given token. + Failure(Token, &'static str), + /// Fatal error (malformed macro?). Abort compilation. + Error(syntax_pos::Span, String), +} + +/// A `ParseResult` where the `Success` variant contains a mapping of `Ident`s to `NamedMatch`es. +/// This represents the mapping of metavars to the token trees they bind to. +crate type NamedParseResult = ParseResult>; + +/// Count how many metavars are named in the given matcher `ms`. +pub(super) fn count_names(ms: &[TokenTree]) -> usize { + ms.iter().fold(0, |count, elt| { + count + match *elt { + TokenTree::Sequence(_, ref seq) => seq.num_captures, + TokenTree::Delimited(_, ref delim) => count_names(&delim.tts), + TokenTree::MetaVar(..) => 0, + TokenTree::MetaVarDecl(..) => 1, + TokenTree::Token(..) => 0, + } + }) +} + +/// `len` `Vec`s (initially shared and empty) that will store matches of metavars. +fn create_matches(len: usize) -> Box<[Lrc]> { + if len == 0 { + vec![] + } else { + let empty_matches = Lrc::new(SmallVec::new()); + vec![empty_matches; len] + }.into_boxed_slice() +} + +/// Generates the top-level matcher position in which the "dot" is before the first token of the +/// matcher `ms` and we are going to start matching at the span `open` in the source. +fn initial_matcher_pos<'root, 'tt>(ms: &'tt [TokenTree], open: Span) -> MatcherPos<'root, 'tt> { + let match_idx_hi = count_names(ms); + let matches = create_matches(match_idx_hi); + MatcherPos { + // Start with the top level matcher given to us + top_elts: TtSeq(ms), // "elts" is an abbr. for "elements" + // The "dot" is before the first token of the matcher + idx: 0, + // We start matching at the span `open` in the source code + sp_open: open, + + // Initialize `matches` to a bunch of empty `Vec`s -- one for each metavar in `top_elts`. + // `match_lo` for `top_elts` is 0 and `match_hi` is `matches.len()`. `match_cur` is 0 since + // we haven't actually matched anything yet. + matches, + match_lo: 0, + match_cur: 0, + match_hi: match_idx_hi, + + // Haven't descended into any delimiters, so empty stack + stack: smallvec![], + + // Haven't descended into any sequences, so both of these are `None`. + seq_op: None, + sep: None, + up: None, + } +} + +/// `NamedMatch` is a pattern-match result for a single `token::MATCH_NONTERMINAL`: +/// so it is associated with a single ident in a parse, and all +/// `MatchedNonterminal`s in the `NamedMatch` have the same non-terminal type +/// (expr, item, etc). Each leaf in a single `NamedMatch` corresponds to a +/// single `token::MATCH_NONTERMINAL` in the `TokenTree` that produced it. +/// +/// The in-memory structure of a particular `NamedMatch` represents the match +/// that occurred when a particular subset of a matcher was applied to a +/// particular token tree. +/// +/// The width of each `MatchedSeq` in the `NamedMatch`, and the identity of +/// the `MatchedNonterminal`s, will depend on the token tree it was applied +/// to: each `MatchedSeq` corresponds to a single `TTSeq` in the originating +/// token tree. The depth of the `NamedMatch` structure will therefore depend +/// only on the nesting depth of `ast::TTSeq`s in the originating +/// token tree it was derived from. +#[derive(Debug, Clone)] +crate enum NamedMatch { + MatchedSeq(Lrc, DelimSpan), + MatchedNonterminal(Lrc), +} + +/// Takes a sequence of token trees `ms` representing a matcher which successfully matched input +/// and an iterator of items that matched input and produces a `NamedParseResult`. +fn nameize>( + sess: &ParseSess, + ms: &[TokenTree], + mut res: I, +) -> NamedParseResult { + // Recursively descend into each type of matcher (e.g., sequences, delimited, metavars) and make + // sure that each metavar has _exactly one_ binding. If a metavar does not have exactly one + // binding, then there is an error. If it does, then we insert the binding into the + // `NamedParseResult`. + fn n_rec>( + sess: &ParseSess, + m: &TokenTree, + res: &mut I, + ret_val: &mut FxHashMap, + ) -> Result<(), (syntax_pos::Span, String)> { + match *m { + TokenTree::Sequence(_, ref seq) => for next_m in &seq.tts { + n_rec(sess, next_m, res.by_ref(), ret_val)? + }, + TokenTree::Delimited(_, ref delim) => for next_m in &delim.tts { + n_rec(sess, next_m, res.by_ref(), ret_val)?; + }, + TokenTree::MetaVarDecl(span, _, id) if id.name == kw::Invalid => { + if sess.missing_fragment_specifiers.borrow_mut().remove(&span) { + return Err((span, "missing fragment specifier".to_string())); + } + } + TokenTree::MetaVarDecl(sp, bind_name, _) => { + match ret_val.entry(bind_name) { + Vacant(spot) => { + spot.insert(res.next().unwrap()); + } + Occupied(..) => { + return Err((sp, format!("duplicated bind name: {}", bind_name))) + } + } + } + TokenTree::MetaVar(..) | TokenTree::Token(..) => (), + } + + Ok(()) + } + + let mut ret_val = FxHashMap::default(); + for m in ms { + match n_rec(sess, m, res.by_ref(), &mut ret_val) { + Ok(_) => {} + Err((sp, msg)) => return Error(sp, msg), + } + } + + Success(ret_val) +} + +/// Performs a token equality check, ignoring syntax context (that is, an unhygienic comparison) +fn token_name_eq(t1: &Token, t2: &Token) -> bool { + if let (Some((ident1, is_raw1)), Some((ident2, is_raw2))) = (t1.ident(), t2.ident()) { + ident1.name == ident2.name && is_raw1 == is_raw2 + } else if let (Some(ident1), Some(ident2)) = (t1.lifetime(), t2.lifetime()) { + ident1.name == ident2.name + } else { + t1.kind == t2.kind + } +} + +/// Process the matcher positions of `cur_items` until it is empty. In the process, this will +/// produce more items in `next_items`, `eof_items`, and `bb_items`. +/// +/// For more info about the how this happens, see the module-level doc comments and the inline +/// comments of this function. +/// +/// # Parameters +/// +/// - `sess`: the parsing session into which errors are emitted. +/// - `cur_items`: the set of current items to be processed. This should be empty by the end of a +/// successful execution of this function. +/// - `next_items`: the set of newly generated items. These are used to replenish `cur_items` in +/// the function `parse`. +/// - `eof_items`: the set of items that would be valid if this was the EOF. +/// - `bb_items`: the set of items that are waiting for the black-box parser. +/// - `token`: the current token of the parser. +/// - `span`: the `Span` in the source code corresponding to the token trees we are trying to match +/// against the matcher positions in `cur_items`. +/// +/// # Returns +/// +/// A `ParseResult`. Note that matches are kept track of through the items generated. +fn inner_parse_loop<'root, 'tt>( + sess: &ParseSess, + cur_items: &mut SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>, + next_items: &mut Vec>, + eof_items: &mut SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>, + bb_items: &mut SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>, + token: &Token, +) -> ParseResult<()> { + // Pop items from `cur_items` until it is empty. + while let Some(mut item) = cur_items.pop() { + // When unzipped trees end, remove them. This corresponds to backtracking out of a + // delimited submatcher into which we already descended. In backtracking out again, we need + // to advance the "dot" past the delimiters in the outer matcher. + while item.idx >= item.top_elts.len() { + match item.stack.pop() { + Some(MatcherTtFrame { elts, idx }) => { + item.top_elts = elts; + item.idx = idx + 1; + } + None => break, + } + } + + // Get the current position of the "dot" (`idx`) in `item` and the number of token trees in + // the matcher (`len`). + let idx = item.idx; + let len = item.top_elts.len(); + + // If `idx >= len`, then we are at or past the end of the matcher of `item`. + if idx >= len { + // We are repeating iff there is a parent. If the matcher is inside of a repetition, + // then we could be at the end of a sequence or at the beginning of the next + // repetition. + if item.up.is_some() { + // At this point, regardless of whether there is a separator, we should add all + // matches from the complete repetition of the sequence to the shared, top-level + // `matches` list (actually, `up.matches`, which could itself not be the top-level, + // but anyway...). Moreover, we add another item to `cur_items` in which the "dot" + // is at the end of the `up` matcher. This ensures that the "dot" in the `up` + // matcher is also advanced sufficiently. + // + // NOTE: removing the condition `idx == len` allows trailing separators. + if idx == len { + // Get the `up` matcher + let mut new_pos = item.up.clone().unwrap(); + + // Add matches from this repetition to the `matches` of `up` + for idx in item.match_lo..item.match_hi { + let sub = item.matches[idx].clone(); + let span = DelimSpan::from_pair(item.sp_open, token.span); + new_pos.push_match(idx, MatchedSeq(sub, span)); + } + + // Move the "dot" past the repetition in `up` + new_pos.match_cur = item.match_hi; + new_pos.idx += 1; + cur_items.push(new_pos); + } + + // Check if we need a separator. + if idx == len && item.sep.is_some() { + // We have a separator, and it is the current token. We can advance past the + // separator token. + if item.sep + .as_ref() + .map(|sep| token_name_eq(token, sep)) + .unwrap_or(false) + { + item.idx += 1; + next_items.push(item); + } + } + // We don't need a separator. Move the "dot" back to the beginning of the matcher + // and try to match again UNLESS we are only allowed to have _one_ repetition. + else if item.seq_op != Some(mbe::KleeneOp::ZeroOrOne) { + item.match_cur = item.match_lo; + item.idx = 0; + cur_items.push(item); + } + } + // If we are not in a repetition, then being at the end of a matcher means that we have + // reached the potential end of the input. + else { + eof_items.push(item); + } + } + // We are in the middle of a matcher. + else { + // Look at what token in the matcher we are trying to match the current token (`token`) + // against. Depending on that, we may generate new items. + match item.top_elts.get_tt(idx) { + // Need to descend into a sequence + TokenTree::Sequence(sp, seq) => { + // Examine the case where there are 0 matches of this sequence. We are + // implicitly disallowing OneOrMore from having 0 matches here. Thus, that will + // result in a "no rules expected token" error by virtue of this matcher not + // working. + if seq.kleene.op == mbe::KleeneOp::ZeroOrMore + || seq.kleene.op == mbe::KleeneOp::ZeroOrOne + { + let mut new_item = item.clone(); + new_item.match_cur += seq.num_captures; + new_item.idx += 1; + for idx in item.match_cur..item.match_cur + seq.num_captures { + new_item.push_match(idx, MatchedSeq(Lrc::new(smallvec![]), sp)); + } + cur_items.push(new_item); + } + + let matches = create_matches(item.matches.len()); + cur_items.push(MatcherPosHandle::Box(Box::new(MatcherPos { + stack: smallvec![], + sep: seq.separator.clone(), + seq_op: Some(seq.kleene.op), + idx: 0, + matches, + match_lo: item.match_cur, + match_cur: item.match_cur, + match_hi: item.match_cur + seq.num_captures, + up: Some(item), + sp_open: sp.open, + top_elts: Tt(TokenTree::Sequence(sp, seq)), + }))); + } + + // We need to match a metavar (but the identifier is invalid)... this is an error + TokenTree::MetaVarDecl(span, _, id) if id.name == kw::Invalid => { + if sess.missing_fragment_specifiers.borrow_mut().remove(&span) { + return Error(span, "missing fragment specifier".to_string()); + } + } + + // We need to match a metavar with a valid ident... call out to the black-box + // parser by adding an item to `bb_items`. + TokenTree::MetaVarDecl(_, _, id) => { + // Built-in nonterminals never start with these tokens, + // so we can eliminate them from consideration. + if may_begin_with(token, id.name) { + bb_items.push(item); + } + } + + // We need to descend into a delimited submatcher or a doc comment. To do this, we + // push the current matcher onto a stack and push a new item containing the + // submatcher onto `cur_items`. + // + // At the beginning of the loop, if we reach the end of the delimited submatcher, + // we pop the stack to backtrack out of the descent. + seq @ TokenTree::Delimited(..) | + seq @ TokenTree::Token(Token { kind: DocComment(..), .. }) => { + let lower_elts = mem::replace(&mut item.top_elts, Tt(seq)); + let idx = item.idx; + item.stack.push(MatcherTtFrame { + elts: lower_elts, + idx, + }); + item.idx = 0; + cur_items.push(item); + } + + // We just matched a normal token. We can just advance the parser. + TokenTree::Token(t) if token_name_eq(&t, token) => { + item.idx += 1; + next_items.push(item); + } + + // There was another token that was not `token`... This means we can't add any + // rules. NOTE that this is not necessarily an error unless _all_ items in + // `cur_items` end up doing this. There may still be some other matchers that do + // end up working out. + TokenTree::Token(..) | TokenTree::MetaVar(..) => {} + } + } + } + + // Yay a successful parse (so far)! + Success(()) +} + +/// Use the given sequence of token trees (`ms`) as a matcher. Match the given token stream `tts` +/// against it and return the match. +/// +/// # Parameters +/// +/// - `sess`: The session into which errors are emitted +/// - `tts`: The tokenstream we are matching against the pattern `ms` +/// - `ms`: A sequence of token trees representing a pattern against which we are matching +/// - `directory`: Information about the file locations (needed for the black-box parser) +/// - `recurse_into_modules`: Whether or not to recurse into modules (needed for the black-box +/// parser) +pub(super) fn parse( + sess: &ParseSess, + tts: TokenStream, + ms: &[TokenTree], + directory: Option>, + recurse_into_modules: bool, +) -> NamedParseResult { + // Create a parser that can be used for the "black box" parts. + let mut parser = Parser::new( + sess, + tts, + directory, + recurse_into_modules, + true, + syntax::MACRO_ARGUMENTS, + ); + + // A queue of possible matcher positions. We initialize it with the matcher position in which + // the "dot" is before the first token of the first token tree in `ms`. `inner_parse_loop` then + // processes all of these possible matcher positions and produces possible next positions into + // `next_items`. After some post-processing, the contents of `next_items` replenish `cur_items` + // and we start over again. + // + // This MatcherPos instance is allocated on the stack. All others -- and + // there are frequently *no* others! -- are allocated on the heap. + let mut initial = initial_matcher_pos(ms, parser.token.span); + let mut cur_items = smallvec![MatcherPosHandle::Ref(&mut initial)]; + let mut next_items = Vec::new(); + + loop { + // Matcher positions black-box parsed by parser.rs (`parser`) + let mut bb_items = SmallVec::new(); + + // Matcher positions that would be valid if the macro invocation was over now + let mut eof_items = SmallVec::new(); + assert!(next_items.is_empty()); + + // Process `cur_items` until either we have finished the input or we need to get some + // parsing from the black-box parser done. The result is that `next_items` will contain a + // bunch of possible next matcher positions in `next_items`. + match inner_parse_loop( + sess, + &mut cur_items, + &mut next_items, + &mut eof_items, + &mut bb_items, + &parser.token, + ) { + Success(_) => {} + Failure(token, msg) => return Failure(token, msg), + Error(sp, msg) => return Error(sp, msg), + } + + // inner parse loop handled all cur_items, so it's empty + assert!(cur_items.is_empty()); + + // We need to do some post processing after the `inner_parser_loop`. + // + // Error messages here could be improved with links to original rules. + + // If we reached the EOF, check that there is EXACTLY ONE possible matcher. Otherwise, + // either the parse is ambiguous (which should never happen) or there is a syntax error. + if parser.token == token::Eof { + if eof_items.len() == 1 { + let matches = eof_items[0] + .matches + .iter_mut() + .map(|dv| Lrc::make_mut(dv).pop().unwrap()); + return nameize(sess, ms, matches); + } else if eof_items.len() > 1 { + return Error( + parser.token.span, + "ambiguity: multiple successful parses".to_string(), + ); + } else { + return Failure( + Token::new(token::Eof, if parser.token.span.is_dummy() { + parser.token.span + } else { + sess.source_map().next_point(parser.token.span) + }), + "missing tokens in macro arguments", + ); + } + } + // Performance hack: eof_items may share matchers via Rc with other things that we want + // to modify. Dropping eof_items now may drop these refcounts to 1, preventing an + // unnecessary implicit clone later in Rc::make_mut. + drop(eof_items); + + // Another possibility is that we need to call out to parse some rust nonterminal + // (black-box) parser. However, if there is not EXACTLY ONE of these, something is wrong. + if (!bb_items.is_empty() && !next_items.is_empty()) || bb_items.len() > 1 { + let nts = bb_items + .iter() + .map(|item| match item.top_elts.get_tt(item.idx) { + TokenTree::MetaVarDecl(_, bind, name) => format!("{} ('{}')", name, bind), + _ => panic!(), + }) + .collect::>() + .join(" or "); + + return Error( + parser.token.span, + format!( + "local ambiguity: multiple parsing options: {}", + match next_items.len() { + 0 => format!("built-in NTs {}.", nts), + 1 => format!("built-in NTs {} or 1 other option.", nts), + n => format!("built-in NTs {} or {} other options.", nts, n), + } + ), + ); + } + // If there are no possible next positions AND we aren't waiting for the black-box parser, + // then there is a syntax error. + else if bb_items.is_empty() && next_items.is_empty() { + return Failure( + parser.token.take(), + "no rules expected this token in macro call", + ); + } + // Dump all possible `next_items` into `cur_items` for the next iteration. + else if !next_items.is_empty() { + // Now process the next token + cur_items.extend(next_items.drain(..)); + parser.bump(); + } + // Finally, we have the case where we need to call the black-box parser to get some + // nonterminal. + else { + assert_eq!(bb_items.len(), 1); + + let mut item = bb_items.pop().unwrap(); + if let TokenTree::MetaVarDecl(span, _, ident) = item.top_elts.get_tt(item.idx) { + let match_cur = item.match_cur; + item.push_match( + match_cur, + MatchedNonterminal(Lrc::new(parse_nt(&mut parser, span, ident.name))), + ); + item.idx += 1; + item.match_cur += 1; + } else { + unreachable!() + } + cur_items.push(item); + } + + assert!(!cur_items.is_empty()); + } +} + +/// The token is an identifier, but not `_`. +/// We prohibit passing `_` to macros expecting `ident` for now. +fn get_macro_name(token: &Token) -> Option<(Name, bool)> { + match token.kind { + token::Ident(name, is_raw) if name != kw::Underscore => Some((name, is_raw)), + _ => None, + } +} + +/// Checks whether a non-terminal may begin with a particular token. +/// +/// Returning `false` is a *stability guarantee* that such a matcher will *never* begin with that +/// token. Be conservative (return true) if not sure. +fn may_begin_with(token: &Token, name: Name) -> bool { + /// Checks whether the non-terminal may contain a single (non-keyword) identifier. + fn may_be_ident(nt: &token::Nonterminal) -> bool { + match *nt { + token::NtItem(_) | token::NtBlock(_) | token::NtVis(_) => false, + _ => true, + } + } + + match name { + sym::expr => token.can_begin_expr() + // This exception is here for backwards compatibility. + && !token.is_keyword(kw::Let), + sym::ty => token.can_begin_type(), + sym::ident => get_macro_name(token).is_some(), + sym::literal => token.can_begin_literal_or_bool(), + sym::vis => match token.kind { + // The follow-set of :vis + "priv" keyword + interpolated + token::Comma | token::Ident(..) | token::Interpolated(_) => true, + _ => token.can_begin_type(), + }, + sym::block => match token.kind { + token::OpenDelim(token::Brace) => true, + token::Interpolated(ref nt) => match **nt { + token::NtItem(_) + | token::NtPat(_) + | token::NtTy(_) + | token::NtIdent(..) + | token::NtMeta(_) + | token::NtPath(_) + | token::NtVis(_) => false, // none of these may start with '{'. + _ => true, + }, + _ => false, + }, + sym::path | sym::meta => match token.kind { + token::ModSep | token::Ident(..) => true, + token::Interpolated(ref nt) => match **nt { + token::NtPath(_) | token::NtMeta(_) => true, + _ => may_be_ident(&nt), + }, + _ => false, + }, + sym::pat => match token.kind { + token::Ident(..) | // box, ref, mut, and other identifiers (can stricten) + token::OpenDelim(token::Paren) | // tuple pattern + token::OpenDelim(token::Bracket) | // slice pattern + token::BinOp(token::And) | // reference + token::BinOp(token::Minus) | // negative literal + token::AndAnd | // double reference + token::Literal(..) | // literal + token::DotDot | // range pattern (future compat) + token::DotDotDot | // range pattern (future compat) + token::ModSep | // path + token::Lt | // path (UFCS constant) + token::BinOp(token::Shl) => true, // path (double UFCS) + token::Interpolated(ref nt) => may_be_ident(nt), + _ => false, + }, + sym::lifetime => match token.kind { + token::Lifetime(_) => true, + token::Interpolated(ref nt) => match **nt { + token::NtLifetime(_) | token::NtTT(_) => true, + _ => false, + }, + _ => false, + }, + _ => match token.kind { + token::CloseDelim(_) => false, + _ => true, + }, + } +} + +/// A call to the "black-box" parser to parse some Rust non-terminal. +/// +/// # Parameters +/// +/// - `p`: the "black-box" parser to use +/// - `sp`: the `Span` we want to parse +/// - `name`: the name of the metavar _matcher_ we want to match (e.g., `tt`, `ident`, `block`, +/// etc...) +/// +/// # Returns +/// +/// The parsed non-terminal. +fn parse_nt(p: &mut Parser<'_>, sp: Span, name: Symbol) -> Nonterminal { + if name == sym::tt { + return token::NtTT(p.parse_token_tree()); + } + // check at the beginning and the parser checks after each bump + p.process_potential_macro_variable(); + match parse_nt_inner(p, sp, name) { + Ok(nt) => nt, + Err(mut err) => { + err.emit(); + FatalError.raise(); + } + } +} + +fn parse_nt_inner<'a>(p: &mut Parser<'a>, sp: Span, name: Symbol) -> PResult<'a, Nonterminal> { + Ok(match name { + sym::item => match p.parse_item()? { + Some(i) => token::NtItem(i), + None => return Err(p.fatal("expected an item keyword")), + }, + sym::block => token::NtBlock(p.parse_block()?), + sym::stmt => match p.parse_stmt()? { + Some(s) => token::NtStmt(s), + None => return Err(p.fatal("expected a statement")), + }, + sym::pat => token::NtPat(p.parse_pat(None)?), + sym::expr => token::NtExpr(p.parse_expr()?), + sym::literal => token::NtLiteral(p.parse_literal_maybe_minus()?), + sym::ty => token::NtTy(p.parse_ty()?), + // this could be handled like a token, since it is one + sym::ident => if let Some((name, is_raw)) = get_macro_name(&p.token) { + let span = p.token.span; + p.bump(); + token::NtIdent(Ident::new(name, span), is_raw) + } else { + let token_str = pprust::token_to_string(&p.token); + return Err(p.fatal(&format!("expected ident, found {}", &token_str))); + } + sym::path => token::NtPath(p.parse_path(PathStyle::Type)?), + sym::meta => token::NtMeta(p.parse_attr_item()?), + sym::vis => token::NtVis(p.parse_visibility(true)?), + sym::lifetime => if p.check_lifetime() { + token::NtLifetime(p.expect_lifetime().ident) + } else { + let token_str = pprust::token_to_string(&p.token); + return Err(p.fatal(&format!("expected a lifetime, found `{}`", &token_str))); + } + // this is not supposed to happen, since it has been checked + // when compiling the macro. + _ => p.span_bug(sp, "invalid fragment specifier"), + }) +} diff --git a/src/libsyntax_expand/mbe/macro_rules.rs b/src/libsyntax_expand/mbe/macro_rules.rs new file mode 100644 index 00000000000..9a4130b2d8d --- /dev/null +++ b/src/libsyntax_expand/mbe/macro_rules.rs @@ -0,0 +1,1192 @@ +use crate::base::{DummyResult, ExtCtxt, MacResult, TTMacroExpander}; +use crate::base::{SyntaxExtension, SyntaxExtensionKind}; +use crate::expand::{AstFragment, AstFragmentKind, ensure_complete_parse, parse_ast_fragment}; +use crate::mbe; +use crate::mbe::macro_check; +use crate::mbe::macro_parser::parse; +use crate::mbe::macro_parser::{Error, Failure, Success}; +use crate::mbe::macro_parser::{MatchedNonterminal, MatchedSeq, NamedParseResult}; +use crate::mbe::transcribe::transcribe; + +use syntax::ast; +use syntax::attr::{self, TransparencyError}; +use syntax::edition::Edition; +use syntax::feature_gate::Features; +use syntax::parse::parser::Parser; +use syntax::parse::token::TokenKind::*; +use syntax::parse::token::{self, NtTT, Token}; +use syntax::parse::Directory; +use syntax::print::pprust; +use syntax::sess::ParseSess; +use syntax::symbol::{kw, sym, Symbol}; +use syntax::tokenstream::{DelimSpan, TokenStream}; + +use errors::{DiagnosticBuilder, FatalError}; +use log::debug; +use syntax_pos::hygiene::Transparency; +use syntax_pos::Span; + +use rustc_data_structures::fx::FxHashMap; +use std::borrow::Cow; +use std::collections::hash_map::Entry; +use std::slice; + +use errors::Applicability; +use rustc_data_structures::sync::Lrc; + +const VALID_FRAGMENT_NAMES_MSG: &str = "valid fragment specifiers are \ + `ident`, `block`, `stmt`, `expr`, `pat`, `ty`, `lifetime`, \ + `literal`, `path`, `meta`, `tt`, `item` and `vis`"; + +crate struct ParserAnyMacro<'a> { + parser: Parser<'a>, + + /// Span of the expansion site of the macro this parser is for + site_span: Span, + /// The ident of the macro we're parsing + macro_ident: ast::Ident, + arm_span: Span, +} + +crate fn annotate_err_with_kind( + err: &mut DiagnosticBuilder<'_>, + kind: AstFragmentKind, + span: Span, +) { + match kind { + AstFragmentKind::Ty => { + err.span_label(span, "this macro call doesn't expand to a type"); + } + AstFragmentKind::Pat => { + err.span_label(span, "this macro call doesn't expand to a pattern"); + } + _ => {} + }; +} + +impl<'a> ParserAnyMacro<'a> { + crate fn make(mut self: Box>, kind: AstFragmentKind) -> AstFragment { + let ParserAnyMacro { site_span, macro_ident, ref mut parser, arm_span } = *self; + let fragment = panictry!(parse_ast_fragment(parser, kind, true).map_err(|mut e| { + if parser.token == token::Eof && e.message().ends_with(", found ``") { + if !e.span.is_dummy() { + // early end of macro arm (#52866) + e.replace_span_with(parser.sess.source_map().next_point(parser.token.span)); + } + let msg = &e.message[0]; + e.message[0] = ( + format!( + "macro expansion ends with an incomplete expression: {}", + msg.0.replace(", found ``", ""), + ), + msg.1, + ); + } + if e.span.is_dummy() { + // Get around lack of span in error (#30128) + e.replace_span_with(site_span); + if parser.sess.source_map().span_to_filename(arm_span).is_real() { + e.span_label(arm_span, "in this macro arm"); + } + } else if !parser.sess.source_map().span_to_filename(parser.token.span).is_real() { + e.span_label(site_span, "in this macro invocation"); + } + match kind { + AstFragmentKind::Pat if macro_ident.name == sym::vec => { + let mut suggestion = None; + if let Ok(code) = parser.sess.source_map().span_to_snippet(site_span) { + if let Some(bang) = code.find('!') { + suggestion = Some(code[bang + 1..].to_string()); + } + } + if let Some(suggestion) = suggestion { + e.span_suggestion( + site_span, + "use a slice pattern here instead", + suggestion, + Applicability::MachineApplicable, + ); + } else { + e.span_label( + site_span, + "use a slice pattern here instead", + ); + } + e.help("for more information, see https://doc.rust-lang.org/edition-guide/\ + rust-2018/slice-patterns.html"); + } + _ => annotate_err_with_kind(&mut e, kind, site_span), + }; + e + })); + + // We allow semicolons at the end of expressions -- e.g., the semicolon in + // `macro_rules! m { () => { panic!(); } }` isn't parsed by `.parse_expr()`, + // but `m!()` is allowed in expression positions (cf. issue #34706). + if kind == AstFragmentKind::Expr && parser.token == token::Semi { + parser.bump(); + } + + // Make sure we don't have any tokens left to parse so we don't silently drop anything. + let path = ast::Path::from_ident(macro_ident.with_span_pos(site_span)); + ensure_complete_parse(parser, &path, kind.name(), site_span); + fragment + } +} + +struct MacroRulesMacroExpander { + name: ast::Ident, + span: Span, + transparency: Transparency, + lhses: Vec, + rhses: Vec, + valid: bool, +} + +impl TTMacroExpander for MacroRulesMacroExpander { + fn expand<'cx>( + &self, + cx: &'cx mut ExtCtxt<'_>, + sp: Span, + input: TokenStream, + ) -> Box { + if !self.valid { + return DummyResult::any(sp); + } + generic_extension( + cx, sp, self.span, self.name, self.transparency, input, &self.lhses, &self.rhses + ) + } +} + +fn trace_macros_note(cx: &mut ExtCtxt<'_>, sp: Span, message: String) { + let sp = sp.macro_backtrace().last().map(|trace| trace.call_site).unwrap_or(sp); + cx.expansions.entry(sp).or_default().push(message); +} + +/// Given `lhses` and `rhses`, this is the new macro we create +fn generic_extension<'cx>( + cx: &'cx mut ExtCtxt<'_>, + sp: Span, + def_span: Span, + name: ast::Ident, + transparency: Transparency, + arg: TokenStream, + lhses: &[mbe::TokenTree], + rhses: &[mbe::TokenTree], +) -> Box { + if cx.trace_macros() { + let msg = format!("expanding `{}! {{ {} }}`", name, pprust::tts_to_string(arg.clone())); + trace_macros_note(cx, sp, msg); + } + + // Which arm's failure should we report? (the one furthest along) + let mut best_failure: Option<(Token, &str)> = None; + + for (i, lhs) in lhses.iter().enumerate() { + // try each arm's matchers + let lhs_tt = match *lhs { + mbe::TokenTree::Delimited(_, ref delim) => &delim.tts[..], + _ => cx.span_bug(sp, "malformed macro lhs"), + }; + + match parse_tt(cx, lhs_tt, arg.clone()) { + Success(named_matches) => { + let rhs = match rhses[i] { + // ignore delimiters + mbe::TokenTree::Delimited(_, ref delimed) => delimed.tts.clone(), + _ => cx.span_bug(sp, "malformed macro rhs"), + }; + let arm_span = rhses[i].span(); + + let rhs_spans = rhs.iter().map(|t| t.span()).collect::>(); + // rhs has holes ( `$id` and `$(...)` that need filled) + let mut tts = transcribe(cx, &named_matches, rhs, transparency); + + // Replace all the tokens for the corresponding positions in the macro, to maintain + // proper positions in error reporting, while maintaining the macro_backtrace. + if rhs_spans.len() == tts.len() { + tts = tts.map_enumerated(|i, mut tt| { + let mut sp = rhs_spans[i]; + sp = sp.with_ctxt(tt.span().ctxt()); + tt.set_span(sp); + tt + }); + } + + if cx.trace_macros() { + let msg = format!("to `{}`", pprust::tts_to_string(tts.clone())); + trace_macros_note(cx, sp, msg); + } + + let directory = Directory { + path: Cow::from(cx.current_expansion.module.directory.as_path()), + ownership: cx.current_expansion.directory_ownership, + }; + let mut p = Parser::new(cx.parse_sess(), tts, Some(directory), true, false, None); + p.root_module_name = + cx.current_expansion.module.mod_path.last().map(|id| id.as_str().to_string()); + p.last_type_ascription = cx.current_expansion.prior_type_ascription; + + p.process_potential_macro_variable(); + // Let the context choose how to interpret the result. + // Weird, but useful for X-macros. + return Box::new(ParserAnyMacro { + parser: p, + + // Pass along the original expansion site and the name of the macro + // so we can print a useful error message if the parse of the expanded + // macro leaves unparsed tokens. + site_span: sp, + macro_ident: name, + arm_span, + }); + } + Failure(token, msg) => match best_failure { + Some((ref best_token, _)) if best_token.span.lo() >= token.span.lo() => {} + _ => best_failure = Some((token, msg)), + }, + Error(err_sp, ref msg) => cx.span_fatal(err_sp.substitute_dummy(sp), &msg[..]), + } + } + + let (token, label) = best_failure.expect("ran no matchers"); + let span = token.span.substitute_dummy(sp); + let mut err = cx.struct_span_err(span, &parse_failure_msg(&token)); + err.span_label(span, label); + if !def_span.is_dummy() && cx.source_map().span_to_filename(def_span).is_real() { + err.span_label(cx.source_map().def_span(def_span), "when calling this macro"); + } + + // Check whether there's a missing comma in this macro call, like `println!("{}" a);` + if let Some((arg, comma_span)) = arg.add_comma() { + for lhs in lhses { + // try each arm's matchers + let lhs_tt = match *lhs { + mbe::TokenTree::Delimited(_, ref delim) => &delim.tts[..], + _ => continue, + }; + match parse_tt(cx, lhs_tt, arg.clone()) { + Success(_) => { + if comma_span.is_dummy() { + err.note("you might be missing a comma"); + } else { + err.span_suggestion_short( + comma_span, + "missing comma here", + ", ".to_string(), + Applicability::MachineApplicable, + ); + } + } + _ => {} + } + } + } + err.emit(); + cx.trace_macros_diag(); + DummyResult::any(sp) +} + +// Note that macro-by-example's input is also matched against a token tree: +// $( $lhs:tt => $rhs:tt );+ +// +// Holy self-referential! + +/// Converts a macro item into a syntax extension. +pub fn compile_declarative_macro( + sess: &ParseSess, + features: &Features, + def: &ast::Item, + edition: Edition, +) -> SyntaxExtension { + let diag = &sess.span_diagnostic; + let lhs_nm = ast::Ident::new(sym::lhs, def.span); + let rhs_nm = ast::Ident::new(sym::rhs, def.span); + let tt_spec = ast::Ident::new(sym::tt, def.span); + + // Parse the macro_rules! invocation + let body = match def.kind { + ast::ItemKind::MacroDef(ref body) => body, + _ => unreachable!(), + }; + + // The pattern that macro_rules matches. + // The grammar for macro_rules! is: + // $( $lhs:tt => $rhs:tt );+ + // ...quasiquoting this would be nice. + // These spans won't matter, anyways + let argument_gram = vec![ + mbe::TokenTree::Sequence( + DelimSpan::dummy(), + Lrc::new(mbe::SequenceRepetition { + tts: vec![ + mbe::TokenTree::MetaVarDecl(def.span, lhs_nm, tt_spec), + mbe::TokenTree::token(token::FatArrow, def.span), + mbe::TokenTree::MetaVarDecl(def.span, rhs_nm, tt_spec), + ], + separator: Some(Token::new( + if body.legacy { token::Semi } else { token::Comma }, + def.span, + )), + kleene: mbe::KleeneToken::new(mbe::KleeneOp::OneOrMore, def.span), + num_captures: 2, + }), + ), + // to phase into semicolon-termination instead of semicolon-separation + mbe::TokenTree::Sequence( + DelimSpan::dummy(), + Lrc::new(mbe::SequenceRepetition { + tts: vec![mbe::TokenTree::token( + if body.legacy { token::Semi } else { token::Comma }, + def.span, + )], + separator: None, + kleene: mbe::KleeneToken::new(mbe::KleeneOp::ZeroOrMore, def.span), + num_captures: 0, + }), + ), + ]; + + let argument_map = match parse(sess, body.stream(), &argument_gram, None, true) { + Success(m) => m, + Failure(token, msg) => { + let s = parse_failure_msg(&token); + let sp = token.span.substitute_dummy(def.span); + let mut err = sess.span_diagnostic.struct_span_fatal(sp, &s); + err.span_label(sp, msg); + err.emit(); + FatalError.raise(); + } + Error(sp, s) => { + sess.span_diagnostic.span_fatal(sp.substitute_dummy(def.span), &s).raise(); + } + }; + + let mut valid = true; + + // Extract the arguments: + let lhses = match argument_map[&lhs_nm] { + MatchedSeq(ref s, _) => s + .iter() + .map(|m| { + if let MatchedNonterminal(ref nt) = *m { + if let NtTT(ref tt) = **nt { + let tt = mbe::quoted::parse( + tt.clone().into(), + true, + sess, + ) + .pop() + .unwrap(); + valid &= check_lhs_nt_follows(sess, features, &def.attrs, &tt); + return tt; + } + } + sess.span_diagnostic.span_bug(def.span, "wrong-structured lhs") + }) + .collect::>(), + _ => sess.span_diagnostic.span_bug(def.span, "wrong-structured lhs"), + }; + + let rhses = match argument_map[&rhs_nm] { + MatchedSeq(ref s, _) => s + .iter() + .map(|m| { + if let MatchedNonterminal(ref nt) = *m { + if let NtTT(ref tt) = **nt { + return mbe::quoted::parse( + tt.clone().into(), + false, + sess, + ) + .pop() + .unwrap(); + } + } + sess.span_diagnostic.span_bug(def.span, "wrong-structured lhs") + }) + .collect::>(), + _ => sess.span_diagnostic.span_bug(def.span, "wrong-structured rhs"), + }; + + for rhs in &rhses { + valid &= check_rhs(sess, rhs); + } + + // don't abort iteration early, so that errors for multiple lhses can be reported + for lhs in &lhses { + valid &= check_lhs_no_empty_seq(sess, slice::from_ref(lhs)); + } + + // We use CRATE_NODE_ID instead of `def.id` otherwise we may emit buffered lints for a node id + // that is not lint-checked and trigger the "failed to process buffered lint here" bug. + valid &= macro_check::check_meta_variables(sess, ast::CRATE_NODE_ID, def.span, &lhses, &rhses); + + let (transparency, transparency_error) = attr::find_transparency(&def.attrs, body.legacy); + match transparency_error { + Some(TransparencyError::UnknownTransparency(value, span)) => + diag.span_err(span, &format!("unknown macro transparency: `{}`", value)), + Some(TransparencyError::MultipleTransparencyAttrs(old_span, new_span)) => + diag.span_err(vec![old_span, new_span], "multiple macro transparency attributes"), + None => {} + } + + let expander: Box<_> = Box::new(MacroRulesMacroExpander { + name: def.ident, span: def.span, transparency, lhses, rhses, valid + }); + + SyntaxExtension::new( + sess, + SyntaxExtensionKind::LegacyBang(expander), + def.span, + Vec::new(), + edition, + def.ident.name, + &def.attrs, + ) +} + +fn check_lhs_nt_follows( + sess: &ParseSess, + features: &Features, + attrs: &[ast::Attribute], + lhs: &mbe::TokenTree, +) -> bool { + // lhs is going to be like TokenTree::Delimited(...), where the + // entire lhs is those tts. Or, it can be a "bare sequence", not wrapped in parens. + if let mbe::TokenTree::Delimited(_, ref tts) = *lhs { + check_matcher(sess, features, attrs, &tts.tts) + } else { + let msg = "invalid macro matcher; matchers must be contained in balanced delimiters"; + sess.span_diagnostic.span_err(lhs.span(), msg); + false + } + // we don't abort on errors on rejection, the driver will do that for us + // after parsing/expansion. we can report every error in every macro this way. +} + +/// Checks that the lhs contains no repetition which could match an empty token +/// tree, because then the matcher would hang indefinitely. +fn check_lhs_no_empty_seq(sess: &ParseSess, tts: &[mbe::TokenTree]) -> bool { + use mbe::TokenTree; + for tt in tts { + match *tt { + TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => (), + TokenTree::Delimited(_, ref del) => { + if !check_lhs_no_empty_seq(sess, &del.tts) { + return false; + } + } + TokenTree::Sequence(span, ref seq) => { + if seq.separator.is_none() + && seq.tts.iter().all(|seq_tt| match *seq_tt { + TokenTree::MetaVarDecl(_, _, id) => id.name == sym::vis, + TokenTree::Sequence(_, ref sub_seq) => { + sub_seq.kleene.op == mbe::KleeneOp::ZeroOrMore + || sub_seq.kleene.op == mbe::KleeneOp::ZeroOrOne + } + _ => false, + }) + { + let sp = span.entire(); + sess.span_diagnostic.span_err(sp, "repetition matches empty token tree"); + return false; + } + if !check_lhs_no_empty_seq(sess, &seq.tts) { + return false; + } + } + } + } + + true +} + +fn check_rhs(sess: &ParseSess, rhs: &mbe::TokenTree) -> bool { + match *rhs { + mbe::TokenTree::Delimited(..) => return true, + _ => sess.span_diagnostic.span_err(rhs.span(), "macro rhs must be delimited"), + } + false +} + +fn check_matcher( + sess: &ParseSess, + features: &Features, + attrs: &[ast::Attribute], + matcher: &[mbe::TokenTree], +) -> bool { + let first_sets = FirstSets::new(matcher); + let empty_suffix = TokenSet::empty(); + let err = sess.span_diagnostic.err_count(); + check_matcher_core(sess, features, attrs, &first_sets, matcher, &empty_suffix); + err == sess.span_diagnostic.err_count() +} + +// `The FirstSets` for a matcher is a mapping from subsequences in the +// matcher to the FIRST set for that subsequence. +// +// This mapping is partially precomputed via a backwards scan over the +// token trees of the matcher, which provides a mapping from each +// repetition sequence to its *first* set. +// +// (Hypothetically, sequences should be uniquely identifiable via their +// spans, though perhaps that is false, e.g., for macro-generated macros +// that do not try to inject artificial span information. My plan is +// to try to catch such cases ahead of time and not include them in +// the precomputed mapping.) +struct FirstSets { + // this maps each TokenTree::Sequence `$(tt ...) SEP OP` that is uniquely identified by its + // span in the original matcher to the First set for the inner sequence `tt ...`. + // + // If two sequences have the same span in a matcher, then map that + // span to None (invalidating the mapping here and forcing the code to + // use a slow path). + first: FxHashMap>, +} + +impl FirstSets { + fn new(tts: &[mbe::TokenTree]) -> FirstSets { + use mbe::TokenTree; + + let mut sets = FirstSets { first: FxHashMap::default() }; + build_recur(&mut sets, tts); + return sets; + + // walks backward over `tts`, returning the FIRST for `tts` + // and updating `sets` at the same time for all sequence + // substructure we find within `tts`. + fn build_recur(sets: &mut FirstSets, tts: &[TokenTree]) -> TokenSet { + let mut first = TokenSet::empty(); + for tt in tts.iter().rev() { + match *tt { + TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => { + first.replace_with(tt.clone()); + } + TokenTree::Delimited(span, ref delimited) => { + build_recur(sets, &delimited.tts[..]); + first.replace_with(delimited.open_tt(span.open)); + } + TokenTree::Sequence(sp, ref seq_rep) => { + let subfirst = build_recur(sets, &seq_rep.tts[..]); + + match sets.first.entry(sp.entire()) { + Entry::Vacant(vac) => { + vac.insert(Some(subfirst.clone())); + } + Entry::Occupied(mut occ) => { + // if there is already an entry, then a span must have collided. + // This should not happen with typical macro_rules macros, + // but syntax extensions need not maintain distinct spans, + // so distinct syntax trees can be assigned the same span. + // In such a case, the map cannot be trusted; so mark this + // entry as unusable. + occ.insert(None); + } + } + + // If the sequence contents can be empty, then the first + // token could be the separator token itself. + + if let (Some(sep), true) = (&seq_rep.separator, subfirst.maybe_empty) { + first.add_one_maybe(TokenTree::Token(sep.clone())); + } + + // Reverse scan: Sequence comes before `first`. + if subfirst.maybe_empty + || seq_rep.kleene.op == mbe::KleeneOp::ZeroOrMore + || seq_rep.kleene.op == mbe::KleeneOp::ZeroOrOne + { + // If sequence is potentially empty, then + // union them (preserving first emptiness). + first.add_all(&TokenSet { maybe_empty: true, ..subfirst }); + } else { + // Otherwise, sequence guaranteed + // non-empty; replace first. + first = subfirst; + } + } + } + } + + first + } + } + + // walks forward over `tts` until all potential FIRST tokens are + // identified. + fn first(&self, tts: &[mbe::TokenTree]) -> TokenSet { + use mbe::TokenTree; + + let mut first = TokenSet::empty(); + for tt in tts.iter() { + assert!(first.maybe_empty); + match *tt { + TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => { + first.add_one(tt.clone()); + return first; + } + TokenTree::Delimited(span, ref delimited) => { + first.add_one(delimited.open_tt(span.open)); + return first; + } + TokenTree::Sequence(sp, ref seq_rep) => { + let subfirst_owned; + let subfirst = match self.first.get(&sp.entire()) { + Some(&Some(ref subfirst)) => subfirst, + Some(&None) => { + subfirst_owned = self.first(&seq_rep.tts[..]); + &subfirst_owned + } + None => { + panic!("We missed a sequence during FirstSets construction"); + } + }; + + // If the sequence contents can be empty, then the first + // token could be the separator token itself. + if let (Some(sep), true) = (&seq_rep.separator, subfirst.maybe_empty) { + first.add_one_maybe(TokenTree::Token(sep.clone())); + } + + assert!(first.maybe_empty); + first.add_all(subfirst); + if subfirst.maybe_empty + || seq_rep.kleene.op == mbe::KleeneOp::ZeroOrMore + || seq_rep.kleene.op == mbe::KleeneOp::ZeroOrOne + { + // Continue scanning for more first + // tokens, but also make sure we + // restore empty-tracking state. + first.maybe_empty = true; + continue; + } else { + return first; + } + } + } + } + + // we only exit the loop if `tts` was empty or if every + // element of `tts` matches the empty sequence. + assert!(first.maybe_empty); + first + } +} + +// A set of `mbe::TokenTree`s, which may include `TokenTree::Match`s +// (for macro-by-example syntactic variables). It also carries the +// `maybe_empty` flag; that is true if and only if the matcher can +// match an empty token sequence. +// +// The First set is computed on submatchers like `$($a:expr b),* $(c)* d`, +// which has corresponding FIRST = {$a:expr, c, d}. +// Likewise, `$($a:expr b),* $(c)+ d` has FIRST = {$a:expr, c}. +// +// (Notably, we must allow for *-op to occur zero times.) +#[derive(Clone, Debug)] +struct TokenSet { + tokens: Vec, + maybe_empty: bool, +} + +impl TokenSet { + // Returns a set for the empty sequence. + fn empty() -> Self { + TokenSet { tokens: Vec::new(), maybe_empty: true } + } + + // Returns the set `{ tok }` for the single-token (and thus + // non-empty) sequence [tok]. + fn singleton(tok: mbe::TokenTree) -> Self { + TokenSet { tokens: vec![tok], maybe_empty: false } + } + + // Changes self to be the set `{ tok }`. + // Since `tok` is always present, marks self as non-empty. + fn replace_with(&mut self, tok: mbe::TokenTree) { + self.tokens.clear(); + self.tokens.push(tok); + self.maybe_empty = false; + } + + // Changes self to be the empty set `{}`; meant for use when + // the particular token does not matter, but we want to + // record that it occurs. + fn replace_with_irrelevant(&mut self) { + self.tokens.clear(); + self.maybe_empty = false; + } + + // Adds `tok` to the set for `self`, marking sequence as non-empy. + fn add_one(&mut self, tok: mbe::TokenTree) { + if !self.tokens.contains(&tok) { + self.tokens.push(tok); + } + self.maybe_empty = false; + } + + // Adds `tok` to the set for `self`. (Leaves `maybe_empty` flag alone.) + fn add_one_maybe(&mut self, tok: mbe::TokenTree) { + if !self.tokens.contains(&tok) { + self.tokens.push(tok); + } + } + + // Adds all elements of `other` to this. + // + // (Since this is a set, we filter out duplicates.) + // + // If `other` is potentially empty, then preserves the previous + // setting of the empty flag of `self`. If `other` is guaranteed + // non-empty, then `self` is marked non-empty. + fn add_all(&mut self, other: &Self) { + for tok in &other.tokens { + if !self.tokens.contains(tok) { + self.tokens.push(tok.clone()); + } + } + if !other.maybe_empty { + self.maybe_empty = false; + } + } +} + +// Checks that `matcher` is internally consistent and that it +// can legally be followed by a token `N`, for all `N` in `follow`. +// (If `follow` is empty, then it imposes no constraint on +// the `matcher`.) +// +// Returns the set of NT tokens that could possibly come last in +// `matcher`. (If `matcher` matches the empty sequence, then +// `maybe_empty` will be set to true.) +// +// Requires that `first_sets` is pre-computed for `matcher`; +// see `FirstSets::new`. +fn check_matcher_core( + sess: &ParseSess, + features: &Features, + attrs: &[ast::Attribute], + first_sets: &FirstSets, + matcher: &[mbe::TokenTree], + follow: &TokenSet, +) -> TokenSet { + use mbe::TokenTree; + + let mut last = TokenSet::empty(); + + // 2. For each token and suffix [T, SUFFIX] in M: + // ensure that T can be followed by SUFFIX, and if SUFFIX may be empty, + // then ensure T can also be followed by any element of FOLLOW. + 'each_token: for i in 0..matcher.len() { + let token = &matcher[i]; + let suffix = &matcher[i + 1..]; + + let build_suffix_first = || { + let mut s = first_sets.first(suffix); + if s.maybe_empty { + s.add_all(follow); + } + s + }; + + // (we build `suffix_first` on demand below; you can tell + // which cases are supposed to fall through by looking for the + // initialization of this variable.) + let suffix_first; + + // First, update `last` so that it corresponds to the set + // of NT tokens that might end the sequence `... token`. + match *token { + TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => { + let can_be_followed_by_any; + if let Err(bad_frag) = has_legal_fragment_specifier(sess, features, attrs, token) { + let msg = format!("invalid fragment specifier `{}`", bad_frag); + sess.span_diagnostic + .struct_span_err(token.span(), &msg) + .help(VALID_FRAGMENT_NAMES_MSG) + .emit(); + // (This eliminates false positives and duplicates + // from error messages.) + can_be_followed_by_any = true; + } else { + can_be_followed_by_any = token_can_be_followed_by_any(token); + } + + if can_be_followed_by_any { + // don't need to track tokens that work with any, + last.replace_with_irrelevant(); + // ... and don't need to check tokens that can be + // followed by anything against SUFFIX. + continue 'each_token; + } else { + last.replace_with(token.clone()); + suffix_first = build_suffix_first(); + } + } + TokenTree::Delimited(span, ref d) => { + let my_suffix = TokenSet::singleton(d.close_tt(span.close)); + check_matcher_core(sess, features, attrs, first_sets, &d.tts, &my_suffix); + // don't track non NT tokens + last.replace_with_irrelevant(); + + // also, we don't need to check delimited sequences + // against SUFFIX + continue 'each_token; + } + TokenTree::Sequence(_, ref seq_rep) => { + suffix_first = build_suffix_first(); + // The trick here: when we check the interior, we want + // to include the separator (if any) as a potential + // (but not guaranteed) element of FOLLOW. So in that + // case, we make a temp copy of suffix and stuff + // delimiter in there. + // + // FIXME: Should I first scan suffix_first to see if + // delimiter is already in it before I go through the + // work of cloning it? But then again, this way I may + // get a "tighter" span? + let mut new; + let my_suffix = if let Some(sep) = &seq_rep.separator { + new = suffix_first.clone(); + new.add_one_maybe(TokenTree::Token(sep.clone())); + &new + } else { + &suffix_first + }; + + // At this point, `suffix_first` is built, and + // `my_suffix` is some TokenSet that we can use + // for checking the interior of `seq_rep`. + let next = + check_matcher_core(sess, features, attrs, first_sets, &seq_rep.tts, my_suffix); + if next.maybe_empty { + last.add_all(&next); + } else { + last = next; + } + + // the recursive call to check_matcher_core already ran the 'each_last + // check below, so we can just keep going forward here. + continue 'each_token; + } + } + + // (`suffix_first` guaranteed initialized once reaching here.) + + // Now `last` holds the complete set of NT tokens that could + // end the sequence before SUFFIX. Check that every one works with `suffix`. + 'each_last: for token in &last.tokens { + if let TokenTree::MetaVarDecl(_, name, frag_spec) = *token { + for next_token in &suffix_first.tokens { + match is_in_follow(next_token, frag_spec.name) { + IsInFollow::Invalid(msg, help) => { + sess.span_diagnostic + .struct_span_err(next_token.span(), &msg) + .help(help) + .emit(); + // don't bother reporting every source of + // conflict for a particular element of `last`. + continue 'each_last; + } + IsInFollow::Yes => {} + IsInFollow::No(possible) => { + let may_be = if last.tokens.len() == 1 && suffix_first.tokens.len() == 1 + { + "is" + } else { + "may be" + }; + + let sp = next_token.span(); + let mut err = sess.span_diagnostic.struct_span_err( + sp, + &format!( + "`${name}:{frag}` {may_be} followed by `{next}`, which \ + is not allowed for `{frag}` fragments", + name = name, + frag = frag_spec, + next = quoted_tt_to_string(next_token), + may_be = may_be + ), + ); + err.span_label( + sp, + format!("not allowed after `{}` fragments", frag_spec), + ); + let msg = "allowed there are: "; + match possible { + &[] => {} + &[t] => { + err.note(&format!( + "only {} is allowed after `{}` fragments", + t, frag_spec, + )); + } + ts => { + err.note(&format!( + "{}{} or {}", + msg, + ts[..ts.len() - 1] + .iter() + .map(|s| *s) + .collect::>() + .join(", "), + ts[ts.len() - 1], + )); + } + } + err.emit(); + } + } + } + } + } + } + last +} + +fn token_can_be_followed_by_any(tok: &mbe::TokenTree) -> bool { + if let mbe::TokenTree::MetaVarDecl(_, _, frag_spec) = *tok { + frag_can_be_followed_by_any(frag_spec.name) + } else { + // (Non NT's can always be followed by anthing in matchers.) + true + } +} + +/// Returns `true` if a fragment of type `frag` can be followed by any sort of +/// token. We use this (among other things) as a useful approximation +/// for when `frag` can be followed by a repetition like `$(...)*` or +/// `$(...)+`. In general, these can be a bit tricky to reason about, +/// so we adopt a conservative position that says that any fragment +/// specifier which consumes at most one token tree can be followed by +/// a fragment specifier (indeed, these fragments can be followed by +/// ANYTHING without fear of future compatibility hazards). +fn frag_can_be_followed_by_any(frag: Symbol) -> bool { + match frag { + sym::item | // always terminated by `}` or `;` + sym::block | // exactly one token tree + sym::ident | // exactly one token tree + sym::literal | // exactly one token tree + sym::meta | // exactly one token tree + sym::lifetime | // exactly one token tree + sym::tt => // exactly one token tree + true, + + _ => + false, + } +} + +enum IsInFollow { + Yes, + No(&'static [&'static str]), + Invalid(String, &'static str), +} + +/// Returns `true` if `frag` can legally be followed by the token `tok`. For +/// fragments that can consume an unbounded number of tokens, `tok` +/// must be within a well-defined follow set. This is intended to +/// guarantee future compatibility: for example, without this rule, if +/// we expanded `expr` to include a new binary operator, we might +/// break macros that were relying on that binary operator as a +/// separator. +// when changing this do not forget to update doc/book/macros.md! +fn is_in_follow(tok: &mbe::TokenTree, frag: Symbol) -> IsInFollow { + use mbe::TokenTree; + + if let TokenTree::Token(Token { kind: token::CloseDelim(_), .. }) = *tok { + // closing a token tree can never be matched by any fragment; + // iow, we always require that `(` and `)` match, etc. + IsInFollow::Yes + } else { + match frag { + sym::item => { + // since items *must* be followed by either a `;` or a `}`, we can + // accept anything after them + IsInFollow::Yes + } + sym::block => { + // anything can follow block, the braces provide an easy boundary to + // maintain + IsInFollow::Yes + } + sym::stmt | sym::expr => { + const TOKENS: &[&str] = &["`=>`", "`,`", "`;`"]; + match tok { + TokenTree::Token(token) => match token.kind { + FatArrow | Comma | Semi => IsInFollow::Yes, + _ => IsInFollow::No(TOKENS), + }, + _ => IsInFollow::No(TOKENS), + } + } + sym::pat => { + const TOKENS: &[&str] = &["`=>`", "`,`", "`=`", "`|`", "`if`", "`in`"]; + match tok { + TokenTree::Token(token) => match token.kind { + FatArrow | Comma | Eq | BinOp(token::Or) => IsInFollow::Yes, + Ident(name, false) if name == kw::If || name == kw::In => IsInFollow::Yes, + _ => IsInFollow::No(TOKENS), + }, + _ => IsInFollow::No(TOKENS), + } + } + sym::path | sym::ty => { + const TOKENS: &[&str] = &[ + "`{`", "`[`", "`=>`", "`,`", "`>`", "`=`", "`:`", "`;`", "`|`", "`as`", + "`where`", + ]; + match tok { + TokenTree::Token(token) => match token.kind { + OpenDelim(token::DelimToken::Brace) + | OpenDelim(token::DelimToken::Bracket) + | Comma + | FatArrow + | Colon + | Eq + | Gt + | BinOp(token::Shr) + | Semi + | BinOp(token::Or) => IsInFollow::Yes, + Ident(name, false) if name == kw::As || name == kw::Where => { + IsInFollow::Yes + } + _ => IsInFollow::No(TOKENS), + }, + TokenTree::MetaVarDecl(_, _, frag) if frag.name == sym::block => { + IsInFollow::Yes + } + _ => IsInFollow::No(TOKENS), + } + } + sym::ident | sym::lifetime => { + // being a single token, idents and lifetimes are harmless + IsInFollow::Yes + } + sym::literal => { + // literals may be of a single token, or two tokens (negative numbers) + IsInFollow::Yes + } + sym::meta | sym::tt => { + // being either a single token or a delimited sequence, tt is + // harmless + IsInFollow::Yes + } + sym::vis => { + // Explicitly disallow `priv`, on the off chance it comes back. + const TOKENS: &[&str] = &["`,`", "an ident", "a type"]; + match tok { + TokenTree::Token(token) => match token.kind { + Comma => IsInFollow::Yes, + Ident(name, is_raw) if is_raw || name != kw::Priv => IsInFollow::Yes, + _ => { + if token.can_begin_type() { + IsInFollow::Yes + } else { + IsInFollow::No(TOKENS) + } + } + }, + TokenTree::MetaVarDecl(_, _, frag) + if frag.name == sym::ident + || frag.name == sym::ty + || frag.name == sym::path => + { + IsInFollow::Yes + } + _ => IsInFollow::No(TOKENS), + } + } + kw::Invalid => IsInFollow::Yes, + _ => IsInFollow::Invalid( + format!("invalid fragment specifier `{}`", frag), + VALID_FRAGMENT_NAMES_MSG, + ), + } + } +} + +fn has_legal_fragment_specifier( + sess: &ParseSess, + features: &Features, + attrs: &[ast::Attribute], + tok: &mbe::TokenTree, +) -> Result<(), String> { + debug!("has_legal_fragment_specifier({:?})", tok); + if let mbe::TokenTree::MetaVarDecl(_, _, ref frag_spec) = *tok { + let frag_span = tok.span(); + if !is_legal_fragment_specifier(sess, features, attrs, frag_spec.name, frag_span) { + return Err(frag_spec.to_string()); + } + } + Ok(()) +} + +fn is_legal_fragment_specifier( + _sess: &ParseSess, + _features: &Features, + _attrs: &[ast::Attribute], + frag_name: Symbol, + _frag_span: Span, +) -> bool { + /* + * If new fragment specifiers are invented in nightly, `_sess`, + * `_features`, `_attrs`, and `_frag_span` will be useful here + * for checking against feature gates. See past versions of + * this function. + */ + match frag_name { + sym::item + | sym::block + | sym::stmt + | sym::expr + | sym::pat + | sym::lifetime + | sym::path + | sym::ty + | sym::ident + | sym::meta + | sym::tt + | sym::vis + | sym::literal + | kw::Invalid => true, + _ => false, + } +} + +fn quoted_tt_to_string(tt: &mbe::TokenTree) -> String { + match *tt { + mbe::TokenTree::Token(ref token) => pprust::token_to_string(&token), + mbe::TokenTree::MetaVar(_, name) => format!("${}", name), + mbe::TokenTree::MetaVarDecl(_, name, kind) => format!("${}:{}", name, kind), + _ => panic!( + "unexpected mbe::TokenTree::{{Sequence or Delimited}} \ + in follow set checker" + ), + } +} + +/// Use this token tree as a matcher to parse given tts. +fn parse_tt(cx: &ExtCtxt<'_>, mtch: &[mbe::TokenTree], tts: TokenStream) -> NamedParseResult { + // `None` is because we're not interpolating + let directory = Directory { + path: Cow::from(cx.current_expansion.module.directory.as_path()), + ownership: cx.current_expansion.directory_ownership, + }; + parse(cx.parse_sess(), tts, mtch, Some(directory), true) +} + +/// Generates an appropriate parsing failure message. For EOF, this is "unexpected end...". For +/// other tokens, this is "unexpected token...". +fn parse_failure_msg(tok: &Token) -> String { + match tok.kind { + token::Eof => "unexpected end of macro invocation".to_string(), + _ => format!( + "no rules expected the token `{}`", + pprust::token_to_string(tok), + ), + } +} diff --git a/src/libsyntax_expand/mbe/quoted.rs b/src/libsyntax_expand/mbe/quoted.rs new file mode 100644 index 00000000000..cedd59233ad --- /dev/null +++ b/src/libsyntax_expand/mbe/quoted.rs @@ -0,0 +1,264 @@ +use crate::mbe::macro_parser; +use crate::mbe::{TokenTree, KleeneOp, KleeneToken, SequenceRepetition, Delimited}; + +use syntax::ast; +use syntax::parse::token::{self, Token}; +use syntax::print::pprust; +use syntax::sess::ParseSess; +use syntax::symbol::kw; +use syntax::tokenstream; + +use syntax_pos::Span; + +use rustc_data_structures::sync::Lrc; + +/// Takes a `tokenstream::TokenStream` and returns a `Vec`. Specifically, this +/// takes a generic `TokenStream`, such as is used in the rest of the compiler, and returns a +/// collection of `TokenTree` for use in parsing a macro. +/// +/// # Parameters +/// +/// - `input`: a token stream to read from, the contents of which we are parsing. +/// - `expect_matchers`: `parse` can be used to parse either the "patterns" or the "body" of a +/// macro. Both take roughly the same form _except_ that in a pattern, metavars are declared with +/// their "matcher" type. For example `$var:expr` or `$id:ident`. In this example, `expr` and +/// `ident` are "matchers". They are not present in the body of a macro rule -- just in the +/// pattern, so we pass a parameter to indicate whether to expect them or not. +/// - `sess`: the parsing session. Any errors will be emitted to this session. +/// - `features`, `attrs`: language feature flags and attributes so that we know whether to use +/// unstable features or not. +/// - `edition`: which edition are we in. +/// - `macro_node_id`: the NodeId of the macro we are parsing. +/// +/// # Returns +/// +/// A collection of `self::TokenTree`. There may also be some errors emitted to `sess`. +pub(super) fn parse( + input: tokenstream::TokenStream, + expect_matchers: bool, + sess: &ParseSess, +) -> Vec { + // Will contain the final collection of `self::TokenTree` + let mut result = Vec::new(); + + // For each token tree in `input`, parse the token into a `self::TokenTree`, consuming + // additional trees if need be. + let mut trees = input.trees(); + while let Some(tree) = trees.next() { + // Given the parsed tree, if there is a metavar and we are expecting matchers, actually + // parse out the matcher (i.e., in `$id:ident` this would parse the `:` and `ident`). + let tree = parse_tree( + tree, + &mut trees, + expect_matchers, + sess, + ); + match tree { + TokenTree::MetaVar(start_sp, ident) if expect_matchers => { + let span = match trees.next() { + Some(tokenstream::TokenTree::Token(Token { kind: token::Colon, span })) => { + match trees.next() { + Some(tokenstream::TokenTree::Token(token)) => match token.ident() { + Some((kind, _)) => { + let span = token.span.with_lo(start_sp.lo()); + result.push(TokenTree::MetaVarDecl(span, ident, kind)); + continue; + } + _ => token.span, + }, + tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span), + } + } + tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(start_sp), + }; + sess.missing_fragment_specifiers.borrow_mut().insert(span); + result.push(TokenTree::MetaVarDecl(span, ident, ast::Ident::invalid())); + } + + // Not a metavar or no matchers allowed, so just return the tree + _ => result.push(tree), + } + } + result +} + +/// Takes a `tokenstream::TokenTree` and returns a `self::TokenTree`. Specifically, this takes a +/// generic `TokenTree`, such as is used in the rest of the compiler, and returns a `TokenTree` +/// for use in parsing a macro. +/// +/// Converting the given tree may involve reading more tokens. +/// +/// # Parameters +/// +/// - `tree`: the tree we wish to convert. +/// - `trees`: an iterator over trees. We may need to read more tokens from it in order to finish +/// converting `tree` +/// - `expect_matchers`: same as for `parse` (see above). +/// - `sess`: the parsing session. Any errors will be emitted to this session. +/// - `features`, `attrs`: language feature flags and attributes so that we know whether to use +/// unstable features or not. +fn parse_tree( + tree: tokenstream::TokenTree, + trees: &mut impl Iterator, + expect_matchers: bool, + sess: &ParseSess, +) -> TokenTree { + // Depending on what `tree` is, we could be parsing different parts of a macro + match tree { + // `tree` is a `$` token. Look at the next token in `trees` + tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }) => match trees.next() { + // `tree` is followed by a delimited set of token trees. This indicates the beginning + // of a repetition sequence in the macro (e.g. `$(pat)*`). + Some(tokenstream::TokenTree::Delimited(span, delim, tts)) => { + // Must have `(` not `{` or `[` + if delim != token::Paren { + let tok = pprust::token_kind_to_string(&token::OpenDelim(delim)); + let msg = format!("expected `(`, found `{}`", tok); + sess.span_diagnostic.span_err(span.entire(), &msg); + } + // Parse the contents of the sequence itself + let sequence = parse( + tts.into(), + expect_matchers, + sess, + ); + // Get the Kleene operator and optional separator + let (separator, kleene) = parse_sep_and_kleene_op(trees, span.entire(), sess); + // Count the number of captured "names" (i.e., named metavars) + let name_captures = macro_parser::count_names(&sequence); + TokenTree::Sequence( + span, + Lrc::new(SequenceRepetition { + tts: sequence, + separator, + kleene, + num_captures: name_captures, + }), + ) + } + + // `tree` is followed by an `ident`. This could be `$meta_var` or the `$crate` special + // metavariable that names the crate of the invocation. + Some(tokenstream::TokenTree::Token(token)) if token.is_ident() => { + let (ident, is_raw) = token.ident().unwrap(); + let span = ident.span.with_lo(span.lo()); + if ident.name == kw::Crate && !is_raw { + TokenTree::token(token::Ident(kw::DollarCrate, is_raw), span) + } else { + TokenTree::MetaVar(span, ident) + } + } + + // `tree` is followed by a random token. This is an error. + Some(tokenstream::TokenTree::Token(token)) => { + let msg = + format!("expected identifier, found `{}`", pprust::token_to_string(&token),); + sess.span_diagnostic.span_err(token.span, &msg); + TokenTree::MetaVar(token.span, ast::Ident::invalid()) + } + + // There are no more tokens. Just return the `$` we already have. + None => TokenTree::token(token::Dollar, span), + }, + + // `tree` is an arbitrary token. Keep it. + tokenstream::TokenTree::Token(token) => TokenTree::Token(token), + + // `tree` is the beginning of a delimited set of tokens (e.g., `(` or `{`). We need to + // descend into the delimited set and further parse it. + tokenstream::TokenTree::Delimited(span, delim, tts) => TokenTree::Delimited( + span, + Lrc::new(Delimited { + delim, + tts: parse( + tts.into(), + expect_matchers, + sess, + ), + }), + ), + } +} + +/// Takes a token and returns `Some(KleeneOp)` if the token is `+` `*` or `?`. Otherwise, return +/// `None`. +fn kleene_op(token: &Token) -> Option { + match token.kind { + token::BinOp(token::Star) => Some(KleeneOp::ZeroOrMore), + token::BinOp(token::Plus) => Some(KleeneOp::OneOrMore), + token::Question => Some(KleeneOp::ZeroOrOne), + _ => None, + } +} + +/// Parse the next token tree of the input looking for a KleeneOp. Returns +/// +/// - Ok(Ok((op, span))) if the next token tree is a KleeneOp +/// - Ok(Err(tok, span)) if the next token tree is a token but not a KleeneOp +/// - Err(span) if the next token tree is not a token +fn parse_kleene_op( + input: &mut impl Iterator, + span: Span, +) -> Result, Span> { + match input.next() { + Some(tokenstream::TokenTree::Token(token)) => match kleene_op(&token) { + Some(op) => Ok(Ok((op, token.span))), + None => Ok(Err(token)), + }, + tree => Err(tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span)), + } +} + +/// Attempt to parse a single Kleene star, possibly with a separator. +/// +/// For example, in a pattern such as `$(a),*`, `a` is the pattern to be repeated, `,` is the +/// separator, and `*` is the Kleene operator. This function is specifically concerned with parsing +/// the last two tokens of such a pattern: namely, the optional separator and the Kleene operator +/// itself. Note that here we are parsing the _macro_ itself, rather than trying to match some +/// stream of tokens in an invocation of a macro. +/// +/// This function will take some input iterator `input` corresponding to `span` and a parsing +/// session `sess`. If the next one (or possibly two) tokens in `input` correspond to a Kleene +/// operator and separator, then a tuple with `(separator, KleeneOp)` is returned. Otherwise, an +/// error with the appropriate span is emitted to `sess` and a dummy value is returned. +fn parse_sep_and_kleene_op( + input: &mut impl Iterator, + span: Span, + sess: &ParseSess, +) -> (Option, KleeneToken) { + // We basically look at two token trees here, denoted as #1 and #2 below + let span = match parse_kleene_op(input, span) { + // #1 is a `?`, `+`, or `*` KleeneOp + Ok(Ok((op, span))) => return (None, KleeneToken::new(op, span)), + + // #1 is a separator followed by #2, a KleeneOp + Ok(Err(token)) => match parse_kleene_op(input, token.span) { + // #2 is the `?` Kleene op, which does not take a separator (error) + Ok(Ok((KleeneOp::ZeroOrOne, span))) => { + // Error! + sess.span_diagnostic.span_err( + token.span, + "the `?` macro repetition operator does not take a separator", + ); + + // Return a dummy + return (None, KleeneToken::new(KleeneOp::ZeroOrMore, span)); + } + + // #2 is a KleeneOp :D + Ok(Ok((op, span))) => return (Some(token), KleeneToken::new(op, span)), + + // #2 is a random token or not a token at all :( + Ok(Err(Token { span, .. })) | Err(span) => span, + }, + + // #1 is not a token + Err(span) => span, + }; + + // If we ever get to this point, we have experienced an "unexpected token" error + sess.span_diagnostic.span_err(span, "expected one of: `*`, `+`, or `?`"); + + // Return a dummy + (None, KleeneToken::new(KleeneOp::ZeroOrMore, span)) +} diff --git a/src/libsyntax_expand/mbe/transcribe.rs b/src/libsyntax_expand/mbe/transcribe.rs new file mode 100644 index 00000000000..94523bbf91b --- /dev/null +++ b/src/libsyntax_expand/mbe/transcribe.rs @@ -0,0 +1,399 @@ +use crate::base::ExtCtxt; +use crate::mbe; +use crate::mbe::macro_parser::{MatchedNonterminal, MatchedSeq, NamedMatch}; + +use syntax::ast::{Ident, Mac}; +use syntax::mut_visit::{self, MutVisitor}; +use syntax::parse::token::{self, NtTT, Token}; +use syntax::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndJoint}; + +use smallvec::{smallvec, SmallVec}; + +use errors::pluralise; +use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::sync::Lrc; +use syntax_pos::hygiene::{ExpnId, Transparency}; +use syntax_pos::Span; + +use std::mem; + +// A Marker adds the given mark to the syntax context. +struct Marker(ExpnId, Transparency); + +impl MutVisitor for Marker { + fn visit_span(&mut self, span: &mut Span) { + *span = span.apply_mark(self.0, self.1) + } + + fn visit_mac(&mut self, mac: &mut Mac) { + mut_visit::noop_visit_mac(mac, self) + } +} + +impl Marker { + fn visit_delim_span(&mut self, dspan: &mut DelimSpan) { + self.visit_span(&mut dspan.open); + self.visit_span(&mut dspan.close); + } +} + +/// An iterator over the token trees in a delimited token tree (`{ ... }`) or a sequence (`$(...)`). +enum Frame { + Delimited { forest: Lrc, idx: usize, span: DelimSpan }, + Sequence { forest: Lrc, idx: usize, sep: Option }, +} + +impl Frame { + /// Construct a new frame around the delimited set of tokens. + fn new(tts: Vec) -> Frame { + let forest = Lrc::new(mbe::Delimited { delim: token::NoDelim, tts }); + Frame::Delimited { forest, idx: 0, span: DelimSpan::dummy() } + } +} + +impl Iterator for Frame { + type Item = mbe::TokenTree; + + fn next(&mut self) -> Option { + match *self { + Frame::Delimited { ref forest, ref mut idx, .. } => { + *idx += 1; + forest.tts.get(*idx - 1).cloned() + } + Frame::Sequence { ref forest, ref mut idx, .. } => { + *idx += 1; + forest.tts.get(*idx - 1).cloned() + } + } + } +} + +/// This can do Macro-By-Example transcription. +/// - `interp` is a map of meta-variables to the tokens (non-terminals) they matched in the +/// invocation. We are assuming we already know there is a match. +/// - `src` is the RHS of the MBE, that is, the "example" we are filling in. +/// +/// For example, +/// +/// ```rust +/// macro_rules! foo { +/// ($id:ident) => { println!("{}", stringify!($id)); } +/// } +/// +/// foo!(bar); +/// ``` +/// +/// `interp` would contain `$id => bar` and `src` would contain `println!("{}", stringify!($id));`. +/// +/// `transcribe` would return a `TokenStream` containing `println!("{}", stringify!(bar));`. +/// +/// Along the way, we do some additional error checking. +pub(super) fn transcribe( + cx: &ExtCtxt<'_>, + interp: &FxHashMap, + src: Vec, + transparency: Transparency, +) -> TokenStream { + // Nothing for us to transcribe... + if src.is_empty() { + return TokenStream::default(); + } + + // We descend into the RHS (`src`), expanding things as we go. This stack contains the things + // we have yet to expand/are still expanding. We start the stack off with the whole RHS. + let mut stack: SmallVec<[Frame; 1]> = smallvec![Frame::new(src)]; + + // As we descend in the RHS, we will need to be able to match nested sequences of matchers. + // `repeats` keeps track of where we are in matching at each level, with the last element being + // the most deeply nested sequence. This is used as a stack. + let mut repeats = Vec::new(); + + // `result` contains resulting token stream from the TokenTree we just finished processing. At + // the end, this will contain the full result of transcription, but at arbitrary points during + // `transcribe`, `result` will contain subsets of the final result. + // + // Specifically, as we descend into each TokenTree, we will push the existing results onto the + // `result_stack` and clear `results`. We will then produce the results of transcribing the + // TokenTree into `results`. Then, as we unwind back out of the `TokenTree`, we will pop the + // `result_stack` and append `results` too it to produce the new `results` up to that point. + // + // Thus, if we try to pop the `result_stack` and it is empty, we have reached the top-level + // again, and we are done transcribing. + let mut result: Vec = Vec::new(); + let mut result_stack = Vec::new(); + let mut marker = Marker(cx.current_expansion.id, transparency); + + loop { + // Look at the last frame on the stack. + let tree = if let Some(tree) = stack.last_mut().unwrap().next() { + // If it still has a TokenTree we have not looked at yet, use that tree. + tree + } + // The else-case never produces a value for `tree` (it `continue`s or `return`s). + else { + // Otherwise, if we have just reached the end of a sequence and we can keep repeating, + // go back to the beginning of the sequence. + if let Frame::Sequence { idx, sep, .. } = stack.last_mut().unwrap() { + let (repeat_idx, repeat_len) = repeats.last_mut().unwrap(); + *repeat_idx += 1; + if repeat_idx < repeat_len { + *idx = 0; + if let Some(sep) = sep { + result.push(TokenTree::Token(sep.clone()).into()); + } + continue; + } + } + + // We are done with the top of the stack. Pop it. Depending on what it was, we do + // different things. Note that the outermost item must be the delimited, wrapped RHS + // that was passed in originally to `transcribe`. + match stack.pop().unwrap() { + // Done with a sequence. Pop from repeats. + Frame::Sequence { .. } => { + repeats.pop(); + } + + // We are done processing a Delimited. If this is the top-level delimited, we are + // done. Otherwise, we unwind the result_stack to append what we have produced to + // any previous results. + Frame::Delimited { forest, span, .. } => { + if result_stack.is_empty() { + // No results left to compute! We are back at the top-level. + return TokenStream::new(result); + } + + // Step back into the parent Delimited. + let tree = + TokenTree::Delimited(span, forest.delim, TokenStream::new(result).into()); + result = result_stack.pop().unwrap(); + result.push(tree.into()); + } + } + continue; + }; + + // At this point, we know we are in the middle of a TokenTree (the last one on `stack`). + // `tree` contains the next `TokenTree` to be processed. + match tree { + // We are descending into a sequence. We first make sure that the matchers in the RHS + // and the matches in `interp` have the same shape. Otherwise, either the caller or the + // macro writer has made a mistake. + seq @ mbe::TokenTree::Sequence(..) => { + match lockstep_iter_size(&seq, interp, &repeats) { + LockstepIterSize::Unconstrained => { + cx.span_fatal( + seq.span(), /* blame macro writer */ + "attempted to repeat an expression containing no syntax variables \ + matched as repeating at this depth", + ); + } + + LockstepIterSize::Contradiction(ref msg) => { + // FIXME: this really ought to be caught at macro definition time... It + // happens when two meta-variables are used in the same repetition in a + // sequence, but they come from different sequence matchers and repeat + // different amounts. + cx.span_fatal(seq.span(), &msg[..]); + } + + LockstepIterSize::Constraint(len, _) => { + // We do this to avoid an extra clone above. We know that this is a + // sequence already. + let (sp, seq) = if let mbe::TokenTree::Sequence(sp, seq) = seq { + (sp, seq) + } else { + unreachable!() + }; + + // Is the repetition empty? + if len == 0 { + if seq.kleene.op == mbe::KleeneOp::OneOrMore { + // FIXME: this really ought to be caught at macro definition + // time... It happens when the Kleene operator in the matcher and + // the body for the same meta-variable do not match. + cx.span_fatal(sp.entire(), "this must repeat at least once"); + } + } else { + // 0 is the initial counter (we have done 0 repretitions so far). `len` + // is the total number of reptitions we should generate. + repeats.push((0, len)); + + // The first time we encounter the sequence we push it to the stack. It + // then gets reused (see the beginning of the loop) until we are done + // repeating. + stack.push(Frame::Sequence { + idx: 0, + sep: seq.separator.clone(), + forest: seq, + }); + } + } + } + } + + // Replace the meta-var with the matched token tree from the invocation. + mbe::TokenTree::MetaVar(mut sp, mut ident) => { + // Find the matched nonterminal from the macro invocation, and use it to replace + // the meta-var. + if let Some(cur_matched) = lookup_cur_matched(ident, interp, &repeats) { + if let MatchedNonterminal(ref nt) = cur_matched { + // FIXME #2887: why do we apply a mark when matching a token tree meta-var + // (e.g. `$x:tt`), but not when we are matching any other type of token + // tree? + if let NtTT(ref tt) = **nt { + result.push(tt.clone().into()); + } else { + marker.visit_span(&mut sp); + let token = TokenTree::token(token::Interpolated(nt.clone()), sp); + result.push(token.into()); + } + } else { + // We were unable to descend far enough. This is an error. + cx.span_fatal( + sp, /* blame the macro writer */ + &format!("variable '{}' is still repeating at this depth", ident), + ); + } + } else { + // If we aren't able to match the meta-var, we push it back into the result but + // with modified syntax context. (I believe this supports nested macros). + marker.visit_span(&mut sp); + marker.visit_ident(&mut ident); + result.push(TokenTree::token(token::Dollar, sp).into()); + result.push(TokenTree::Token(Token::from_ast_ident(ident)).into()); + } + } + + // If we are entering a new delimiter, we push its contents to the `stack` to be + // processed, and we push all of the currently produced results to the `result_stack`. + // We will produce all of the results of the inside of the `Delimited` and then we will + // jump back out of the Delimited, pop the result_stack and add the new results back to + // the previous results (from outside the Delimited). + mbe::TokenTree::Delimited(mut span, delimited) => { + marker.visit_delim_span(&mut span); + stack.push(Frame::Delimited { forest: delimited, idx: 0, span }); + result_stack.push(mem::take(&mut result)); + } + + // Nothing much to do here. Just push the token to the result, being careful to + // preserve syntax context. + mbe::TokenTree::Token(token) => { + let mut tt = TokenTree::Token(token); + marker.visit_tt(&mut tt); + result.push(tt.into()); + } + + // There should be no meta-var declarations in the invocation of a macro. + mbe::TokenTree::MetaVarDecl(..) => panic!("unexpected `TokenTree::MetaVarDecl"), + } + } +} + +/// Lookup the meta-var named `ident` and return the matched token tree from the invocation using +/// the set of matches `interpolations`. +/// +/// See the definition of `repeats` in the `transcribe` function. `repeats` is used to descend +/// into the right place in nested matchers. If we attempt to descend too far, the macro writer has +/// made a mistake, and we return `None`. +fn lookup_cur_matched<'a>( + ident: Ident, + interpolations: &'a FxHashMap, + repeats: &[(usize, usize)], +) -> Option<&'a NamedMatch> { + interpolations.get(&ident).map(|matched| { + let mut matched = matched; + for &(idx, _) in repeats { + match matched { + MatchedNonterminal(_) => break, + MatchedSeq(ref ads, _) => matched = ads.get(idx).unwrap(), + } + } + + matched + }) +} + +/// An accumulator over a TokenTree to be used with `fold`. During transcription, we need to make +/// sure that the size of each sequence and all of its nested sequences are the same as the sizes +/// of all the matched (nested) sequences in the macro invocation. If they don't match, somebody +/// has made a mistake (either the macro writer or caller). +#[derive(Clone)] +enum LockstepIterSize { + /// No constraints on length of matcher. This is true for any TokenTree variants except a + /// `MetaVar` with an actual `MatchedSeq` (as opposed to a `MatchedNonterminal`). + Unconstrained, + + /// A `MetaVar` with an actual `MatchedSeq`. The length of the match and the name of the + /// meta-var are returned. + Constraint(usize, Ident), + + /// Two `Constraint`s on the same sequence had different lengths. This is an error. + Contradiction(String), +} + +impl LockstepIterSize { + /// Find incompatibilities in matcher/invocation sizes. + /// - `Unconstrained` is compatible with everything. + /// - `Contradiction` is incompatible with everything. + /// - `Constraint(len)` is only compatible with other constraints of the same length. + fn with(self, other: LockstepIterSize) -> LockstepIterSize { + match self { + LockstepIterSize::Unconstrained => other, + LockstepIterSize::Contradiction(_) => self, + LockstepIterSize::Constraint(l_len, ref l_id) => match other { + LockstepIterSize::Unconstrained => self, + LockstepIterSize::Contradiction(_) => other, + LockstepIterSize::Constraint(r_len, _) if l_len == r_len => self, + LockstepIterSize::Constraint(r_len, r_id) => { + let msg = format!( + "meta-variable `{}` repeats {} time{}, but `{}` repeats {} time{}", + l_id, + l_len, + pluralise!(l_len), + r_id, + r_len, + pluralise!(r_len), + ); + LockstepIterSize::Contradiction(msg) + } + }, + } + } +} + +/// Given a `tree`, make sure that all sequences have the same length as the matches for the +/// appropriate meta-vars in `interpolations`. +/// +/// Note that if `repeats` does not match the exact correct depth of a meta-var, +/// `lookup_cur_matched` will return `None`, which is why this still works even in the presnece of +/// multiple nested matcher sequences. +fn lockstep_iter_size( + tree: &mbe::TokenTree, + interpolations: &FxHashMap, + repeats: &[(usize, usize)], +) -> LockstepIterSize { + use mbe::TokenTree; + match *tree { + TokenTree::Delimited(_, ref delimed) => { + delimed.tts.iter().fold(LockstepIterSize::Unconstrained, |size, tt| { + size.with(lockstep_iter_size(tt, interpolations, repeats)) + }) + } + TokenTree::Sequence(_, ref seq) => { + seq.tts.iter().fold(LockstepIterSize::Unconstrained, |size, tt| { + size.with(lockstep_iter_size(tt, interpolations, repeats)) + }) + } + TokenTree::MetaVar(_, name) | TokenTree::MetaVarDecl(_, name, _) => { + match lookup_cur_matched(name, interpolations, repeats) { + Some(matched) => match matched { + MatchedNonterminal(_) => LockstepIterSize::Unconstrained, + MatchedSeq(ref ads, _) => LockstepIterSize::Constraint(ads.len(), name), + }, + _ => LockstepIterSize::Unconstrained, + } + } + TokenTree::Token(..) => LockstepIterSize::Unconstrained, + } +} diff --git a/src/libsyntax_expand/placeholders.rs b/src/libsyntax_expand/placeholders.rs new file mode 100644 index 00000000000..f2c89e14b53 --- /dev/null +++ b/src/libsyntax_expand/placeholders.rs @@ -0,0 +1,350 @@ +use crate::base::ExtCtxt; +use crate::expand::{AstFragment, AstFragmentKind}; + +use syntax::ast::{self, NodeId}; +use syntax::source_map::{DUMMY_SP, dummy_spanned}; +use syntax::tokenstream::TokenStream; +use syntax::mut_visit::*; +use syntax::ptr::P; +use syntax::ThinVec; + +use smallvec::{smallvec, SmallVec}; + +use rustc_data_structures::fx::FxHashMap; + +pub fn placeholder(kind: AstFragmentKind, id: ast::NodeId) -> AstFragment { + fn mac_placeholder() -> ast::Mac { + ast::Mac { + path: ast::Path { span: DUMMY_SP, segments: Vec::new() }, + tts: TokenStream::default().into(), + delim: ast::MacDelimiter::Brace, + span: DUMMY_SP, + prior_type_ascription: None, + } + } + + let ident = ast::Ident::invalid(); + let attrs = Vec::new(); + let generics = ast::Generics::default(); + let vis = dummy_spanned(ast::VisibilityKind::Inherited); + let span = DUMMY_SP; + let expr_placeholder = || P(ast::Expr { + id, span, + attrs: ThinVec::new(), + kind: ast::ExprKind::Mac(mac_placeholder()), + }); + let ty = || P(ast::Ty { + id, + kind: ast::TyKind::Mac(mac_placeholder()), + span, + }); + let pat = || P(ast::Pat { + id, + kind: ast::PatKind::Mac(mac_placeholder()), + span, + }); + + match kind { + AstFragmentKind::Expr => AstFragment::Expr(expr_placeholder()), + AstFragmentKind::OptExpr => AstFragment::OptExpr(Some(expr_placeholder())), + AstFragmentKind::Items => AstFragment::Items(smallvec![P(ast::Item { + id, span, ident, vis, attrs, + kind: ast::ItemKind::Mac(mac_placeholder()), + tokens: None, + })]), + AstFragmentKind::TraitItems => AstFragment::TraitItems(smallvec![ast::TraitItem { + id, span, ident, attrs, generics, + kind: ast::TraitItemKind::Macro(mac_placeholder()), + tokens: None, + }]), + AstFragmentKind::ImplItems => AstFragment::ImplItems(smallvec![ast::ImplItem { + id, span, ident, vis, attrs, generics, + kind: ast::ImplItemKind::Macro(mac_placeholder()), + defaultness: ast::Defaultness::Final, + tokens: None, + }]), + AstFragmentKind::ForeignItems => + AstFragment::ForeignItems(smallvec![ast::ForeignItem { + id, span, ident, vis, attrs, + kind: ast::ForeignItemKind::Macro(mac_placeholder()), + }]), + AstFragmentKind::Pat => AstFragment::Pat(P(ast::Pat { + id, span, kind: ast::PatKind::Mac(mac_placeholder()), + })), + AstFragmentKind::Ty => AstFragment::Ty(P(ast::Ty { + id, span, kind: ast::TyKind::Mac(mac_placeholder()), + })), + AstFragmentKind::Stmts => AstFragment::Stmts(smallvec![{ + let mac = P((mac_placeholder(), ast::MacStmtStyle::Braces, ThinVec::new())); + ast::Stmt { id, span, kind: ast::StmtKind::Mac(mac) } + }]), + AstFragmentKind::Arms => AstFragment::Arms(smallvec![ + ast::Arm { + attrs: Default::default(), + body: expr_placeholder(), + guard: None, + id, + pat: pat(), + span, + is_placeholder: true, + } + ]), + AstFragmentKind::Fields => AstFragment::Fields(smallvec![ + ast::Field { + attrs: Default::default(), + expr: expr_placeholder(), + id, + ident, + is_shorthand: false, + span, + is_placeholder: true, + } + ]), + AstFragmentKind::FieldPats => AstFragment::FieldPats(smallvec![ + ast::FieldPat { + attrs: Default::default(), + id, + ident, + is_shorthand: false, + pat: pat(), + span, + is_placeholder: true, + } + ]), + AstFragmentKind::GenericParams => AstFragment::GenericParams(smallvec![{ + ast::GenericParam { + attrs: Default::default(), + bounds: Default::default(), + id, + ident, + is_placeholder: true, + kind: ast::GenericParamKind::Lifetime, + } + }]), + AstFragmentKind::Params => AstFragment::Params(smallvec![ + ast::Param { + attrs: Default::default(), + id, + pat: pat(), + span, + ty: ty(), + is_placeholder: true, + } + ]), + AstFragmentKind::StructFields => AstFragment::StructFields(smallvec![ + ast::StructField { + attrs: Default::default(), + id, + ident: None, + span, + ty: ty(), + vis, + is_placeholder: true, + } + ]), + AstFragmentKind::Variants => AstFragment::Variants(smallvec![ + ast::Variant { + attrs: Default::default(), + data: ast::VariantData::Struct(Default::default(), false), + disr_expr: None, + id, + ident, + span, + is_placeholder: true, + } + ]) + } +} + +pub struct PlaceholderExpander<'a, 'b> { + expanded_fragments: FxHashMap, + cx: &'a mut ExtCtxt<'b>, + monotonic: bool, +} + +impl<'a, 'b> PlaceholderExpander<'a, 'b> { + pub fn new(cx: &'a mut ExtCtxt<'b>, monotonic: bool) -> Self { + PlaceholderExpander { + cx, + expanded_fragments: FxHashMap::default(), + monotonic, + } + } + + pub fn add(&mut self, id: ast::NodeId, mut fragment: AstFragment, placeholders: Vec) { + fragment.mut_visit_with(self); + if let AstFragment::Items(mut items) = fragment { + for placeholder in placeholders { + match self.remove(placeholder) { + AstFragment::Items(derived_items) => items.extend(derived_items), + _ => unreachable!(), + } + } + fragment = AstFragment::Items(items); + } + self.expanded_fragments.insert(id, fragment); + } + + fn remove(&mut self, id: ast::NodeId) -> AstFragment { + self.expanded_fragments.remove(&id).unwrap() + } +} + +impl<'a, 'b> MutVisitor for PlaceholderExpander<'a, 'b> { + fn flat_map_arm(&mut self, arm: ast::Arm) -> SmallVec<[ast::Arm; 1]> { + if arm.is_placeholder { + self.remove(arm.id).make_arms() + } else { + noop_flat_map_arm(arm, self) + } + } + + fn flat_map_field(&mut self, field: ast::Field) -> SmallVec<[ast::Field; 1]> { + if field.is_placeholder { + self.remove(field.id).make_fields() + } else { + noop_flat_map_field(field, self) + } + } + + fn flat_map_field_pattern(&mut self, fp: ast::FieldPat) -> SmallVec<[ast::FieldPat; 1]> { + if fp.is_placeholder { + self.remove(fp.id).make_field_patterns() + } else { + noop_flat_map_field_pattern(fp, self) + } + } + + fn flat_map_generic_param( + &mut self, + param: ast::GenericParam + ) -> SmallVec<[ast::GenericParam; 1]> + { + if param.is_placeholder { + self.remove(param.id).make_generic_params() + } else { + noop_flat_map_generic_param(param, self) + } + } + + fn flat_map_param(&mut self, p: ast::Param) -> SmallVec<[ast::Param; 1]> { + if p.is_placeholder { + self.remove(p.id).make_params() + } else { + noop_flat_map_param(p, self) + } + } + + fn flat_map_struct_field(&mut self, sf: ast::StructField) -> SmallVec<[ast::StructField; 1]> { + if sf.is_placeholder { + self.remove(sf.id).make_struct_fields() + } else { + noop_flat_map_struct_field(sf, self) + } + } + + fn flat_map_variant(&mut self, variant: ast::Variant) -> SmallVec<[ast::Variant; 1]> { + if variant.is_placeholder { + self.remove(variant.id).make_variants() + } else { + noop_flat_map_variant(variant, self) + } + } + + fn flat_map_item(&mut self, item: P) -> SmallVec<[P; 1]> { + match item.kind { + ast::ItemKind::Mac(_) => return self.remove(item.id).make_items(), + ast::ItemKind::MacroDef(_) => return smallvec![item], + _ => {} + } + + noop_flat_map_item(item, self) + } + + fn flat_map_trait_item(&mut self, item: ast::TraitItem) -> SmallVec<[ast::TraitItem; 1]> { + match item.kind { + ast::TraitItemKind::Macro(_) => self.remove(item.id).make_trait_items(), + _ => noop_flat_map_trait_item(item, self), + } + } + + fn flat_map_impl_item(&mut self, item: ast::ImplItem) -> SmallVec<[ast::ImplItem; 1]> { + match item.kind { + ast::ImplItemKind::Macro(_) => self.remove(item.id).make_impl_items(), + _ => noop_flat_map_impl_item(item, self), + } + } + + fn flat_map_foreign_item(&mut self, item: ast::ForeignItem) -> SmallVec<[ast::ForeignItem; 1]> { + match item.kind { + ast::ForeignItemKind::Macro(_) => self.remove(item.id).make_foreign_items(), + _ => noop_flat_map_foreign_item(item, self), + } + } + + fn visit_expr(&mut self, expr: &mut P) { + match expr.kind { + ast::ExprKind::Mac(_) => *expr = self.remove(expr.id).make_expr(), + _ => noop_visit_expr(expr, self), + } + } + + fn filter_map_expr(&mut self, expr: P) -> Option> { + match expr.kind { + ast::ExprKind::Mac(_) => self.remove(expr.id).make_opt_expr(), + _ => noop_filter_map_expr(expr, self), + } + } + + fn flat_map_stmt(&mut self, stmt: ast::Stmt) -> SmallVec<[ast::Stmt; 1]> { + let (style, mut stmts) = match stmt.kind { + ast::StmtKind::Mac(mac) => (mac.1, self.remove(stmt.id).make_stmts()), + _ => return noop_flat_map_stmt(stmt, self), + }; + + if style == ast::MacStmtStyle::Semicolon { + if let Some(stmt) = stmts.pop() { + stmts.push(stmt.add_trailing_semicolon()); + } + } + + stmts + } + + fn visit_pat(&mut self, pat: &mut P) { + match pat.kind { + ast::PatKind::Mac(_) => *pat = self.remove(pat.id).make_pat(), + _ => noop_visit_pat(pat, self), + } + } + + fn visit_ty(&mut self, ty: &mut P) { + match ty.kind { + ast::TyKind::Mac(_) => *ty = self.remove(ty.id).make_ty(), + _ => noop_visit_ty(ty, self), + } + } + + fn visit_block(&mut self, block: &mut P) { + noop_visit_block(block, self); + + for stmt in block.stmts.iter_mut() { + if self.monotonic { + assert_eq!(stmt.id, ast::DUMMY_NODE_ID); + stmt.id = self.cx.resolver.next_node_id(); + } + } + } + + fn visit_mod(&mut self, module: &mut ast::Mod) { + noop_visit_mod(module, self); + module.items.retain(|item| match item.kind { + ast::ItemKind::Mac(_) if !self.cx.ecfg.keep_macs => false, // remove macro definitions + _ => true, + }); + } + + fn visit_mac(&mut self, _mac: &mut ast::Mac) { + // Do nothing. + } +} diff --git a/src/libsyntax_expand/proc_macro.rs b/src/libsyntax_expand/proc_macro.rs new file mode 100644 index 00000000000..07b618c99a5 --- /dev/null +++ b/src/libsyntax_expand/proc_macro.rs @@ -0,0 +1,215 @@ +use crate::base::{self, *}; +use crate::proc_macro_server; + +use syntax::ast::{self, ItemKind, Attribute, Mac}; +use syntax::attr::{mark_used, mark_known}; +use syntax::errors::{Applicability, FatalError}; +use syntax::parse::{self, token}; +use syntax::symbol::sym; +use syntax::tokenstream::{self, TokenStream}; +use syntax::visit::Visitor; + +use rustc_data_structures::sync::Lrc; +use syntax_pos::{Span, DUMMY_SP}; + +const EXEC_STRATEGY: pm::bridge::server::SameThread = pm::bridge::server::SameThread; + +pub struct BangProcMacro { + pub client: pm::bridge::client::Client< + fn(pm::TokenStream) -> pm::TokenStream, + >, +} + +impl base::ProcMacro for BangProcMacro { + fn expand<'cx>(&self, + ecx: &'cx mut ExtCtxt<'_>, + span: Span, + input: TokenStream) + -> TokenStream { + let server = proc_macro_server::Rustc::new(ecx); + match self.client.run(&EXEC_STRATEGY, server, input) { + Ok(stream) => stream, + Err(e) => { + let msg = "proc macro panicked"; + let mut err = ecx.struct_span_fatal(span, msg); + if let Some(s) = e.as_str() { + err.help(&format!("message: {}", s)); + } + + err.emit(); + FatalError.raise(); + } + } + } +} + +pub struct AttrProcMacro { + pub client: pm::bridge::client::Client pm::TokenStream>, +} + +impl base::AttrProcMacro for AttrProcMacro { + fn expand<'cx>(&self, + ecx: &'cx mut ExtCtxt<'_>, + span: Span, + annotation: TokenStream, + annotated: TokenStream) + -> TokenStream { + let server = proc_macro_server::Rustc::new(ecx); + match self.client.run(&EXEC_STRATEGY, server, annotation, annotated) { + Ok(stream) => stream, + Err(e) => { + let msg = "custom attribute panicked"; + let mut err = ecx.struct_span_fatal(span, msg); + if let Some(s) = e.as_str() { + err.help(&format!("message: {}", s)); + } + + err.emit(); + FatalError.raise(); + } + } + } +} + +pub struct ProcMacroDerive { + pub client: pm::bridge::client::Client pm::TokenStream>, +} + +impl MultiItemModifier for ProcMacroDerive { + fn expand(&self, + ecx: &mut ExtCtxt<'_>, + span: Span, + _meta_item: &ast::MetaItem, + item: Annotatable) + -> Vec { + let item = match item { + Annotatable::Arm(..) | + Annotatable::Field(..) | + Annotatable::FieldPat(..) | + Annotatable::GenericParam(..) | + Annotatable::Param(..) | + Annotatable::StructField(..) | + Annotatable::Variant(..) + => panic!("unexpected annotatable"), + Annotatable::Item(item) => item, + Annotatable::ImplItem(_) | + Annotatable::TraitItem(_) | + Annotatable::ForeignItem(_) | + Annotatable::Stmt(_) | + Annotatable::Expr(_) => { + ecx.span_err(span, "proc-macro derives may only be \ + applied to a struct, enum, or union"); + return Vec::new() + } + }; + match item.kind { + ItemKind::Struct(..) | + ItemKind::Enum(..) | + ItemKind::Union(..) => {}, + _ => { + ecx.span_err(span, "proc-macro derives may only be \ + applied to a struct, enum, or union"); + return Vec::new() + } + } + + let token = token::Interpolated(Lrc::new(token::NtItem(item))); + let input = tokenstream::TokenTree::token(token, DUMMY_SP).into(); + + let server = proc_macro_server::Rustc::new(ecx); + let stream = match self.client.run(&EXEC_STRATEGY, server, input) { + Ok(stream) => stream, + Err(e) => { + let msg = "proc-macro derive panicked"; + let mut err = ecx.struct_span_fatal(span, msg); + if let Some(s) = e.as_str() { + err.help(&format!("message: {}", s)); + } + + err.emit(); + FatalError.raise(); + } + }; + + let error_count_before = ecx.parse_sess.span_diagnostic.err_count(); + let msg = "proc-macro derive produced unparseable tokens"; + + let mut parser = parse::stream_to_parser(ecx.parse_sess, stream, Some("proc-macro derive")); + let mut items = vec![]; + + loop { + match parser.parse_item() { + Ok(None) => break, + Ok(Some(item)) => { + items.push(Annotatable::Item(item)) + } + Err(mut err) => { + // FIXME: handle this better + err.cancel(); + ecx.struct_span_fatal(span, msg).emit(); + FatalError.raise(); + } + } + } + + + // fail if there have been errors emitted + if ecx.parse_sess.span_diagnostic.err_count() > error_count_before { + ecx.struct_span_fatal(span, msg).emit(); + FatalError.raise(); + } + + items + } +} + +crate struct MarkAttrs<'a>(crate &'a [ast::Name]); + +impl<'a> Visitor<'a> for MarkAttrs<'a> { + fn visit_attribute(&mut self, attr: &Attribute) { + if let Some(ident) = attr.ident() { + if self.0.contains(&ident.name) { + mark_used(attr); + mark_known(attr); + } + } + } + + fn visit_mac(&mut self, _mac: &Mac) {} +} + +pub fn is_proc_macro_attr(attr: &Attribute) -> bool { + [sym::proc_macro, sym::proc_macro_attribute, sym::proc_macro_derive] + .iter().any(|kind| attr.check_name(*kind)) +} + +crate fn collect_derives(cx: &mut ExtCtxt<'_>, attrs: &mut Vec) -> Vec { + let mut result = Vec::new(); + attrs.retain(|attr| { + if attr.path != sym::derive { + return true; + } + if !attr.is_meta_item_list() { + cx.struct_span_err(attr.span, "malformed `derive` attribute input") + .span_suggestion( + attr.span, + "missing traits to be derived", + "#[derive(Trait1, Trait2, ...)]".to_owned(), + Applicability::HasPlaceholders, + ).emit(); + return false; + } + + match attr.parse_derive_paths(cx.parse_sess) { + Ok(traits) => { + result.extend(traits); + true + } + Err(mut e) => { + e.emit(); + false + } + } + }); + result +} diff --git a/src/libsyntax_expand/proc_macro_server.rs b/src/libsyntax_expand/proc_macro_server.rs new file mode 100644 index 00000000000..4ce99cfe73b --- /dev/null +++ b/src/libsyntax_expand/proc_macro_server.rs @@ -0,0 +1,713 @@ +use crate::base::ExtCtxt; + +use syntax::ast; +use syntax::parse::{self, token}; +use syntax::parse::lexer::comments; +use syntax::print::pprust; +use syntax::sess::ParseSess; +use syntax::tokenstream::{self, DelimSpan, IsJoint::*, TokenStream, TreeAndJoint}; + +use errors::Diagnostic; +use rustc_data_structures::sync::Lrc; +use syntax_pos::{BytePos, FileName, MultiSpan, Pos, SourceFile, Span}; +use syntax_pos::symbol::{kw, sym, Symbol}; + +use pm::{Delimiter, Level, LineColumn, Spacing}; +use pm::bridge::{server, TokenTree}; +use std::{ascii, panic}; +use std::ops::Bound; + +trait FromInternal { + fn from_internal(x: T) -> Self; +} + +trait ToInternal { + fn to_internal(self) -> T; +} + +impl FromInternal for Delimiter { + fn from_internal(delim: token::DelimToken) -> Delimiter { + match delim { + token::Paren => Delimiter::Parenthesis, + token::Brace => Delimiter::Brace, + token::Bracket => Delimiter::Bracket, + token::NoDelim => Delimiter::None, + } + } +} + +impl ToInternal for Delimiter { + fn to_internal(self) -> token::DelimToken { + match self { + Delimiter::Parenthesis => token::Paren, + Delimiter::Brace => token::Brace, + Delimiter::Bracket => token::Bracket, + Delimiter::None => token::NoDelim, + } + } +} + +impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec)> + for TokenTree +{ + fn from_internal(((tree, is_joint), sess, stack): (TreeAndJoint, &ParseSess, &mut Vec)) + -> Self { + use syntax::parse::token::*; + + let joint = is_joint == Joint; + let Token { kind, span } = match tree { + tokenstream::TokenTree::Delimited(span, delim, tts) => { + let delimiter = Delimiter::from_internal(delim); + return TokenTree::Group(Group { + delimiter, + stream: tts.into(), + span, + }); + } + tokenstream::TokenTree::Token(token) => token, + }; + + macro_rules! tt { + ($ty:ident { $($field:ident $(: $value:expr)*),+ $(,)? }) => ( + TokenTree::$ty(self::$ty { + $($field $(: $value)*,)+ + span, + }) + ); + ($ty:ident::$method:ident($($value:expr),*)) => ( + TokenTree::$ty(self::$ty::$method($($value,)* span)) + ); + } + macro_rules! op { + ($a:expr) => { + tt!(Punct::new($a, joint)) + }; + ($a:expr, $b:expr) => {{ + stack.push(tt!(Punct::new($b, joint))); + tt!(Punct::new($a, true)) + }}; + ($a:expr, $b:expr, $c:expr) => {{ + stack.push(tt!(Punct::new($c, joint))); + stack.push(tt!(Punct::new($b, true))); + tt!(Punct::new($a, true)) + }}; + } + + match kind { + Eq => op!('='), + Lt => op!('<'), + Le => op!('<', '='), + EqEq => op!('=', '='), + Ne => op!('!', '='), + Ge => op!('>', '='), + Gt => op!('>'), + AndAnd => op!('&', '&'), + OrOr => op!('|', '|'), + Not => op!('!'), + Tilde => op!('~'), + BinOp(Plus) => op!('+'), + BinOp(Minus) => op!('-'), + BinOp(Star) => op!('*'), + BinOp(Slash) => op!('/'), + BinOp(Percent) => op!('%'), + BinOp(Caret) => op!('^'), + BinOp(And) => op!('&'), + BinOp(Or) => op!('|'), + BinOp(Shl) => op!('<', '<'), + BinOp(Shr) => op!('>', '>'), + BinOpEq(Plus) => op!('+', '='), + BinOpEq(Minus) => op!('-', '='), + BinOpEq(Star) => op!('*', '='), + BinOpEq(Slash) => op!('/', '='), + BinOpEq(Percent) => op!('%', '='), + BinOpEq(Caret) => op!('^', '='), + BinOpEq(And) => op!('&', '='), + BinOpEq(Or) => op!('|', '='), + BinOpEq(Shl) => op!('<', '<', '='), + BinOpEq(Shr) => op!('>', '>', '='), + At => op!('@'), + Dot => op!('.'), + DotDot => op!('.', '.'), + DotDotDot => op!('.', '.', '.'), + DotDotEq => op!('.', '.', '='), + Comma => op!(','), + Semi => op!(';'), + Colon => op!(':'), + ModSep => op!(':', ':'), + RArrow => op!('-', '>'), + LArrow => op!('<', '-'), + FatArrow => op!('=', '>'), + Pound => op!('#'), + Dollar => op!('$'), + Question => op!('?'), + SingleQuote => op!('\''), + + Ident(name, false) if name == kw::DollarCrate => tt!(Ident::dollar_crate()), + Ident(name, is_raw) => tt!(Ident::new(name, is_raw)), + Lifetime(name) => { + let ident = ast::Ident::new(name, span).without_first_quote(); + stack.push(tt!(Ident::new(ident.name, false))); + tt!(Punct::new('\'', true)) + } + Literal(lit) => tt!(Literal { lit }), + DocComment(c) => { + let style = comments::doc_comment_style(&c.as_str()); + let stripped = comments::strip_doc_comment_decoration(&c.as_str()); + let mut escaped = String::new(); + for ch in stripped.chars() { + escaped.extend(ch.escape_debug()); + } + let stream = vec![ + Ident(sym::doc, false), + Eq, + TokenKind::lit(token::Str, Symbol::intern(&escaped), None), + ] + .into_iter() + .map(|kind| tokenstream::TokenTree::token(kind, span)) + .collect(); + stack.push(TokenTree::Group(Group { + delimiter: Delimiter::Bracket, + stream, + span: DelimSpan::from_single(span), + })); + if style == ast::AttrStyle::Inner { + stack.push(tt!(Punct::new('!', false))); + } + tt!(Punct::new('#', false)) + } + + Interpolated(nt) => { + let stream = parse::nt_to_tokenstream(&nt, sess, span); + TokenTree::Group(Group { + delimiter: Delimiter::None, + stream, + span: DelimSpan::from_single(span), + }) + } + + OpenDelim(..) | CloseDelim(..) => unreachable!(), + Whitespace | Comment | Shebang(..) | Unknown(..) | Eof => unreachable!(), + } + } +} + +impl ToInternal for TokenTree { + fn to_internal(self) -> TokenStream { + use syntax::parse::token::*; + + let (ch, joint, span) = match self { + TokenTree::Punct(Punct { ch, joint, span }) => (ch, joint, span), + TokenTree::Group(Group { + delimiter, + stream, + span, + }) => { + return tokenstream::TokenTree::Delimited( + span, + delimiter.to_internal(), + stream.into(), + ) + .into(); + } + TokenTree::Ident(self::Ident { sym, is_raw, span }) => { + return tokenstream::TokenTree::token(Ident(sym, is_raw), span).into(); + } + TokenTree::Literal(self::Literal { + lit: token::Lit { kind: token::Integer, symbol, suffix }, + span, + }) if symbol.as_str().starts_with("-") => { + let minus = BinOp(BinOpToken::Minus); + let symbol = Symbol::intern(&symbol.as_str()[1..]); + let integer = TokenKind::lit(token::Integer, symbol, suffix); + let a = tokenstream::TokenTree::token(minus, span); + let b = tokenstream::TokenTree::token(integer, span); + return vec![a, b].into_iter().collect(); + } + TokenTree::Literal(self::Literal { + lit: token::Lit { kind: token::Float, symbol, suffix }, + span, + }) if symbol.as_str().starts_with("-") => { + let minus = BinOp(BinOpToken::Minus); + let symbol = Symbol::intern(&symbol.as_str()[1..]); + let float = TokenKind::lit(token::Float, symbol, suffix); + let a = tokenstream::TokenTree::token(minus, span); + let b = tokenstream::TokenTree::token(float, span); + return vec![a, b].into_iter().collect(); + } + TokenTree::Literal(self::Literal { lit, span }) => { + return tokenstream::TokenTree::token(Literal(lit), span).into() + } + }; + + let kind = match ch { + '=' => Eq, + '<' => Lt, + '>' => Gt, + '!' => Not, + '~' => Tilde, + '+' => BinOp(Plus), + '-' => BinOp(Minus), + '*' => BinOp(Star), + '/' => BinOp(Slash), + '%' => BinOp(Percent), + '^' => BinOp(Caret), + '&' => BinOp(And), + '|' => BinOp(Or), + '@' => At, + '.' => Dot, + ',' => Comma, + ';' => Semi, + ':' => Colon, + '#' => Pound, + '$' => Dollar, + '?' => Question, + '\'' => SingleQuote, + _ => unreachable!(), + }; + + let tree = tokenstream::TokenTree::token(kind, span); + TokenStream::new(vec![(tree, if joint { Joint } else { NonJoint })]) + } +} + +impl ToInternal for Level { + fn to_internal(self) -> errors::Level { + match self { + Level::Error => errors::Level::Error, + Level::Warning => errors::Level::Warning, + Level::Note => errors::Level::Note, + Level::Help => errors::Level::Help, + _ => unreachable!("unknown proc_macro::Level variant: {:?}", self), + } + } +} + +#[derive(Clone)] +pub struct TokenStreamIter { + cursor: tokenstream::Cursor, + stack: Vec>, +} + +#[derive(Clone)] +pub struct Group { + delimiter: Delimiter, + stream: TokenStream, + span: DelimSpan, +} + +#[derive(Copy, Clone, PartialEq, Eq, Hash)] +pub struct Punct { + ch: char, + // NB. not using `Spacing` here because it doesn't implement `Hash`. + joint: bool, + span: Span, +} + +impl Punct { + fn new(ch: char, joint: bool, span: Span) -> Punct { + const LEGAL_CHARS: &[char] = &['=', '<', '>', '!', '~', '+', '-', '*', '/', '%', '^', + '&', '|', '@', '.', ',', ';', ':', '#', '$', '?', '\'']; + if !LEGAL_CHARS.contains(&ch) { + panic!("unsupported character `{:?}`", ch) + } + Punct { ch, joint, span } + } +} + +#[derive(Copy, Clone, PartialEq, Eq, Hash)] +pub struct Ident { + sym: Symbol, + is_raw: bool, + span: Span, +} + +impl Ident { + fn is_valid(string: &str) -> bool { + let mut chars = string.chars(); + if let Some(start) = chars.next() { + rustc_lexer::is_id_start(start) && chars.all(rustc_lexer::is_id_continue) + } else { + false + } + } + fn new(sym: Symbol, is_raw: bool, span: Span) -> Ident { + let string = sym.as_str(); + if !Self::is_valid(&string) { + panic!("`{:?}` is not a valid identifier", string) + } + if is_raw && !sym.can_be_raw() { + panic!("`{}` cannot be a raw identifier", string); + } + Ident { sym, is_raw, span } + } + fn dollar_crate(span: Span) -> Ident { + // `$crate` is accepted as an ident only if it comes from the compiler. + Ident { sym: kw::DollarCrate, is_raw: false, span } + } +} + +// FIXME(eddyb) `Literal` should not expose internal `Debug` impls. +#[derive(Clone, Debug)] +pub struct Literal { + lit: token::Lit, + span: Span, +} + +pub(crate) struct Rustc<'a> { + sess: &'a ParseSess, + def_site: Span, + call_site: Span, + mixed_site: Span, +} + +impl<'a> Rustc<'a> { + pub fn new(cx: &'a ExtCtxt<'_>) -> Self { + let expn_data = cx.current_expansion.id.expn_data(); + Rustc { + sess: cx.parse_sess, + def_site: cx.with_def_site_ctxt(expn_data.def_site), + call_site: cx.with_call_site_ctxt(expn_data.call_site), + mixed_site: cx.with_mixed_site_ctxt(expn_data.call_site), + } + } + + fn lit(&mut self, kind: token::LitKind, symbol: Symbol, suffix: Option) -> Literal { + Literal { + lit: token::Lit::new(kind, symbol, suffix), + span: server::Span::call_site(self), + } + } +} + +impl server::Types for Rustc<'_> { + type TokenStream = TokenStream; + type TokenStreamBuilder = tokenstream::TokenStreamBuilder; + type TokenStreamIter = TokenStreamIter; + type Group = Group; + type Punct = Punct; + type Ident = Ident; + type Literal = Literal; + type SourceFile = Lrc; + type MultiSpan = Vec; + type Diagnostic = Diagnostic; + type Span = Span; +} + +impl server::TokenStream for Rustc<'_> { + fn new(&mut self) -> Self::TokenStream { + TokenStream::default() + } + fn is_empty(&mut self, stream: &Self::TokenStream) -> bool { + stream.is_empty() + } + fn from_str(&mut self, src: &str) -> Self::TokenStream { + parse::parse_stream_from_source_str( + FileName::proc_macro_source_code(src), + src.to_string(), + self.sess, + Some(self.call_site), + ) + } + fn to_string(&mut self, stream: &Self::TokenStream) -> String { + pprust::tts_to_string(stream.clone()) + } + fn from_token_tree( + &mut self, + tree: TokenTree, + ) -> Self::TokenStream { + tree.to_internal() + } + fn into_iter(&mut self, stream: Self::TokenStream) -> Self::TokenStreamIter { + TokenStreamIter { + cursor: stream.trees(), + stack: vec![], + } + } +} + +impl server::TokenStreamBuilder for Rustc<'_> { + fn new(&mut self) -> Self::TokenStreamBuilder { + tokenstream::TokenStreamBuilder::new() + } + fn push(&mut self, builder: &mut Self::TokenStreamBuilder, stream: Self::TokenStream) { + builder.push(stream); + } + fn build(&mut self, builder: Self::TokenStreamBuilder) -> Self::TokenStream { + builder.build() + } +} + +impl server::TokenStreamIter for Rustc<'_> { + fn next( + &mut self, + iter: &mut Self::TokenStreamIter, + ) -> Option> { + loop { + let tree = iter.stack.pop().or_else(|| { + let next = iter.cursor.next_with_joint()?; + Some(TokenTree::from_internal((next, self.sess, &mut iter.stack))) + })?; + // HACK: The condition "dummy span + group with empty delimiter" represents an AST + // fragment approximately converted into a token stream. This may happen, for + // example, with inputs to proc macro attributes, including derives. Such "groups" + // need to flattened during iteration over stream's token trees. + // Eventually this needs to be removed in favor of keeping original token trees + // and not doing the roundtrip through AST. + if let TokenTree::Group(ref group) = tree { + if group.delimiter == Delimiter::None && group.span.entire().is_dummy() { + iter.cursor.append(group.stream.clone()); + continue; + } + } + return Some(tree); + } + } +} + +impl server::Group for Rustc<'_> { + fn new(&mut self, delimiter: Delimiter, stream: Self::TokenStream) -> Self::Group { + Group { + delimiter, + stream, + span: DelimSpan::from_single(server::Span::call_site(self)), + } + } + fn delimiter(&mut self, group: &Self::Group) -> Delimiter { + group.delimiter + } + fn stream(&mut self, group: &Self::Group) -> Self::TokenStream { + group.stream.clone() + } + fn span(&mut self, group: &Self::Group) -> Self::Span { + group.span.entire() + } + fn span_open(&mut self, group: &Self::Group) -> Self::Span { + group.span.open + } + fn span_close(&mut self, group: &Self::Group) -> Self::Span { + group.span.close + } + fn set_span(&mut self, group: &mut Self::Group, span: Self::Span) { + group.span = DelimSpan::from_single(span); + } +} + +impl server::Punct for Rustc<'_> { + fn new(&mut self, ch: char, spacing: Spacing) -> Self::Punct { + Punct::new(ch, spacing == Spacing::Joint, server::Span::call_site(self)) + } + fn as_char(&mut self, punct: Self::Punct) -> char { + punct.ch + } + fn spacing(&mut self, punct: Self::Punct) -> Spacing { + if punct.joint { + Spacing::Joint + } else { + Spacing::Alone + } + } + fn span(&mut self, punct: Self::Punct) -> Self::Span { + punct.span + } + fn with_span(&mut self, punct: Self::Punct, span: Self::Span) -> Self::Punct { + Punct { span, ..punct } + } +} + +impl server::Ident for Rustc<'_> { + fn new(&mut self, string: &str, span: Self::Span, is_raw: bool) -> Self::Ident { + Ident::new(Symbol::intern(string), is_raw, span) + } + fn span(&mut self, ident: Self::Ident) -> Self::Span { + ident.span + } + fn with_span(&mut self, ident: Self::Ident, span: Self::Span) -> Self::Ident { + Ident { span, ..ident } + } +} + +impl server::Literal for Rustc<'_> { + // FIXME(eddyb) `Literal` should not expose internal `Debug` impls. + fn debug(&mut self, literal: &Self::Literal) -> String { + format!("{:?}", literal) + } + fn integer(&mut self, n: &str) -> Self::Literal { + self.lit(token::Integer, Symbol::intern(n), None) + } + fn typed_integer(&mut self, n: &str, kind: &str) -> Self::Literal { + self.lit(token::Integer, Symbol::intern(n), Some(Symbol::intern(kind))) + } + fn float(&mut self, n: &str) -> Self::Literal { + self.lit(token::Float, Symbol::intern(n), None) + } + fn f32(&mut self, n: &str) -> Self::Literal { + self.lit(token::Float, Symbol::intern(n), Some(sym::f32)) + } + fn f64(&mut self, n: &str) -> Self::Literal { + self.lit(token::Float, Symbol::intern(n), Some(sym::f64)) + } + fn string(&mut self, string: &str) -> Self::Literal { + let mut escaped = String::new(); + for ch in string.chars() { + escaped.extend(ch.escape_debug()); + } + self.lit(token::Str, Symbol::intern(&escaped), None) + } + fn character(&mut self, ch: char) -> Self::Literal { + let mut escaped = String::new(); + escaped.extend(ch.escape_unicode()); + self.lit(token::Char, Symbol::intern(&escaped), None) + } + fn byte_string(&mut self, bytes: &[u8]) -> Self::Literal { + let string = bytes + .iter() + .cloned() + .flat_map(ascii::escape_default) + .map(Into::::into) + .collect::(); + self.lit(token::ByteStr, Symbol::intern(&string), None) + } + fn span(&mut self, literal: &Self::Literal) -> Self::Span { + literal.span + } + fn set_span(&mut self, literal: &mut Self::Literal, span: Self::Span) { + literal.span = span; + } + fn subspan( + &mut self, + literal: &Self::Literal, + start: Bound, + end: Bound, + ) -> Option { + let span = literal.span; + let length = span.hi().to_usize() - span.lo().to_usize(); + + let start = match start { + Bound::Included(lo) => lo, + Bound::Excluded(lo) => lo + 1, + Bound::Unbounded => 0, + }; + + let end = match end { + Bound::Included(hi) => hi + 1, + Bound::Excluded(hi) => hi, + Bound::Unbounded => length, + }; + + // Bounds check the values, preventing addition overflow and OOB spans. + if start > u32::max_value() as usize + || end > u32::max_value() as usize + || (u32::max_value() - start as u32) < span.lo().to_u32() + || (u32::max_value() - end as u32) < span.lo().to_u32() + || start >= end + || end > length + { + return None; + } + + let new_lo = span.lo() + BytePos::from_usize(start); + let new_hi = span.lo() + BytePos::from_usize(end); + Some(span.with_lo(new_lo).with_hi(new_hi)) + } +} + +impl server::SourceFile for Rustc<'_> { + fn eq(&mut self, file1: &Self::SourceFile, file2: &Self::SourceFile) -> bool { + Lrc::ptr_eq(file1, file2) + } + fn path(&mut self, file: &Self::SourceFile) -> String { + match file.name { + FileName::Real(ref path) => path + .to_str() + .expect("non-UTF8 file path in `proc_macro::SourceFile::path`") + .to_string(), + _ => file.name.to_string(), + } + } + fn is_real(&mut self, file: &Self::SourceFile) -> bool { + file.is_real_file() + } +} + +impl server::MultiSpan for Rustc<'_> { + fn new(&mut self) -> Self::MultiSpan { + vec![] + } + fn push(&mut self, spans: &mut Self::MultiSpan, span: Self::Span) { + spans.push(span) + } +} + +impl server::Diagnostic for Rustc<'_> { + fn new(&mut self, level: Level, msg: &str, spans: Self::MultiSpan) -> Self::Diagnostic { + let mut diag = Diagnostic::new(level.to_internal(), msg); + diag.set_span(MultiSpan::from_spans(spans)); + diag + } + fn sub( + &mut self, + diag: &mut Self::Diagnostic, + level: Level, + msg: &str, + spans: Self::MultiSpan, + ) { + diag.sub(level.to_internal(), msg, MultiSpan::from_spans(spans), None); + } + fn emit(&mut self, diag: Self::Diagnostic) { + self.sess.span_diagnostic.emit_diagnostic(&diag); + } +} + +impl server::Span for Rustc<'_> { + fn debug(&mut self, span: Self::Span) -> String { + format!("{:?} bytes({}..{})", span.ctxt(), span.lo().0, span.hi().0) + } + fn def_site(&mut self) -> Self::Span { + self.def_site + } + fn call_site(&mut self) -> Self::Span { + self.call_site + } + fn mixed_site(&mut self) -> Self::Span { + self.mixed_site + } + fn source_file(&mut self, span: Self::Span) -> Self::SourceFile { + self.sess.source_map().lookup_char_pos(span.lo()).file + } + fn parent(&mut self, span: Self::Span) -> Option { + span.parent() + } + fn source(&mut self, span: Self::Span) -> Self::Span { + span.source_callsite() + } + fn start(&mut self, span: Self::Span) -> LineColumn { + let loc = self.sess.source_map().lookup_char_pos(span.lo()); + LineColumn { + line: loc.line, + column: loc.col.to_usize(), + } + } + fn end(&mut self, span: Self::Span) -> LineColumn { + let loc = self.sess.source_map().lookup_char_pos(span.hi()); + LineColumn { + line: loc.line, + column: loc.col.to_usize(), + } + } + fn join(&mut self, first: Self::Span, second: Self::Span) -> Option { + let self_loc = self.sess.source_map().lookup_char_pos(first.lo()); + let other_loc = self.sess.source_map().lookup_char_pos(second.lo()); + + if self_loc.file.name != other_loc.file.name { + return None; + } + + Some(first.to(second)) + } + fn resolved_at(&mut self, span: Self::Span, at: Self::Span) -> Self::Span { + span.with_ctxt(at.ctxt()) + } + fn source_text(&mut self, span: Self::Span) -> Option { + self.sess.source_map().span_to_snippet(span).ok() + } +} diff --git a/src/libsyntax_ext/Cargo.toml b/src/libsyntax_ext/Cargo.toml index 73310df305b..440873f3c2b 100644 --- a/src/libsyntax_ext/Cargo.toml +++ b/src/libsyntax_ext/Cargo.toml @@ -17,4 +17,5 @@ rustc_data_structures = { path = "../librustc_data_structures" } rustc_target = { path = "../librustc_target" } smallvec = { version = "0.6.7", features = ["union", "may_dangle"] } syntax = { path = "../libsyntax" } +syntax_expand = { path = "../libsyntax_expand" } syntax_pos = { path = "../libsyntax_pos" } diff --git a/src/libsyntax_ext/asm.rs b/src/libsyntax_ext/asm.rs index becbf6d60a0..8c9a34713ea 100644 --- a/src/libsyntax_ext/asm.rs +++ b/src/libsyntax_ext/asm.rs @@ -7,7 +7,7 @@ use rustc_data_structures::thin_vec::ThinVec; use errors::DiagnosticBuilder; use syntax::ast; -use syntax::ext::base::{self, *}; +use syntax_expand::base::{self, *}; use syntax::parse::token::{self, Token}; use syntax::ptr::P; use syntax::symbol::{kw, sym, Symbol}; diff --git a/src/libsyntax_ext/assert.rs b/src/libsyntax_ext/assert.rs index cbfe14fa439..f4d1f7fb09c 100644 --- a/src/libsyntax_ext/assert.rs +++ b/src/libsyntax_ext/assert.rs @@ -1,7 +1,7 @@ use errors::{Applicability, DiagnosticBuilder}; use syntax::ast::{self, *}; -use syntax::ext::base::*; +use syntax_expand::base::*; use syntax::parse::token::{self, TokenKind}; use syntax::parse::parser::Parser; use syntax::print::pprust; diff --git a/src/libsyntax_ext/cfg.rs b/src/libsyntax_ext/cfg.rs index 3c33baf95a5..9e693f29c5a 100644 --- a/src/libsyntax_ext/cfg.rs +++ b/src/libsyntax_ext/cfg.rs @@ -5,7 +5,7 @@ use errors::DiagnosticBuilder; use syntax::ast; -use syntax::ext::base::{self, *}; +use syntax_expand::base::{self, *}; use syntax::attr; use syntax::tokenstream::TokenStream; use syntax::parse::token; diff --git a/src/libsyntax_ext/cmdline_attrs.rs b/src/libsyntax_ext/cmdline_attrs.rs index 2c3159739e2..2d981526a39 100644 --- a/src/libsyntax_ext/cmdline_attrs.rs +++ b/src/libsyntax_ext/cmdline_attrs.rs @@ -2,9 +2,9 @@ use syntax::ast::{self, AttrItem, AttrStyle}; use syntax::attr::mk_attr; -use syntax::panictry; use syntax::parse::{self, token}; use syntax::sess::ParseSess; +use syntax_expand::panictry; use syntax_pos::FileName; pub fn inject(mut krate: ast::Crate, parse_sess: &ParseSess, attrs: &[String]) -> ast::Crate { diff --git a/src/libsyntax_ext/compile_error.rs b/src/libsyntax_ext/compile_error.rs index 24f3a66d4ae..cd7f78e9e34 100644 --- a/src/libsyntax_ext/compile_error.rs +++ b/src/libsyntax_ext/compile_error.rs @@ -1,6 +1,6 @@ // The compiler code necessary to support the compile_error! extension. -use syntax::ext::base::{self, *}; +use syntax_expand::base::{self, *}; use syntax_pos::Span; use syntax::tokenstream::TokenStream; diff --git a/src/libsyntax_ext/concat.rs b/src/libsyntax_ext/concat.rs index 790fdad5b3f..47bade698a8 100644 --- a/src/libsyntax_ext/concat.rs +++ b/src/libsyntax_ext/concat.rs @@ -1,5 +1,5 @@ use syntax::ast; -use syntax::ext::base::{self, DummyResult}; +use syntax_expand::base::{self, DummyResult}; use syntax::symbol::Symbol; use syntax::tokenstream::TokenStream; diff --git a/src/libsyntax_ext/concat_idents.rs b/src/libsyntax_ext/concat_idents.rs index f6747658c07..a132a4136ea 100644 --- a/src/libsyntax_ext/concat_idents.rs +++ b/src/libsyntax_ext/concat_idents.rs @@ -1,7 +1,7 @@ use rustc_data_structures::thin_vec::ThinVec; use syntax::ast; -use syntax::ext::base::{self, *}; +use syntax_expand::base::{self, *}; use syntax::parse::token::{self, Token}; use syntax::ptr::P; use syntax_pos::Span; diff --git a/src/libsyntax_ext/deriving/bounds.rs b/src/libsyntax_ext/deriving/bounds.rs index d5b8a00c75b..6a9b7092024 100644 --- a/src/libsyntax_ext/deriving/bounds.rs +++ b/src/libsyntax_ext/deriving/bounds.rs @@ -3,7 +3,7 @@ use crate::deriving::generic::*; use crate::deriving::generic::ty::*; use syntax::ast::MetaItem; -use syntax::ext::base::{Annotatable, ExtCtxt}; +use syntax_expand::base::{Annotatable, ExtCtxt}; use syntax_pos::Span; pub fn expand_deriving_copy(cx: &mut ExtCtxt<'_>, diff --git a/src/libsyntax_ext/deriving/clone.rs b/src/libsyntax_ext/deriving/clone.rs index 9ef2c033b07..eb7d480aa98 100644 --- a/src/libsyntax_ext/deriving/clone.rs +++ b/src/libsyntax_ext/deriving/clone.rs @@ -3,7 +3,7 @@ use crate::deriving::generic::*; use crate::deriving::generic::ty::*; use syntax::ast::{self, Expr, GenericArg, Generics, ItemKind, MetaItem, VariantData}; -use syntax::ext::base::{Annotatable, ExtCtxt, SpecialDerives}; +use syntax_expand::base::{Annotatable, ExtCtxt, SpecialDerives}; use syntax::ptr::P; use syntax::symbol::{kw, sym, Symbol}; use syntax_pos::Span; diff --git a/src/libsyntax_ext/deriving/cmp/eq.rs b/src/libsyntax_ext/deriving/cmp/eq.rs index c92339dd2fb..92721dab878 100644 --- a/src/libsyntax_ext/deriving/cmp/eq.rs +++ b/src/libsyntax_ext/deriving/cmp/eq.rs @@ -3,7 +3,7 @@ use crate::deriving::generic::*; use crate::deriving::generic::ty::*; use syntax::ast::{self, Ident, Expr, MetaItem, GenericArg}; -use syntax::ext::base::{Annotatable, ExtCtxt, SpecialDerives}; +use syntax_expand::base::{Annotatable, ExtCtxt, SpecialDerives}; use syntax::ptr::P; use syntax::symbol::{sym, Symbol}; use syntax_pos::Span; diff --git a/src/libsyntax_ext/deriving/cmp/ord.rs b/src/libsyntax_ext/deriving/cmp/ord.rs index 1f4f5aa3709..3eeed95aff7 100644 --- a/src/libsyntax_ext/deriving/cmp/ord.rs +++ b/src/libsyntax_ext/deriving/cmp/ord.rs @@ -3,7 +3,7 @@ use crate::deriving::generic::*; use crate::deriving::generic::ty::*; use syntax::ast::{self, Expr, MetaItem}; -use syntax::ext::base::{Annotatable, ExtCtxt}; +use syntax_expand::base::{Annotatable, ExtCtxt}; use syntax::ptr::P; use syntax::symbol::sym; use syntax_pos::Span; diff --git a/src/libsyntax_ext/deriving/cmp/partial_eq.rs b/src/libsyntax_ext/deriving/cmp/partial_eq.rs index 91e1e80e4fb..1615d991792 100644 --- a/src/libsyntax_ext/deriving/cmp/partial_eq.rs +++ b/src/libsyntax_ext/deriving/cmp/partial_eq.rs @@ -3,7 +3,7 @@ use crate::deriving::generic::*; use crate::deriving::generic::ty::*; use syntax::ast::{BinOpKind, Expr, MetaItem}; -use syntax::ext::base::{Annotatable, ExtCtxt, SpecialDerives}; +use syntax_expand::base::{Annotatable, ExtCtxt, SpecialDerives}; use syntax::ptr::P; use syntax::symbol::sym; use syntax_pos::Span; diff --git a/src/libsyntax_ext/deriving/cmp/partial_ord.rs b/src/libsyntax_ext/deriving/cmp/partial_ord.rs index 13d63aaf2a8..af8aacc6eb9 100644 --- a/src/libsyntax_ext/deriving/cmp/partial_ord.rs +++ b/src/libsyntax_ext/deriving/cmp/partial_ord.rs @@ -5,7 +5,7 @@ use crate::deriving::generic::*; use crate::deriving::generic::ty::*; use syntax::ast::{self, BinOpKind, Expr, MetaItem}; -use syntax::ext::base::{Annotatable, ExtCtxt}; +use syntax_expand::base::{Annotatable, ExtCtxt}; use syntax::ptr::P; use syntax::symbol::{sym, Symbol}; use syntax_pos::Span; diff --git a/src/libsyntax_ext/deriving/debug.rs b/src/libsyntax_ext/deriving/debug.rs index 003c2423576..35298211e4d 100644 --- a/src/libsyntax_ext/deriving/debug.rs +++ b/src/libsyntax_ext/deriving/debug.rs @@ -6,7 +6,7 @@ use rustc_data_structures::thin_vec::ThinVec; use syntax::ast::{self, Ident}; use syntax::ast::{Expr, MetaItem}; -use syntax::ext::base::{Annotatable, ExtCtxt}; +use syntax_expand::base::{Annotatable, ExtCtxt}; use syntax::ptr::P; use syntax::symbol::sym; use syntax_pos::{DUMMY_SP, Span}; diff --git a/src/libsyntax_ext/deriving/decodable.rs b/src/libsyntax_ext/deriving/decodable.rs index cde72abbdef..3a0379a0eb0 100644 --- a/src/libsyntax_ext/deriving/decodable.rs +++ b/src/libsyntax_ext/deriving/decodable.rs @@ -6,7 +6,7 @@ use crate::deriving::generic::ty::*; use syntax::ast; use syntax::ast::{Expr, MetaItem, Mutability}; -use syntax::ext::base::{Annotatable, ExtCtxt}; +use syntax_expand::base::{Annotatable, ExtCtxt}; use syntax::ptr::P; use syntax::symbol::Symbol; use syntax_pos::Span; diff --git a/src/libsyntax_ext/deriving/default.rs b/src/libsyntax_ext/deriving/default.rs index 2fdea10b76f..6176791c31b 100644 --- a/src/libsyntax_ext/deriving/default.rs +++ b/src/libsyntax_ext/deriving/default.rs @@ -3,7 +3,7 @@ use crate::deriving::generic::*; use crate::deriving::generic::ty::*; use syntax::ast::{Expr, MetaItem}; -use syntax::ext::base::{Annotatable, DummyResult, ExtCtxt}; +use syntax_expand::base::{Annotatable, DummyResult, ExtCtxt}; use syntax::ptr::P; use syntax::symbol::{kw, sym}; use syntax::span_err; diff --git a/src/libsyntax_ext/deriving/encodable.rs b/src/libsyntax_ext/deriving/encodable.rs index 655d3bb7c4a..2105946b666 100644 --- a/src/libsyntax_ext/deriving/encodable.rs +++ b/src/libsyntax_ext/deriving/encodable.rs @@ -90,7 +90,7 @@ use crate::deriving::generic::*; use crate::deriving::generic::ty::*; use syntax::ast::{Expr, ExprKind, MetaItem, Mutability}; -use syntax::ext::base::{Annotatable, ExtCtxt}; +use syntax_expand::base::{Annotatable, ExtCtxt}; use syntax::ptr::P; use syntax::symbol::Symbol; use syntax_pos::Span; diff --git a/src/libsyntax_ext/deriving/generic/mod.rs b/src/libsyntax_ext/deriving/generic/mod.rs index 9c3ec06d59a..1886a5154b7 100644 --- a/src/libsyntax_ext/deriving/generic/mod.rs +++ b/src/libsyntax_ext/deriving/generic/mod.rs @@ -186,12 +186,12 @@ use rustc_target::spec::abi::Abi; use syntax::ast::{self, BinOpKind, EnumDef, Expr, Generics, Ident, PatKind}; use syntax::ast::{VariantData, GenericParamKind, GenericArg}; use syntax::attr; -use syntax::ext::base::{Annotatable, ExtCtxt, SpecialDerives}; use syntax::source_map::respan; use syntax::util::map_in_place::MapInPlace; use syntax::ptr::P; use syntax::sess::ParseSess; use syntax::symbol::{Symbol, kw, sym}; +use syntax_expand::base::{Annotatable, ExtCtxt, SpecialDerives}; use syntax_pos::{Span}; use ty::{LifetimeBounds, Path, Ptr, PtrTy, Self_, Ty}; diff --git a/src/libsyntax_ext/deriving/generic/ty.rs b/src/libsyntax_ext/deriving/generic/ty.rs index 6ae02a5cab1..607746597a5 100644 --- a/src/libsyntax_ext/deriving/generic/ty.rs +++ b/src/libsyntax_ext/deriving/generic/ty.rs @@ -5,7 +5,7 @@ pub use PtrTy::*; pub use Ty::*; use syntax::ast::{self, Expr, GenericParamKind, Generics, Ident, SelfKind, GenericArg}; -use syntax::ext::base::ExtCtxt; +use syntax_expand::base::ExtCtxt; use syntax::source_map::{respan, DUMMY_SP}; use syntax::ptr::P; use syntax_pos::Span; diff --git a/src/libsyntax_ext/deriving/hash.rs b/src/libsyntax_ext/deriving/hash.rs index 2fc594abd70..fe9ef78bb1b 100644 --- a/src/libsyntax_ext/deriving/hash.rs +++ b/src/libsyntax_ext/deriving/hash.rs @@ -3,7 +3,7 @@ use crate::deriving::generic::*; use crate::deriving::generic::ty::*; use syntax::ast::{Expr, MetaItem, Mutability}; -use syntax::ext::base::{Annotatable, ExtCtxt}; +use syntax_expand::base::{Annotatable, ExtCtxt}; use syntax::ptr::P; use syntax::symbol::sym; use syntax_pos::Span; diff --git a/src/libsyntax_ext/deriving/mod.rs b/src/libsyntax_ext/deriving/mod.rs index 60b6eba7a4b..f0471a857dc 100644 --- a/src/libsyntax_ext/deriving/mod.rs +++ b/src/libsyntax_ext/deriving/mod.rs @@ -1,7 +1,7 @@ //! The compiler code necessary to implement the `#[derive]` extensions. use syntax::ast::{self, MetaItem}; -use syntax::ext::base::{Annotatable, ExtCtxt, MultiItemModifier}; +use syntax_expand::base::{Annotatable, ExtCtxt, MultiItemModifier}; use syntax::ptr::P; use syntax::symbol::{Symbol, sym}; use syntax_pos::Span; diff --git a/src/libsyntax_ext/env.rs b/src/libsyntax_ext/env.rs index 02757bf6b16..58fe56bd235 100644 --- a/src/libsyntax_ext/env.rs +++ b/src/libsyntax_ext/env.rs @@ -4,7 +4,7 @@ // use syntax::ast::{self, Ident, GenericArg}; -use syntax::ext::base::{self, *}; +use syntax_expand::base::{self, *}; use syntax::symbol::{kw, sym, Symbol}; use syntax_pos::Span; use syntax::tokenstream::TokenStream; diff --git a/src/libsyntax_ext/format.rs b/src/libsyntax_ext/format.rs index 8fc64021b51..45d9f79c28f 100644 --- a/src/libsyntax_ext/format.rs +++ b/src/libsyntax_ext/format.rs @@ -8,7 +8,7 @@ use errors::Applicability; use errors::pluralise; use syntax::ast; -use syntax::ext::base::{self, *}; +use syntax_expand::base::{self, *}; use syntax::parse::token; use syntax::ptr::P; use syntax::symbol::{Symbol, sym}; diff --git a/src/libsyntax_ext/global_allocator.rs b/src/libsyntax_ext/global_allocator.rs index f014b4f4f9f..75dda9535b3 100644 --- a/src/libsyntax_ext/global_allocator.rs +++ b/src/libsyntax_ext/global_allocator.rs @@ -2,8 +2,8 @@ use crate::util::check_builtin_macro_attribute; use syntax::ast::{ItemKind, Mutability, Stmt, Ty, TyKind, Unsafety}; use syntax::ast::{self, Param, Attribute, Expr, FnHeader, Generics, Ident}; -use syntax::ext::allocator::{AllocatorKind, AllocatorMethod, AllocatorTy, ALLOCATOR_METHODS}; -use syntax::ext::base::{Annotatable, ExtCtxt}; +use syntax_expand::allocator::{AllocatorKind, AllocatorMethod, AllocatorTy, ALLOCATOR_METHODS}; +use syntax_expand::base::{Annotatable, ExtCtxt}; use syntax::ptr::P; use syntax::symbol::{kw, sym, Symbol}; use syntax_pos::Span; diff --git a/src/libsyntax_ext/global_asm.rs b/src/libsyntax_ext/global_asm.rs index 72fb5b47c21..879ae1e4215 100644 --- a/src/libsyntax_ext/global_asm.rs +++ b/src/libsyntax_ext/global_asm.rs @@ -12,7 +12,7 @@ use errors::DiagnosticBuilder; use syntax::ast; use syntax::source_map::respan; -use syntax::ext::base::{self, *}; +use syntax_expand::base::{self, *}; use syntax::parse::token; use syntax::ptr::P; use syntax_pos::Span; diff --git a/src/libsyntax_ext/lib.rs b/src/libsyntax_ext/lib.rs index 5ba76a52350..5516f276422 100644 --- a/src/libsyntax_ext/lib.rs +++ b/src/libsyntax_ext/lib.rs @@ -15,9 +15,9 @@ use crate::deriving::*; use syntax::ast::Ident; use syntax::edition::Edition; -use syntax::ext::base::{SyntaxExtension, SyntaxExtensionKind, MacroExpanderFn}; -use syntax::ext::proc_macro::BangProcMacro; use syntax::symbol::sym; +use syntax_expand::base::{Resolver, SyntaxExtension, SyntaxExtensionKind, MacroExpanderFn}; +use syntax_expand::proc_macro::BangProcMacro; mod error_codes; @@ -45,7 +45,7 @@ pub mod proc_macro_harness; pub mod standard_library_imports; pub mod test_harness; -pub fn register_builtin_macros(resolver: &mut dyn syntax::ext::base::Resolver, edition: Edition) { +pub fn register_builtin_macros(resolver: &mut dyn Resolver, edition: Edition) { let mut register = |name, kind| resolver.register_builtin_macro( Ident::with_dummy_span(name), SyntaxExtension { is_builtin: true, ..SyntaxExtension::default(kind, edition) diff --git a/src/libsyntax_ext/log_syntax.rs b/src/libsyntax_ext/log_syntax.rs index 92130bfaf68..2202375e5e7 100644 --- a/src/libsyntax_ext/log_syntax.rs +++ b/src/libsyntax_ext/log_syntax.rs @@ -1,4 +1,4 @@ -use syntax::ext::base; +use syntax_expand::base; use syntax::print; use syntax::tokenstream::TokenStream; use syntax_pos; diff --git a/src/libsyntax_ext/plugin_macro_defs.rs b/src/libsyntax_ext/plugin_macro_defs.rs index 62c7e188eba..1ca9422eb9d 100644 --- a/src/libsyntax_ext/plugin_macro_defs.rs +++ b/src/libsyntax_ext/plugin_macro_defs.rs @@ -4,7 +4,7 @@ use syntax::ast::*; use syntax::attr; use syntax::edition::Edition; -use syntax::ext::base::{Resolver, NamedSyntaxExtension}; +use syntax_expand::base::{Resolver, NamedSyntaxExtension}; use syntax::parse::token; use syntax::ptr::P; use syntax::source_map::respan; diff --git a/src/libsyntax_ext/proc_macro_harness.rs b/src/libsyntax_ext/proc_macro_harness.rs index 1b7068818b0..96d0c3fcab1 100644 --- a/src/libsyntax_ext/proc_macro_harness.rs +++ b/src/libsyntax_ext/proc_macro_harness.rs @@ -3,14 +3,14 @@ use std::mem; use smallvec::smallvec; use syntax::ast::{self, Ident}; use syntax::attr; -use syntax::ext::base::ExtCtxt; -use syntax::ext::expand::{AstFragment, ExpansionConfig}; -use syntax::ext::proc_macro::is_proc_macro_attr; use syntax::print::pprust; use syntax::ptr::P; use syntax::sess::ParseSess; use syntax::symbol::{kw, sym}; use syntax::visit::{self, Visitor}; +use syntax_expand::base::{ExtCtxt, Resolver}; +use syntax_expand::expand::{AstFragment, ExpansionConfig}; +use syntax_expand::proc_macro::is_proc_macro_attr; use syntax_pos::{Span, DUMMY_SP}; use syntax_pos::hygiene::AstPass; @@ -46,7 +46,7 @@ struct CollectProcMacros<'a> { } pub fn inject(sess: &ParseSess, - resolver: &mut dyn (::syntax::ext::base::Resolver), + resolver: &mut dyn Resolver, mut krate: ast::Crate, is_proc_macro_crate: bool, has_proc_macro_decls: bool, diff --git a/src/libsyntax_ext/source_util.rs b/src/libsyntax_ext/source_util.rs index f74507dcc21..438e199ebdb 100644 --- a/src/libsyntax_ext/source_util.rs +++ b/src/libsyntax_ext/source_util.rs @@ -1,5 +1,6 @@ -use syntax::{ast, panictry}; -use syntax::ext::base::{self, *}; +use syntax_expand::panictry; +use syntax_expand::base::{self, *}; +use syntax::ast; use syntax::parse::{self, token, DirectoryOwnership}; use syntax::print::pprust; use syntax::ptr::P; diff --git a/src/libsyntax_ext/standard_library_imports.rs b/src/libsyntax_ext/standard_library_imports.rs index 4f17acf2d0f..fd27a218906 100644 --- a/src/libsyntax_ext/standard_library_imports.rs +++ b/src/libsyntax_ext/standard_library_imports.rs @@ -1,11 +1,11 @@ use syntax::{ast, attr}; use syntax::edition::Edition; -use syntax::ext::expand::ExpansionConfig; -use syntax::ext::hygiene::AstPass; -use syntax::ext::base::{ExtCtxt, Resolver}; use syntax::ptr::P; use syntax::sess::ParseSess; use syntax::symbol::{Ident, Symbol, kw, sym}; +use syntax_expand::expand::ExpansionConfig; +use syntax_expand::hygiene::AstPass; +use syntax_expand::base::{ExtCtxt, Resolver}; use syntax_pos::DUMMY_SP; pub fn inject( diff --git a/src/libsyntax_ext/test.rs b/src/libsyntax_ext/test.rs index 4d7441ca0b6..b0da413d63a 100644 --- a/src/libsyntax_ext/test.rs +++ b/src/libsyntax_ext/test.rs @@ -5,7 +5,7 @@ use crate::util::check_builtin_macro_attribute; use syntax::ast; use syntax::attr; -use syntax::ext::base::*; +use syntax_expand::base::*; use syntax::print::pprust; use syntax::source_map::respan; use syntax::symbol::{Symbol, sym}; diff --git a/src/libsyntax_ext/test_harness.rs b/src/libsyntax_ext/test_harness.rs index 0bb279c0cb0..33d41a7f53e 100644 --- a/src/libsyntax_ext/test_harness.rs +++ b/src/libsyntax_ext/test_harness.rs @@ -6,8 +6,8 @@ use rustc_target::spec::PanicStrategy; use syntax::ast::{self, Ident}; use syntax::attr; use syntax::entry::{self, EntryPointType}; -use syntax::ext::base::{ExtCtxt, Resolver}; -use syntax::ext::expand::{AstFragment, ExpansionConfig}; +use syntax_expand::base::{ExtCtxt, Resolver}; +use syntax_expand::expand::{AstFragment, ExpansionConfig}; use syntax::feature_gate::Features; use syntax::mut_visit::{*, ExpectOne}; use syntax::ptr::P; diff --git a/src/libsyntax_ext/trace_macros.rs b/src/libsyntax_ext/trace_macros.rs index d83c24046d9..dbf96d3b561 100644 --- a/src/libsyntax_ext/trace_macros.rs +++ b/src/libsyntax_ext/trace_macros.rs @@ -1,4 +1,4 @@ -use syntax::ext::base::{self, ExtCtxt}; +use syntax_expand::base::{self, ExtCtxt}; use syntax::symbol::kw; use syntax_pos::Span; use syntax::tokenstream::{TokenTree, TokenStream}; diff --git a/src/libsyntax_ext/util.rs b/src/libsyntax_ext/util.rs index fbae68057c5..d84fe19b3ea 100644 --- a/src/libsyntax_ext/util.rs +++ b/src/libsyntax_ext/util.rs @@ -1,7 +1,7 @@ use syntax_pos::Symbol; use syntax::ast::MetaItem; use syntax::attr::{check_builtin_attribute, AttributeTemplate}; -use syntax::ext::base::ExtCtxt; +use syntax_expand::base::ExtCtxt; pub fn check_builtin_macro_attribute(ecx: &ExtCtxt<'_>, meta_item: &MetaItem, name: Symbol) { // All the built-in macro attributes are "words" at the moment. -- cgit 1.4.1-3-g733a5